done

.dockerignore

@@ -0,0 +1 @@
+node_modules

.gitignore

@@ -4,7 +4,7 @@
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
 #poppeteer's output
-crawler_assets/**/*
+sites_assets/**/*
 
 
 # User-specific stuff:

Dockerfile

@@ -1,13 +1,14 @@
-FROM ubuntu:20.04 AS base
+FROM node:14-alpine AS base
 
 # Set non-interactive mode
-ENV DEBIAN_FRONTEND noninteractive
+ENV DEBIAN_FRONTEND=noninteractive
 ENV NODE_ENV=production
 
-# Install required packages
+# Install chromium
-RUN apt-get update && \
+RUN apk add --no-cache chromium chromium-chromedriver
-    curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
+
-    apt-get install -y nodejs chromium-browser
+# Create a symbolic link for google-chrome
+RUN ln -s /usr/bin/chromium-browser /usr/bin/google-chrome
 
 # Set the working directory
 WORKDIR /app
@@ -16,10 +17,11 @@ WORKDIR /app
 COPY package*.json ./
 
 # Install dependencies
-RUN npm install
+RUN npm install -g npm@9.6.3 && \
+    npm install
 
 # Copy the source code
 COPY . .
 
 # Start the application
-CMD ["npm", "run", "start:prod"]
+CMD ["npm", "run", "start"]

README.md

@@ -6,9 +6,7 @@ The crawler is a simple crawler that crawls the web and stores the results in a
 ## Crawler ##
 
 
-
 ### Usage ###
-
 Post a JSON object to the crawler with the following format:
 
   `domain.com/crawl`
@@ -18,3 +16,19 @@ Post a JSON object to the crawler with the following format:
 
 The crawler will then crawl the given url and store the results in a database and assets in a file system
 `crawler_assests/www.example.com/`.
+
+
+# API # 
+
+The API is a simple API that serves the results of the crawler.
+
+# Routes # 
+  ## GET ## 
+    /sites - Returns a list of all sites
+    /sites/:id - Returns the site object for the given site Id
+    sites/domain/:domain - Returns the domain object for the given domain
+  ## DELETE ##
+    /sites/:id - Deletes the site object for the given site Id
+    sites/domain/:domain - Deletes the domain object for the given domain
+  ## Post ## 
+    sites/:id - Updates the site object for the given site Id

docker-compose.yaml

@@ -5,17 +5,17 @@ services:
     ports:
       - '3000:3000'
     depends_on:
-      - db
+      - mongo
     environment:
-      - MONGO_URL=mongodb://db:27017/nestjs
+      - MONGO_URL=mongodb://mongo:27017/mydatabase
     networks:
       - appnet
-  db:
+  mongo:
     image: mongo
     environment:
-      - MONGO_INITDB_DATABASE=nestjs
+      - MONGO_INITDB_DATABASE=mydatabase
-      - MONGO_INITDB_ROOT_USERNAME=admin
+      - MONGO_INITDB_ROOT_USERNAME=akamai
-      - MONGO_INITDB_ROOT_PASSWORD=adminpassword
+      - MONGO_INITDB_ROOT_PASSWORD=password
     volumes:
       - dbdata:/data/db
     networks:

package.json

@@ -25,13 +25,14 @@
     "@nestjs/platform-express": "^9.0.0",
     "axios": "^1.3.5",
     "dotenv": "^16.0.3",
+    "minimatch": "^9.0.0",
     "mongoose": "^7.0.4",
     "puppeteer": "^19.9.1",
     "reflect-metadata": "^0.1.13",
+    "@nestjs/cli": "^9.0.0",
     "rxjs": "^7.5.5"
   },
   "devDependencies": {
-    "@nestjs/cli": "^9.0.0",
     "@nestjs/schematics": "^9.0.0",
     "@nestjs/testing": "^9.0.0",
     "@types/express": "^4.17.13",

src/api/api.controller.ts

@@ -1,21 +1,68 @@
-import { Body, Controller, Post } from '@nestjs/common';
+import { Body, Controller, Delete, Get, Param, Post } from '@nestjs/common';
 import { CrawlerService } from '../crawler/crawler.service';
-import { DbService } from '../db/db.service';
+import { InjectModel } from '@nestjs/mongoose';
+import { Model } from 'mongoose';
+import { Site } from 'src/interfaces/site.interface';
 
 @Controller('/')
 export class ApiController {
-  constructor(private crawlerService: CrawlerService, private DbService: DbService) {}
+
+  constructor(private crawlerService: CrawlerService, @InjectModel('Site') private readonly siteModel: Model<Site> ) {}
+  
   @Post('crawl')
   async crawl(@Body() body: { url: string }) {
+
     const results = this.crawlerService.crawl(body.url);
     results.then((data) => {
-      console.log(data)
+      console.log("Done crawling !", data);
-      this.DbService.insert(data, 'crawler');
+      const newSite = new this.siteModel(data);
+      newSite.save().then((result) => {
+        console.log("Site saved !", result);
+      }).catch((err) => {
+        console.log("Error saving site !", err.message);
+      });
     }).catch((err) => {
+      console.log("** Error crawling ! **", err);
       console.log(err);
     });
     return {
-      message: 'Crawling in progress'
+      message: 'Got your request for ' + body.url
     }
   }
+
+  // Get all
+  @Get('sites')
+  async getSites() {
+    const sites = await this.siteModel.find().exec();
+    return sites || {};
+  }
+
+  // Get by id
+  @Get('sites/:id')
+  async getSite(@Param('id') id: string) {
+    const site = await this.siteModel.findById(id).exec();
+    return site || {};
+  }
+
+  // Get by domain
+  @Get('sites/domain/:domain')
+  async getSiteByDomain(@Param('domain') domain: string) {
+    const site = await this.siteModel.findOne({ domain }).exec();
+    return site || {};
+  }
+
+  // Delete by domain
+  @Delete('sites/domain/:domain')
+  async deleteSiteByDomain(@Param('domain') domain: string) {
+    const site = await this.siteModel.findOneAndDelete({ domain }).exec();
+    return site || {};
+  }
+
+  // Delete by id
+  @Delete('sites/:id')
+  async deleteSite(@Param('id') id: string) {
+    const site = await this.siteModel.findByIdAndDelete(id).exec();
+    return site || {};
+  }
+  
 }

src/api/api.module.ts

@@ -1,11 +1,13 @@
 import { Module } from '@nestjs/common';
 import { ApiController } from './api.controller';
 import { CrawlerService } from '../crawler/crawler.service';
-import { DbService } from '../db/db.service';
+import { SitesSchema } from '../schema/sites.schema';
+import { MongooseModule } from '@nestjs/mongoose';
 
 @Module({
+  imports: [MongooseModule.forFeature([{ name: 'Site', schema: SitesSchema }])],
   controllers: [ApiController],
-  providers: [CrawlerService, DbService]
+  providers: [CrawlerService]
 
 })
 export class ApiModule {}

src/app.module.ts

@@ -1,9 +1,20 @@
 import { Module } from '@nestjs/common';
+import { MongooseModule } from '@nestjs/mongoose';
 import { ApiModule } from './api/api.module';
 import { CrawlerModule } from './crawler/crawler.module';
-import { DbModule } from './db/db.module';
+const dotenv = require('dotenv');
+
+dotenv.config();
 
 @Module({
-  imports: [ApiModule, CrawlerModule, DbModule]
+  imports: [
+    MongooseModule.forRoot(process.env.MONGO_URL, {
+      useNewUrlParser: true,
+      useUnifiedTopology: true,
+    }),
+    ApiModule,
+    CrawlerModule
+  ]
+
 })
-export class AppModule {}
+export class AppModule { }

src/crawler/crawler.module.ts

@@ -1,9 +1,7 @@
 import { Module } from '@nestjs/common';
 import { CrawlerService } from './crawler.service';
-import { DbModule } from '../db/db.module';
 
 @Module({
-  imports: [DbModule],
   providers: [CrawlerService]
 })
 export class CrawlerModule {}

src/crawler/crawler.service.ts

@@ -4,21 +4,31 @@ import * as fs from 'fs';
 import puppeteer from 'puppeteer';
 import { URL } from 'url';
 import axios from 'axios';
-
+import { Site } from '../interfaces/site.interface';
-const environment = process.env.NODE_ENV || 'development';
 
 @Injectable()
 export class CrawlerService {
-  async crawl(url: string): Promise<any> {
+
-    const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
+  constructor() {
+    if (!fs.existsSync('sites_assets')) {
+      mkdirSync('sites_assets');
+    }
+  }
+
+  async crawl(url: string): Promise<Site> {
+
+    console.log("start crawl website", url);
+    const browser = await puppeteer.launch({ executablePath: '/usr/bin/chromium-browser', headless: true, args: ['--no-sandbox'] });
     const page = await browser.newPage();
     const domain = this.extractDomain(url);
     await page.goto(url);
-    const directory = `crawler_assets/${domain}/`;
+    const directory = `sites_assets/${domain}/`;
     if (!fs.existsSync(directory)) {
       mkdirSync(directory);
     }
+
     // STYLESHEETS //
+    console.log("start stylesheets")
     const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
     let cssDir = `${directory}/css/`
     const cssSheetsLocation = await this.downloadFiles(stylesheetsUrls, cssDir);
@@ -34,18 +44,28 @@ export class CrawlerService {
 
     // SCREENSHOT //
     const screenshotBuffer: Buffer = await page.screenshot({ fullPage: true });
-    fs.writeFile(`${directory}screenshot.png`, screenshotBuffer, (err) => {
+    await new Promise((resolve, reject) => {
-      if (err) throw err;
+      fs.writeFile(`${directory}screenshot.png`, screenshotBuffer, (err) => {
-      // console.log(`Screenshot saved! ${directory}screenshot.png`);
+        if (err) {
+          reject(err);
+        } else {
+          resolve(true);
+        }
+      });
     });
     // SCREENSHOT //
 
     // URLS //
     const urls = await page.$$eval('a', links => links.map(link => link.href));
+    const urlsList = urls.filter((url) => url.startsWith('http'));
+    console.log(urlsList);
+
     await browser.close();
     return {
+      domain,
       cssSheetsLocation,
-      scriptsSheetsLocation
+      scriptsSheetsLocation,
+      urlsList
     }
   }
 
@@ -68,8 +88,7 @@ export class CrawlerService {
         if (fileLocation.length > 10) {
           fileLocation = fileLocation.substring(0, 10);
         }
-        console.log("fileLocation: " + fileLocation)
+        finalUrls.push(`${path}${fileLocation}`);
-        finalUrls.push(fileLocation);
         console.log(`Saving file ${path}${fileLocation}`);
         fs.writeFileSync(`${path}${fileLocation}`, content);
       })

src/db/db.module.ts

@@ -1,7 +0,0 @@
-import { Module } from '@nestjs/common';
-import { DbService } from './db.service';
-
-@Module({
-  providers: [DbService]
-})
-export class DbModule {}

src/db/db.service.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-@Injectable()
-export class DbService {
-  constructor() {
-    console.log(`DbService constructor`);
-  }
-
-  insert(data: {
-    cssSheetsLocation: string[];
-    scriptsSheetsLocation: string[];
-  }, collection: string)  {
-
-    console.log({data, collection});
-  
-  }
-}

src/interfaces/site.interface.ts

@@ -0,0 +1,6 @@
+export interface Site {
+  domain: string;
+  cssSheetsLocation: string[];
+  scriptsSheetsLocation: string[];
+  urlsList: string[];
+}

src/schema/sites.schema.ts

@@ -0,0 +1,21 @@
+import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
+import { Document } from 'mongoose';
+
+export type SitesDocument = Sites & Document;
+
+@Schema()
+export class Sites {
+  @Prop({ required: true, unique: true})
+  domain: string;
+  
+  @Prop()
+  cssSheetsLocation: string[];
+
+  @Prop()
+  scriptsSheetsLocation: string[];
+
+  @Prop({ required: true})
+  urlsList: string[];
+}
+
+export const SitesSchema = SchemaFactory.createForClass(Sites);