.dockerignore

@@ -1 +0,0 @@
-node_modules

.gitignore

@@ -4,7 +4,7 @@
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
 #poppeteer's output
-sites_assets/**/*
+crawler_assets/**/*
 
 
 # User-specific stuff:

Dockerfile

@@ -1,14 +1,13 @@
-FROM node:14-alpine AS base
+FROM ubuntu:20.04 AS base
 
 # Set non-interactive mode
-ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND noninteractive
 ENV NODE_ENV=production
 
-# Install chromium
+# Install required packages
-RUN apk add --no-cache chromium chromium-chromedriver
+RUN apt-get update && \
-
+    curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
-# Create a symbolic link for google-chrome
+    apt-get install -y nodejs chromium-browser
-RUN ln -s /usr/bin/chromium-browser /usr/bin/google-chrome
 
 # Set the working directory
 WORKDIR /app
@@ -17,11 +16,10 @@ WORKDIR /app
 COPY package*.json ./
 
 # Install dependencies
-RUN npm install -g npm@9.6.3 && \
+RUN npm install
-    npm install
 
 # Copy the source code
 COPY . .
 
 # Start the application
-CMD ["npm", "run", "start"]
+CMD ["npm", "run", "start:prod"]

README.md

@@ -6,7 +6,9 @@ The crawler is a simple crawler that crawls the web and stores the results in a
 ## Crawler ##
 
 
+
 ### Usage ###
+
 Post a JSON object to the crawler with the following format:
 
   `domain.com/crawl`
@@ -16,19 +18,3 @@ Post a JSON object to the crawler with the following format:
 
 The crawler will then crawl the given url and store the results in a database and assets in a file system
 `crawler_assests/www.example.com/`.
-
-
-# API # 
-
-The API is a simple API that serves the results of the crawler.
-
-# Routes # 
-  ## GET ## 
-    /sites - Returns a list of all sites
-    /sites/:id - Returns the site object for the given site Id
-    sites/domain/:domain - Returns the domain object for the given domain
-  ## DELETE ##
-    /sites/:id - Deletes the site object for the given site Id
-    sites/domain/:domain - Deletes the domain object for the given domain
-  ## Post ## 
-    sites/:id - Updates the site object for the given site Id

docker-compose.yaml

@@ -5,17 +5,17 @@ services:
     ports:
       - '3000:3000'
     depends_on:
-      - mongo
+      - db
     environment:
-      - MONGO_URL=mongodb://mongo:27017/mydatabase
+      - MONGO_URL=mongodb://db:27017/nestjs
     networks:
       - appnet
-  mongo:
+  db:
     image: mongo
     environment:
-      - MONGO_INITDB_DATABASE=mydatabase
+      - MONGO_INITDB_DATABASE=nestjs
-      - MONGO_INITDB_ROOT_USERNAME=akamai
+      - MONGO_INITDB_ROOT_USERNAME=admin
-      - MONGO_INITDB_ROOT_PASSWORD=password
+      - MONGO_INITDB_ROOT_PASSWORD=adminpassword
     volumes:
       - dbdata:/data/db
     networks:

package.json

@@ -25,14 +25,13 @@
     "@nestjs/platform-express": "^9.0.0",
     "axios": "^1.3.5",
     "dotenv": "^16.0.3",
-    "minimatch": "^9.0.0",
     "mongoose": "^7.0.4",
     "puppeteer": "^19.9.1",
     "reflect-metadata": "^0.1.13",
-    "@nestjs/cli": "^9.0.0",
     "rxjs": "^7.5.5"
   },
   "devDependencies": {
+    "@nestjs/cli": "^9.0.0",
     "@nestjs/schematics": "^9.0.0",
     "@nestjs/testing": "^9.0.0",
     "@types/express": "^4.17.13",

src/api/api.controller.ts

@@ -1,68 +1,21 @@
-import { Body, Controller, Delete, Get, Param, Post } from '@nestjs/common';
+import { Body, Controller, Post } from '@nestjs/common';
 import { CrawlerService } from '../crawler/crawler.service';
-import { InjectModel } from '@nestjs/mongoose';
+import { DbService } from '../db/db.service';
-import { Model } from 'mongoose';
-import { Site } from 'src/interfaces/site.interface';
 
 @Controller('/')
 export class ApiController {
-
+  constructor(private crawlerService: CrawlerService, private DbService: DbService) {}
-  constructor(private crawlerService: CrawlerService, @InjectModel('Site') private readonly siteModel: Model<Site> ) {}
-  
   @Post('crawl')
   async crawl(@Body() body: { url: string }) {
-
     const results = this.crawlerService.crawl(body.url);
     results.then((data) => {
-      console.log("Done crawling !", data);
+      console.log(data)
-      const newSite = new this.siteModel(data);
+      this.DbService.insert(data, 'crawler');
-      newSite.save().then((result) => {
-        console.log("Site saved !", result);
     }).catch((err) => {
-        console.log("Error saving site !", err.message);
-      });
-    }).catch((err) => {
-      console.log("** Error crawling ! **", err);
       console.log(err);
     });
     return {
-      message: 'Got your request for ' + body.url
+      message: 'Crawling in progress'
     }
   }
-
-  // Get all
-  @Get('sites')
-  async getSites() {
-    const sites = await this.siteModel.find().exec();
-    return sites || {};
-  }
-
-  // Get by id
-  @Get('sites/:id')
-  async getSite(@Param('id') id: string) {
-    const site = await this.siteModel.findById(id).exec();
-    return site || {};
-  }
-
-  // Get by domain
-  @Get('sites/domain/:domain')
-  async getSiteByDomain(@Param('domain') domain: string) {
-    const site = await this.siteModel.findOne({ domain }).exec();
-    return site || {};
-  }
-
-  // Delete by domain
-  @Delete('sites/domain/:domain')
-  async deleteSiteByDomain(@Param('domain') domain: string) {
-    const site = await this.siteModel.findOneAndDelete({ domain }).exec();
-    return site || {};
-  }
-
-  // Delete by id
-  @Delete('sites/:id')
-  async deleteSite(@Param('id') id: string) {
-    const site = await this.siteModel.findByIdAndDelete(id).exec();
-    return site || {};
-  }
-  
 }

src/api/api.module.ts

@@ -1,13 +1,11 @@
 import { Module } from '@nestjs/common';
 import { ApiController } from './api.controller';
 import { CrawlerService } from '../crawler/crawler.service';
-import { SitesSchema } from '../schema/sites.schema';
+import { DbService } from '../db/db.service';
-import { MongooseModule } from '@nestjs/mongoose';
 
 @Module({
-  imports: [MongooseModule.forFeature([{ name: 'Site', schema: SitesSchema }])],
   controllers: [ApiController],
-  providers: [CrawlerService]
+  providers: [CrawlerService, DbService]
 
 })
 export class ApiModule {}

src/app.module.ts

@@ -1,20 +1,9 @@
 import { Module } from '@nestjs/common';
-import { MongooseModule } from '@nestjs/mongoose';
 import { ApiModule } from './api/api.module';
 import { CrawlerModule } from './crawler/crawler.module';
-const dotenv = require('dotenv');
+import { DbModule } from './db/db.module';
-
-dotenv.config();
 
 @Module({
-  imports: [
+  imports: [ApiModule, CrawlerModule, DbModule]
-    MongooseModule.forRoot(process.env.MONGO_URL, {
-      useNewUrlParser: true,
-      useUnifiedTopology: true,
-    }),
-    ApiModule,
-    CrawlerModule
-  ]
-
 })
-export class AppModule { }
+export class AppModule {}

src/crawler/crawler.module.ts

@@ -1,7 +1,9 @@
 import { Module } from '@nestjs/common';
 import { CrawlerService } from './crawler.service';
+import { DbModule } from '../db/db.module';
 
 @Module({
+  imports: [DbModule],
   providers: [CrawlerService]
 })
 export class CrawlerModule {}

src/crawler/crawler.service.ts

@@ -4,31 +4,21 @@ import * as fs from 'fs';
 import puppeteer from 'puppeteer';
 import { URL } from 'url';
 import axios from 'axios';
-import { Site } from '../interfaces/site.interface';
+
+const environment = process.env.NODE_ENV || 'development';
 
 @Injectable()
 export class CrawlerService {
-
+  async crawl(url: string): Promise<any> {
-  constructor() {
+    const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
-    if (!fs.existsSync('sites_assets')) {
-      mkdirSync('sites_assets');
-    }
-  }
-
-  async crawl(url: string): Promise<Site> {
-
-    console.log("start crawl website", url);
-    const browser = await puppeteer.launch({ executablePath: '/usr/bin/chromium-browser', headless: true, args: ['--no-sandbox'] });
     const page = await browser.newPage();
     const domain = this.extractDomain(url);
     await page.goto(url);
-    const directory = `sites_assets/${domain}/`;
+    const directory = `crawler_assets/${domain}/`;
     if (!fs.existsSync(directory)) {
       mkdirSync(directory);
     }
-
     // STYLESHEETS //
-    console.log("start stylesheets")
     const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
     let cssDir = `${directory}/css/`
     const cssSheetsLocation = await this.downloadFiles(stylesheetsUrls, cssDir);
@@ -44,28 +34,18 @@ export class CrawlerService {
 
     // SCREENSHOT //
     const screenshotBuffer: Buffer = await page.screenshot({ fullPage: true });
-    await new Promise((resolve, reject) => {
     fs.writeFile(`${directory}screenshot.png`, screenshotBuffer, (err) => {
-        if (err) {
+      if (err) throw err;
-          reject(err);
+      // console.log(`Screenshot saved! ${directory}screenshot.png`);
-        } else {
-          resolve(true);
-        }
-      });
     });
     // SCREENSHOT //
 
     // URLS //
     const urls = await page.$$eval('a', links => links.map(link => link.href));
-    const urlsList = urls.filter((url) => url.startsWith('http'));
-    console.log(urlsList);
-
     await browser.close();
     return {
-      domain,
       cssSheetsLocation,
-      scriptsSheetsLocation,
+      scriptsSheetsLocation
-      urlsList
     }
   }
 
@@ -88,7 +68,8 @@ export class CrawlerService {
         if (fileLocation.length > 10) {
           fileLocation = fileLocation.substring(0, 10);
         }
-        finalUrls.push(`${path}${fileLocation}`);
+        console.log("fileLocation: " + fileLocation)
+        finalUrls.push(fileLocation);
         console.log(`Saving file ${path}${fileLocation}`);
         fs.writeFileSync(`${path}${fileLocation}`, content);
       })

src/db/db.module.ts

@@ -0,0 +1,7 @@
+import { Module } from '@nestjs/common';
+import { DbService } from './db.service';
+
+@Module({
+  providers: [DbService]
+})
+export class DbModule {}

src/db/db.service.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class DbService {
+  constructor() {
+    console.log(`DbService constructor`);
+  }
+
+  insert(data: {
+    cssSheetsLocation: string[];
+    scriptsSheetsLocation: string[];
+  }, collection: string)  {
+
+    console.log({data, collection});
+  
+  }
+}

src/interfaces/site.interface.ts

@@ -1,6 +0,0 @@
-export interface Site {
-  domain: string;
-  cssSheetsLocation: string[];
-  scriptsSheetsLocation: string[];
-  urlsList: string[];
-}

src/schema/sites.schema.ts

@@ -1,21 +0,0 @@
-import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
-import { Document } from 'mongoose';
-
-export type SitesDocument = Sites & Document;
-
-@Schema()
-export class Sites {
-  @Prop({ required: true, unique: true})
-  domain: string;
-  
-  @Prop()
-  cssSheetsLocation: string[];
-
-  @Prop()
-  scriptsSheetsLocation: string[];
-
-  @Prop({ required: true})
-  urlsList: string[];
-}
-
-export const SitesSchema = SchemaFactory.createForClass(Sites);