diff --git a/Dockerfile b/Dockerfile index 8510f0f..c61ee52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM ubuntu:20.04 AS base # Set non-interactive mode ENV DEBIAN_FRONTEND noninteractive +ENV NODE_ENV=production # Install required packages RUN apt-get update && \ diff --git a/package-lock.json b/package-lock.json index a0da3f7..db1c873 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@nestjs/mongoose": "^9.2.2", "@nestjs/platform-express": "^9.0.0", "axios": "^1.3.5", + "dotenv": "^16.0.3", "mongoose": "^7.0.4", "puppeteer": "^19.9.1", "reflect-metadata": "^0.1.13", @@ -3643,6 +3644,14 @@ "node": ">=6.0.0" } }, + "node_modules/dotenv": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", + "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "engines": { + "node": ">=12" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", diff --git a/package.json b/package.json index 8b8e295..0d75941 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@nestjs/mongoose": "^9.2.2", "@nestjs/platform-express": "^9.0.0", "axios": "^1.3.5", + "dotenv": "^16.0.3", "mongoose": "^7.0.4", "puppeteer": "^19.9.1", "reflect-metadata": "^0.1.13", diff --git a/src/crawler/crawler.service.ts b/src/crawler/crawler.service.ts index 0deb083..283b4a6 100644 --- a/src/crawler/crawler.service.ts +++ b/src/crawler/crawler.service.ts @@ -5,11 +5,12 @@ import puppeteer from 'puppeteer'; import { URL } from 'url'; import axios from 'axios'; +const environment = process.env.NODE_ENV || 'development'; + @Injectable() export class CrawlerService { - async crawl(url: string) { - - const browser = await puppeteer.launch(); + async crawl(url: string): Promise { + const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] }); const page = await browser.newPage(); const domain = this.extractDomain(url); await page.goto(url); diff --git a/src/db/db.service.ts b/src/db/db.service.ts index 9d7c5a2..8dd86ff 100644 --- a/src/db/db.service.ts +++ b/src/db/db.service.ts @@ -10,6 +10,8 @@ export class DbService { cssSheetsLocation: string[]; scriptsSheetsLocation: string[]; }, collection: string) { + console.log({data, collection}); + } }