fixing docker issue
This commit is contained in:
parent
52a338b8a2
commit
245327de72
5 changed files with 17 additions and 3 deletions
|
@ -2,6 +2,7 @@ FROM ubuntu:20.04 AS base
|
||||||
|
|
||||||
# Set non-interactive mode
|
# Set non-interactive mode
|
||||||
ENV DEBIAN_FRONTEND noninteractive
|
ENV DEBIAN_FRONTEND noninteractive
|
||||||
|
ENV NODE_ENV=production
|
||||||
|
|
||||||
# Install required packages
|
# Install required packages
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
|
|
9
package-lock.json
generated
9
package-lock.json
generated
|
@ -14,6 +14,7 @@
|
||||||
"@nestjs/mongoose": "^9.2.2",
|
"@nestjs/mongoose": "^9.2.2",
|
||||||
"@nestjs/platform-express": "^9.0.0",
|
"@nestjs/platform-express": "^9.0.0",
|
||||||
"axios": "^1.3.5",
|
"axios": "^1.3.5",
|
||||||
|
"dotenv": "^16.0.3",
|
||||||
"mongoose": "^7.0.4",
|
"mongoose": "^7.0.4",
|
||||||
"puppeteer": "^19.9.1",
|
"puppeteer": "^19.9.1",
|
||||||
"reflect-metadata": "^0.1.13",
|
"reflect-metadata": "^0.1.13",
|
||||||
|
@ -3643,6 +3644,14 @@
|
||||||
"node": ">=6.0.0"
|
"node": ">=6.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/dotenv": {
|
||||||
|
"version": "16.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
|
||||||
|
"integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/ee-first": {
|
"node_modules/ee-first": {
|
||||||
"version": "1.1.1",
|
"version": "1.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
|
||||||
|
|
|
@ -24,6 +24,7 @@
|
||||||
"@nestjs/mongoose": "^9.2.2",
|
"@nestjs/mongoose": "^9.2.2",
|
||||||
"@nestjs/platform-express": "^9.0.0",
|
"@nestjs/platform-express": "^9.0.0",
|
||||||
"axios": "^1.3.5",
|
"axios": "^1.3.5",
|
||||||
|
"dotenv": "^16.0.3",
|
||||||
"mongoose": "^7.0.4",
|
"mongoose": "^7.0.4",
|
||||||
"puppeteer": "^19.9.1",
|
"puppeteer": "^19.9.1",
|
||||||
"reflect-metadata": "^0.1.13",
|
"reflect-metadata": "^0.1.13",
|
||||||
|
|
|
@ -5,11 +5,12 @@ import puppeteer from 'puppeteer';
|
||||||
import { URL } from 'url';
|
import { URL } from 'url';
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
|
|
||||||
|
const environment = process.env.NODE_ENV || 'development';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class CrawlerService {
|
export class CrawlerService {
|
||||||
async crawl(url: string) {
|
async crawl(url: string): Promise<any> {
|
||||||
|
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
|
||||||
const browser = await puppeteer.launch();
|
|
||||||
const page = await browser.newPage();
|
const page = await browser.newPage();
|
||||||
const domain = this.extractDomain(url);
|
const domain = this.extractDomain(url);
|
||||||
await page.goto(url);
|
await page.goto(url);
|
||||||
|
|
|
@ -10,6 +10,8 @@ export class DbService {
|
||||||
cssSheetsLocation: string[];
|
cssSheetsLocation: string[];
|
||||||
scriptsSheetsLocation: string[];
|
scriptsSheetsLocation: string[];
|
||||||
}, collection: string) {
|
}, collection: string) {
|
||||||
|
|
||||||
console.log({data, collection});
|
console.log({data, collection});
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue