This commit is contained in:
Kfir Dayan 2023-04-18 13:23:34 +03:00
parent 6d3ed58526
commit 661d5e9880
3 changed files with 18 additions and 1 deletions

View file

@ -1,8 +1,9 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ApiModule } from './api/api.module'; import { ApiModule } from './api/api.module';
import { CrawlerModule } from './crawler/crawler.module'; import { CrawlerModule } from './crawler/crawler.module';
import { DbModule } from './db/db.module';
@Module({ @Module({
imports: [ApiModule, CrawlerModule] imports: [ApiModule, CrawlerModule, DbModule]
}) })
export class AppModule {} export class AppModule {}

View file

@ -42,6 +42,18 @@ export class CrawlerService {
// URLS // // URLS //
const urls = await page.$$eval('a', links => links.map(link => link.href)); const urls = await page.$$eval('a', links => links.map(link => link.href));
await browser.close(); await browser.close();
// // save to db
// this.dbService.insert({
// url,
// domain,
// cssSheetsLocation,
// scriptsSheetsLocation,
// screenshotBuffer,
// urls
// }, 'crawler');
} }
async downloadFiles(urls: string[], path: string) { async downloadFiles(urls: string[], path: string) {

View file

@ -5,4 +5,8 @@ export class DbService {
constructor() { constructor() {
console.log(`DbService constructor`); console.log(`DbService constructor`);
} }
insert(data, collection) {
console.log(`DbService insert`);
}
} }