This commit is contained in:
Kfir Dayan 2023-04-18 13:23:34 +03:00
parent 6d3ed58526
commit 661d5e9880
3 changed files with 18 additions and 1 deletions

View file

@ -1,8 +1,9 @@
import { Module } from '@nestjs/common';
import { ApiModule } from './api/api.module';
import { CrawlerModule } from './crawler/crawler.module';
import { DbModule } from './db/db.module';
@Module({
imports: [ApiModule, CrawlerModule]
imports: [ApiModule, CrawlerModule, DbModule]
})
export class AppModule {}

View file

@ -42,6 +42,18 @@ export class CrawlerService {
// URLS //
const urls = await page.$$eval('a', links => links.map(link => link.href));
await browser.close();
// // save to db
// this.dbService.insert({
// url,
// domain,
// cssSheetsLocation,
// scriptsSheetsLocation,
// screenshotBuffer,
// urls
// }, 'crawler');
}
async downloadFiles(urls: string[], path: string) {

View file

@ -5,4 +5,8 @@ export class DbService {
constructor() {
console.log(`DbService constructor`);
}
insert(data, collection) {
console.log(`DbService insert`);
}
}