db add
This commit is contained in:
parent
6d3ed58526
commit
661d5e9880
3 changed files with 18 additions and 1 deletions
|
@ -1,8 +1,9 @@
|
||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { ApiModule } from './api/api.module';
|
import { ApiModule } from './api/api.module';
|
||||||
import { CrawlerModule } from './crawler/crawler.module';
|
import { CrawlerModule } from './crawler/crawler.module';
|
||||||
|
import { DbModule } from './db/db.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [ApiModule, CrawlerModule]
|
imports: [ApiModule, CrawlerModule, DbModule]
|
||||||
})
|
})
|
||||||
export class AppModule {}
|
export class AppModule {}
|
||||||
|
|
|
@ -42,6 +42,18 @@ export class CrawlerService {
|
||||||
// URLS //
|
// URLS //
|
||||||
const urls = await page.$$eval('a', links => links.map(link => link.href));
|
const urls = await page.$$eval('a', links => links.map(link => link.href));
|
||||||
await browser.close();
|
await browser.close();
|
||||||
|
|
||||||
|
// // save to db
|
||||||
|
// this.dbService.insert({
|
||||||
|
// url,
|
||||||
|
// domain,
|
||||||
|
// cssSheetsLocation,
|
||||||
|
// scriptsSheetsLocation,
|
||||||
|
// screenshotBuffer,
|
||||||
|
// urls
|
||||||
|
// }, 'crawler');
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async downloadFiles(urls: string[], path: string) {
|
async downloadFiles(urls: string[], path: string) {
|
||||||
|
|
|
@ -5,4 +5,8 @@ export class DbService {
|
||||||
constructor() {
|
constructor() {
|
||||||
console.log(`DbService constructor`);
|
console.log(`DbService constructor`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
insert(data, collection) {
|
||||||
|
console.log(`DbService insert`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue