module + service is DB has added

This commit is contained in:
Kfir Dayan 2023-04-18 12:36:56 +03:00
parent eab79001ab
commit 6d3ed58526
4 changed files with 20 additions and 4 deletions

View file

@ -1,7 +1,9 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { CrawlerService } from './crawler.service'; import { CrawlerService } from './crawler.service';
import { DbModule } from '../db/db.module';
@Module({ @Module({
imports: [DbModule],
providers: [CrawlerService] providers: [CrawlerService]
}) })
export class CrawlerModule {} export class CrawlerModule {}

View file

@ -21,14 +21,14 @@ export class CrawlerService {
const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href)); const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
let cssDir = `${directory}/css/` let cssDir = `${directory}/css/`
const cssSheetsLocation = await this.downloadFiles(stylesheetsUrls, cssDir); const cssSheetsLocation = await this.downloadFiles(stylesheetsUrls, cssDir);
// console.log(`cssSheetsLocation: `, cssSheetsLocation); console.log(`cssSheetsLocation: `, cssSheetsLocation);
// STYLESHEETS // // STYLESHEETS //
// SCRIPTS // // SCRIPTS //
const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src)); const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src));
let scriptsDir = `${directory}/scripts/` let scriptsDir = `${directory}/scripts/`
const scriptsSheetsLocation = await this.downloadFiles(scriptsUrls, scriptsDir); const scriptsSheetsLocation = await this.downloadFiles(scriptsUrls, scriptsDir);
// console.log(`scriptsSheetsLocation: `, scriptsSheetsLocation) console.log(`scriptsSheetsLocation: `, scriptsSheetsLocation)
// SCRIPTS // // SCRIPTS //
// SCREENSHOT // // SCREENSHOT //
@ -65,12 +65,11 @@ export class CrawlerService {
} }
console.log("fileLocation: " + fileLocation) console.log("fileLocation: " + fileLocation)
finalUrls.push(fileLocation); finalUrls.push(fileLocation);
// console.log(`Saving file ${path}${url}`); console.log(`Saving file ${path}${fileLocation}`);
fs.writeFileSync(`${path}${fileLocation}`, content); fs.writeFileSync(`${path}${fileLocation}`, content);
}) })
); );
console.log(`finalUrls: `, finalUrls);
return finalUrls; return finalUrls;
} }

7
src/db/db.module.ts Normal file
View file

@ -0,0 +1,7 @@
import { Module } from '@nestjs/common';
import { DbService } from './db.service';
@Module({
providers: [DbService]
})
export class DbModule {}

8
src/db/db.service.ts Normal file
View file

@ -0,0 +1,8 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class DbService {
constructor() {
console.log(`DbService constructor`);
}
}