kfir/crawling-and-serving
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

pushing for testing

Browse source
This commit is contained in:
Kfir Dayan 2023-04-18 11:53:24 +03:00
parent b8965e2e3e
commit 1376b14e2a
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/crawler/crawler.service.ts
View file

@ -19,13 +19,15 @@ export class CrawlerService {
// STYLESHEETS // // STYLESHEETS //
const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href)); const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
let cssDir = `${directory}/css/` let cssDir = `${directory}/css/`
await this.downloadFiles(stylesheetsUrls, cssDir); const cssSheetsLocation = await this.downloadFiles(stylesheetsUrls, cssDir);
console.log(`cssSheetsLocation: `, cssSheetsLocation);
// STYLESHEETS // // STYLESHEETS //
// SCRIPTS // // SCRIPTS //
const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src)); const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src));
let scriptsDir = `${directory}/scripts/` let scriptsDir = `${directory}/scripts/`
await this.downloadFiles(scriptsUrls, scriptsDir); const scriptsSheetsLocation = await this.downloadFiles(scriptsUrls, scriptsDir);
console.log(`scriptsSheetsLocation: `, scriptsSheetsLocation)
// SCRIPTS // // SCRIPTS //
// SCREENSHOT // // SCREENSHOT //
@ -42,6 +44,7 @@ export class CrawlerService {
} }
async downloadFiles(urls: string[], path: string) { async downloadFiles(urls: string[], path: string) {
const finalUrls = [];
if (!fs.existsSync(path)) { if (!fs.existsSync(path)) {
mkdirSync(path); mkdirSync(path);
} }
@ -58,9 +61,12 @@ export class CrawlerService {
if(url.length > 10) { if(url.length > 10) {
url = url.substring(0, 10); url = url.substring(0, 10);
} }
finalUrls.push(url);
console.log(`Saving file ${path}${url}`); console.log(`Saving file ${path}${url}`);
fs.writeFileSync(`${path}${url}`, content); fs.writeFileSync(`${path}${url}`, content);
}); });
console.log(`finalUrls: `, finalUrls);
return finalUrls;
} }
extractDomain(urlString: string) { extractDomain(urlString: string) {