kfir/crawling-and-serving
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

reOrder the methods

Browse source
This commit is contained in:
Kfir Dayan 2023-04-18 11:15:22 +03:00
parent e4d982891f
commit babed7889b
1 changed files with 16 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/crawler/crawler.service.ts
View file

@ -25,31 +25,13 @@ export class CrawlerService {
// STYLESHEETS // // STYLESHEETS //
const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href)); const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
let cssDir = `${directory}/css/` let cssDir = `${directory}/css/`
if (!fs.existsSync(cssDir)) { await this.downloadFile(stylesheetsUrls, cssDir);
mkdirSync(cssDir);
}
stylesheetsUrls.forEach(async (stylesheetUrl) => {
if(!stylesheetUrl.startsWith('http')) return;
const response = await axios.get(stylesheetUrl);
const content = response.data;
fs.writeFileSync(`${cssDir}${stylesheetUrl.split('/').pop()}`, content);
});
// STYLESHEETS // // STYLESHEETS //
// SCRIPTS // // SCRIPTS //
const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src)); const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src));
let scriptsDir = `${directory}/scripts/` let scriptsDir = `${directory}/scripts/`
if (!fs.existsSync(scriptsDir)) { await this.downloadFile(scriptsUrls, scriptsDir);
mkdirSync(scriptsDir);
}
console.log(scriptsUrls)
scriptsUrls.forEach(async (scriptUrl) => {
if (!scriptUrl.startsWith('http')) return;
const response = await axios.get(scriptUrl);
const content = response.data;
fs.writeFileSync(`${scriptsDir}${scriptUrl.split('/').pop()}`, content);
});
// SCRIPTS // // SCRIPTS //
// SCREENSHOT // // SCREENSHOT //
@ -70,6 +52,20 @@ export class CrawlerService {
await browser.close(); await browser.close();
} }
async downloadFile(urls: string[], path: string) {
if (!fs.existsSync(path)) {
mkdirSync(path);
}
console.log(urls)
urls.forEach(async (url) => {
if (!url.startsWith('http')) return;
const response = await axios.get(url);
const content = response.data;
fs.writeFileSync(`${path}${url.split('/').pop()}`, content);
});
}
extractDomain(urlString: string) { extractDomain(urlString: string) {
const url = new URL(urlString); const url = new URL(urlString);
return url.hostname; return url.hostname;