reOrder the methods
This commit is contained in:
parent
e4d982891f
commit
babed7889b
1 changed files with 16 additions and 20 deletions
|
@ -25,31 +25,13 @@ export class CrawlerService {
|
|||
// STYLESHEETS //
|
||||
const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
|
||||
let cssDir = `${directory}/css/`
|
||||
if (!fs.existsSync(cssDir)) {
|
||||
mkdirSync(cssDir);
|
||||
}
|
||||
stylesheetsUrls.forEach(async (stylesheetUrl) => {
|
||||
if(!stylesheetUrl.startsWith('http')) return;
|
||||
const response = await axios.get(stylesheetUrl);
|
||||
const content = response.data;
|
||||
fs.writeFileSync(`${cssDir}${stylesheetUrl.split('/').pop()}`, content);
|
||||
});
|
||||
await this.downloadFile(stylesheetsUrls, cssDir);
|
||||
// STYLESHEETS //
|
||||
|
||||
// SCRIPTS //
|
||||
const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src));
|
||||
let scriptsDir = `${directory}/scripts/`
|
||||
if (!fs.existsSync(scriptsDir)) {
|
||||
mkdirSync(scriptsDir);
|
||||
}
|
||||
console.log(scriptsUrls)
|
||||
|
||||
scriptsUrls.forEach(async (scriptUrl) => {
|
||||
if (!scriptUrl.startsWith('http')) return;
|
||||
const response = await axios.get(scriptUrl);
|
||||
const content = response.data;
|
||||
fs.writeFileSync(`${scriptsDir}${scriptUrl.split('/').pop()}`, content);
|
||||
});
|
||||
await this.downloadFile(scriptsUrls, scriptsDir);
|
||||
// SCRIPTS //
|
||||
|
||||
// SCREENSHOT //
|
||||
|
@ -70,6 +52,20 @@ export class CrawlerService {
|
|||
await browser.close();
|
||||
}
|
||||
|
||||
async downloadFile(urls: string[], path: string) {
|
||||
if (!fs.existsSync(path)) {
|
||||
mkdirSync(path);
|
||||
}
|
||||
console.log(urls)
|
||||
|
||||
urls.forEach(async (url) => {
|
||||
if (!url.startsWith('http')) return;
|
||||
const response = await axios.get(url);
|
||||
const content = response.data;
|
||||
fs.writeFileSync(`${path}${url.split('/').pop()}`, content);
|
||||
});
|
||||
}
|
||||
|
||||
extractDomain(urlString: string) {
|
||||
const url = new URL(urlString);
|
||||
return url.hostname;
|
||||
|
|
Loading…
Reference in a new issue