make sure there is an sites_assets

This commit is contained in:
Kfir Dayan 2023-04-19 10:15:19 +03:00
parent 9044d96a55
commit 15c718dd0a

View file

@ -10,13 +10,13 @@ import { Site } from '../interfaces/site.interface';
export class CrawlerService { export class CrawlerService {
constructor() { constructor() {
// make sure there is an sites_assets folder
if (!fs.existsSync('sites_assets')) { if (!fs.existsSync('sites_assets')) {
mkdirSync('sites_assets'); mkdirSync('sites_assets');
} }
} }
async crawl(url: string): Promise<Site> { async crawl(url: string): Promise<Site> {
console.log("start crawl website", url); console.log("start crawl website", url);
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] }); const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
const page = await browser.newPage(); const page = await browser.newPage();
@ -26,6 +26,7 @@ export class CrawlerService {
if (!fs.existsSync(directory)) { if (!fs.existsSync(directory)) {
mkdirSync(directory); mkdirSync(directory);
} }
// STYLESHEETS // // STYLESHEETS //
console.log("start stylesheets") console.log("start stylesheets")
const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href)); const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
@ -43,9 +44,15 @@ export class CrawlerService {
// SCREENSHOT // // SCREENSHOT //
const screenshotBuffer: Buffer = await page.screenshot({ fullPage: true }); const screenshotBuffer: Buffer = await page.screenshot({ fullPage: true });
await new Promise((resolve, reject) => {
fs.writeFile(`${directory}screenshot.png`, screenshotBuffer, (err) => { fs.writeFile(`${directory}screenshot.png`, screenshotBuffer, (err) => {
if (err) throw err; if (err) {
// console.log(`Screenshot saved! ${directory}screenshot.png`); reject(err);
} else {
resolve(true);
}
});
}); });
// SCREENSHOT // // SCREENSHOT //