make sure there is an sites_assets

This commit is contained in:
Kfir Dayan 2023-04-19 10:15:19 +03:00
parent 9044d96a55
commit 15c718dd0a

View file

@ -10,13 +10,13 @@ import { Site } from '../interfaces/site.interface';
export class CrawlerService {
constructor() {
// make sure there is an sites_assets folder
if (!fs.existsSync('sites_assets')) {
mkdirSync('sites_assets');
}
}
async crawl(url: string): Promise<Site> {
console.log("start crawl website", url);
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
const page = await browser.newPage();
@ -26,6 +26,7 @@ export class CrawlerService {
if (!fs.existsSync(directory)) {
mkdirSync(directory);
}
// STYLESHEETS //
console.log("start stylesheets")
const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
@ -43,9 +44,15 @@ export class CrawlerService {
// SCREENSHOT //
const screenshotBuffer: Buffer = await page.screenshot({ fullPage: true });
fs.writeFile(`${directory}screenshot.png`, screenshotBuffer, (err) => {
if (err) throw err;
// console.log(`Screenshot saved! ${directory}screenshot.png`);
await new Promise((resolve, reject) => {
fs.writeFile(`${directory}screenshot.png`, screenshotBuffer, (err) => {
if (err) {
reject(err);
} else {
resolve(true);
}
});
});
// SCREENSHOT //