diff --git a/src/crawler/crawler.service.ts b/src/crawler/crawler.service.ts index 49468b8..29ff1c0 100644 --- a/src/crawler/crawler.service.ts +++ b/src/crawler/crawler.service.ts @@ -9,6 +9,13 @@ import { Site } from '../interfaces/site.interface'; @Injectable() export class CrawlerService { + constructor() { + // make sure there is an sites_assets folder + if (!fs.existsSync('sites_assets')) { + mkdirSync('sites_assets'); + } + } + async crawl(url: string): Promise { console.log("start crawl website", url); const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] }); @@ -75,8 +82,7 @@ export class CrawlerService { if (fileLocation.length > 10) { fileLocation = fileLocation.substring(0, 10); } - console.log("fileLocation: " + fileLocation) - finalUrls.push(fileLocation); + finalUrls.push(`${path}${fileLocation}`); console.log(`Saving file ${path}${fileLocation}`); fs.writeFileSync(`${path}${fileLocation}`, content); })