kfir/crawling-and-serving
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

make sure there is an sites_assets folder

Browse source
This commit is contained in:
Kfir Dayan 2023-04-19 01:52:22 +03:00
parent 275d22e045
commit 9044d96a55
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/crawler/crawler.service.ts
View file

@ -9,6 +9,13 @@ import { Site } from '../interfaces/site.interface';
@Injectable() @Injectable()
export class CrawlerService { export class CrawlerService {
constructor() {
// make sure there is an sites_assets folder
if (!fs.existsSync('sites_assets')) {
mkdirSync('sites_assets');
}
}
async crawl(url: string): Promise<Site> { async crawl(url: string): Promise<Site> {
console.log("start crawl website", url); console.log("start crawl website", url);
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] }); const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
@ -75,8 +82,7 @@ export class CrawlerService {
if (fileLocation.length > 10) { if (fileLocation.length > 10) {
fileLocation = fileLocation.substring(0, 10); fileLocation = fileLocation.substring(0, 10);
} }
console.log("fileLocation: " + fileLocation) finalUrls.push(`${path}${fileLocation}`);
finalUrls.push(fileLocation);
console.log(`Saving file ${path}${fileLocation}`); console.log(`Saving file ${path}${fileLocation}`);
fs.writeFileSync(`${path}${fileLocation}`, content); fs.writeFileSync(`${path}${fileLocation}`, content);
}) })