kfir/crawling-and-serving
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

make sure there is an sites_assets folder

Browse source
This commit is contained in:
Kfir Dayan 2023-04-19 01:52:22 +03:00
parent 275d22e045
commit 9044d96a55
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/crawler/crawler.service.ts
View file

@ -9,6 +9,13 @@ import { Site } from '../interfaces/site.interface';
@Injectable()
export class CrawlerService {
constructor() {
// make sure there is an sites_assets folder
if (!fs.existsSync('sites_assets')) {
mkdirSync('sites_assets');
}
}
async crawl(url: string): Promise<Site> {
console.log("start crawl website", url);
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
@ -75,8 +82,7 @@ export class CrawlerService {
if (fileLocation.length > 10) {
fileLocation = fileLocation.substring(0, 10);
}
console.log("fileLocation: " + fileLocation)
finalUrls.push(fileLocation);
finalUrls.push(`${path}${fileLocation}`);
console.log(`Saving file ${path}${fileLocation}`);
fs.writeFileSync(`${path}${fileLocation}`, content);
})