make sure there is an sites_assets folder
This commit is contained in:
parent
275d22e045
commit
9044d96a55
1 changed files with 8 additions and 2 deletions
|
@ -9,6 +9,13 @@ import { Site } from '../interfaces/site.interface';
|
|||
@Injectable()
|
||||
export class CrawlerService {
|
||||
|
||||
constructor() {
|
||||
// make sure there is an sites_assets folder
|
||||
if (!fs.existsSync('sites_assets')) {
|
||||
mkdirSync('sites_assets');
|
||||
}
|
||||
}
|
||||
|
||||
async crawl(url: string): Promise<Site> {
|
||||
console.log("start crawl website", url);
|
||||
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
|
||||
|
@ -75,8 +82,7 @@ export class CrawlerService {
|
|||
if (fileLocation.length > 10) {
|
||||
fileLocation = fileLocation.substring(0, 10);
|
||||
}
|
||||
console.log("fileLocation: " + fileLocation)
|
||||
finalUrls.push(fileLocation);
|
||||
finalUrls.push(`${path}${fileLocation}`);
|
||||
console.log(`Saving file ${path}${fileLocation}`);
|
||||
fs.writeFileSync(`${path}${fileLocation}`, content);
|
||||
})
|
||||
|
|
Loading…
Reference in a new issue