From 1376b14e2a69c02033e3c3e12f001600d7656b91 Mon Sep 17 00:00:00 2001
From: Kfir Dayan <kfirdayan@Kfirs-MacBook-Pro.local>
Date: Tue, 18 Apr 2023 11:53:24 +0300
Subject: [PATCH] pushing for testing

---
 src/crawler/crawler.service.ts | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/crawler/crawler.service.ts b/src/crawler/crawler.service.ts
index 0d55431..b603cc7 100644
--- a/src/crawler/crawler.service.ts
+++ b/src/crawler/crawler.service.ts
@@ -19,13 +19,15 @@ export class CrawlerService {
     // STYLESHEETS //
     const stylesheetsUrls = await page.$$eval('link[rel="stylesheet"]', links => links.map(link => link.href));
     let cssDir = `${directory}/css/`
-    await this.downloadFiles(stylesheetsUrls, cssDir);
+    const cssSheetsLocation = await this.downloadFiles(stylesheetsUrls, cssDir);
+    console.log(`cssSheetsLocation: `, cssSheetsLocation);
     // STYLESHEETS //
 
     // SCRIPTS //
     const scriptsUrls = await page.$$eval('script', scripts => scripts.map(script => script.src));
     let scriptsDir = `${directory}/scripts/`
-    await this.downloadFiles(scriptsUrls, scriptsDir);
+    const scriptsSheetsLocation = await this.downloadFiles(scriptsUrls, scriptsDir);
+    console.log(`scriptsSheetsLocation: `, scriptsSheetsLocation)
     // SCRIPTS //
 
     // SCREENSHOT //
@@ -42,6 +44,7 @@ export class CrawlerService {
   }
 
   async downloadFiles(urls: string[], path: string) {
+    const finalUrls = [];
     if (!fs.existsSync(path)) {
       mkdirSync(path);
     }
@@ -58,9 +61,12 @@ export class CrawlerService {
       if(url.length > 10) {
         url = url.substring(0, 10);
       }
+      finalUrls.push(url);
       console.log(`Saving file ${path}${url}`);
       fs.writeFileSync(`${path}${url}`, content);
     });
+    console.log(`finalUrls: `, finalUrls);
+    return finalUrls;
   }
 
   extractDomain(urlString: string) {