fixing docker-compose

Dockerfile

@@ -2,10 +2,10 @@ FROM ubuntu:20.04 AS base
 
 # Set non-interactive mode
 ENV DEBIAN_FRONTEND noninteractive
+ENV NODE_ENV=production
 
 # Install required packages
 RUN apt-get update && \
-    apt-get install -y curl wget gnupg2 libgbm-dev ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils && \
     curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
     apt-get install -y nodejs chromium-browser
 

package-lock.json

@@ -11,8 +11,11 @@
       "dependencies": {
         "@nestjs/common": "^9.0.0",
         "@nestjs/core": "^9.0.5",
+        "@nestjs/mongoose": "^9.2.2",
         "@nestjs/platform-express": "^9.0.0",
         "axios": "^1.3.5",
+        "dotenv": "^16.0.3",
+        "mongoose": "^7.0.4",
         "puppeteer": "^19.9.1",
         "reflect-metadata": "^0.1.13",
         "rxjs": "^7.5.5"
@@ -1564,6 +1567,18 @@
         }
       }
     },
+    "node_modules/@nestjs/mongoose": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/@nestjs/mongoose/-/mongoose-9.2.2.tgz",
+      "integrity": "sha512-szNuSUCwwbQSSeiTh8+tZ9fHV4nuzHwBDROb0hX0s7crwY15TunCfwyKbB2XjqkEQWUAasDeCBuKOJSL9N6tTg==",
+      "peerDependencies": {
+        "@nestjs/common": "^8.0.0 || ^9.0.0",
+        "@nestjs/core": "^8.0.0 || ^9.0.0",
+        "mongoose": "^6.0.2 || ^7.0.0",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^7.0.0"
+      }
+    },
     "node_modules/@nestjs/platform-express": {
       "version": "9.0.0",
       "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.0.0.tgz",
@@ -1971,8 +1986,7 @@
     "node_modules/@types/node": {
       "version": "18.0.3",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.0.3.tgz",
-      "integrity": "sha512-HzNRZtp4eepNitP+BD6k2L6DROIDG4Q0fm4x+dwfsr6LGmROENnok75VGw40628xf+iR24WeMFcHuuBDUAzzsQ==",
+      "integrity": "sha512-HzNRZtp4eepNitP+BD6k2L6DROIDG4Q0fm4x+dwfsr6LGmROENnok75VGw40628xf+iR24WeMFcHuuBDUAzzsQ=="
-      "devOptional": true
     },
     "node_modules/@types/parse-json": {
       "version": "4.0.0",
@@ -2033,6 +2047,20 @@
         "@types/superagent": "*"
       }
     },
+    "node_modules/@types/webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog=="
+    },
+    "node_modules/@types/whatwg-url": {
+      "version": "8.2.2",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz",
+      "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==",
+      "dependencies": {
+        "@types/node": "*",
+        "@types/webidl-conversions": "*"
+      }
+    },
     "node_modules/@types/yargs": {
       "version": "17.0.10",
       "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.10.tgz",
@@ -3011,6 +3039,14 @@
         "node-int64": "^0.4.0"
       }
     },
+    "node_modules/bson": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-5.2.0.tgz",
+      "integrity": "sha512-HevkSpDbpUfsrHWmWiAsNavANKYIErV2ePXllp1bwq5CDreAaFVj6RVlZpJnxK4WWDCJ/5jMUpaY6G526q3Hjg==",
+      "engines": {
+        "node": ">=14.20.1"
+      }
+    },
     "node_modules/buffer": {
       "version": "5.7.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -3608,6 +3644,14 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/dotenv": {
+      "version": "16.0.3",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
+      "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -4938,6 +4982,11 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/ip": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
+      "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ=="
+    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -6209,6 +6258,14 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/kareem": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz",
+      "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA==",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/kleur": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -6411,6 +6468,12 @@
         "node": ">= 4.0.0"
       }
     },
+    "node_modules/memory-pager": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
+      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
+      "optional": true
+    },
     "node_modules/merge-descriptors": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
@@ -6529,6 +6592,144 @@
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
+    "node_modules/mongodb": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.1.0.tgz",
+      "integrity": "sha512-qgKb7y+EI90y4weY3z5+lIgm8wmexbonz0GalHkSElQXVKtRuwqXuhXKccyvIjXCJVy9qPV82zsinY0W1FBnJw==",
+      "dependencies": {
+        "bson": "^5.0.1",
+        "mongodb-connection-string-url": "^2.6.0",
+        "socks": "^2.7.1"
+      },
+      "engines": {
+        "node": ">=14.20.1"
+      },
+      "optionalDependencies": {
+        "saslprep": "^1.0.3"
+      },
+      "peerDependencies": {
+        "@aws-sdk/credential-providers": "^3.201.0",
+        "mongodb-client-encryption": "^2.3.0",
+        "snappy": "^7.2.2"
+      },
+      "peerDependenciesMeta": {
+        "@aws-sdk/credential-providers": {
+          "optional": true
+        },
+        "mongodb-client-encryption": {
+          "optional": true
+        },
+        "snappy": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/mongodb-connection-string-url": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz",
+      "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==",
+      "dependencies": {
+        "@types/whatwg-url": "^8.2.1",
+        "whatwg-url": "^11.0.0"
+      }
+    },
+    "node_modules/mongodb-connection-string-url/node_modules/tr46": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",
+      "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
+      "dependencies": {
+        "punycode": "^2.1.1"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz",
+      "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
+      "dependencies": {
+        "tr46": "^3.0.0",
+        "webidl-conversions": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/mongoose": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-7.0.4.tgz",
+      "integrity": "sha512-MEmQOOqQUvW1PJcji64NtA2EFGHrEvk9o4g//isVYSJW2+8Y8u49C2qFBKzn1t6/l9onQn012o/PcFqR6ixQpQ==",
+      "dependencies": {
+        "bson": "^5.0.1",
+        "kareem": "2.5.1",
+        "mongodb": "5.1.0",
+        "mpath": "0.9.0",
+        "mquery": "5.0.0",
+        "ms": "2.1.3",
+        "sift": "16.0.1"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/mongoose"
+      }
+    },
+    "node_modules/mongoose/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
+    "node_modules/mpath": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz",
+      "integrity": "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/mquery": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/mquery/-/mquery-5.0.0.tgz",
+      "integrity": "sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==",
+      "dependencies": {
+        "debug": "4.x"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/mquery/node_modules/debug": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "dependencies": {
+        "ms": "2.1.2"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/mquery/node_modules/ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+    },
     "node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -7067,7 +7268,6 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
       "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
-      "dev": true,
       "engines": {
         "node": ">=6"
       }
@@ -7458,6 +7658,18 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "node_modules/saslprep": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
+      "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
+      "optional": true,
+      "dependencies": {
+        "sparse-bitfield": "^3.0.3"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/schema-utils": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz",
@@ -7598,6 +7810,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/sift": {
+      "version": "16.0.1",
+      "resolved": "https://registry.npmjs.org/sift/-/sift-16.0.1.tgz",
+      "integrity": "sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ=="
+    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -7619,6 +7836,28 @@
         "node": ">=8"
       }
     },
+    "node_modules/smart-buffer": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
+      "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+      "engines": {
+        "node": ">= 6.0.0",
+        "npm": ">= 3.0.0"
+      }
+    },
+    "node_modules/socks": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz",
+      "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
+      "dependencies": {
+        "ip": "^2.0.0",
+        "smart-buffer": "^4.2.0"
+      },
+      "engines": {
+        "node": ">= 10.13.0",
+        "npm": ">= 3.0.0"
+      }
+    },
     "node_modules/source-map": {
       "version": "0.7.3",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
@@ -7654,6 +7893,15 @@
       "deprecated": "Please use @jridgewell/sourcemap-codec instead",
       "dev": true
     },
+    "node_modules/sparse-bitfield": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
+      "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+      "optional": true,
+      "dependencies": {
+        "memory-pager": "^1.0.2"
+      }
+    },
     "node_modules/sprintf-js": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",

package.json

@@ -21,8 +21,11 @@
   "dependencies": {
     "@nestjs/common": "^9.0.0",
     "@nestjs/core": "^9.0.5",
+    "@nestjs/mongoose": "^9.2.2",
     "@nestjs/platform-express": "^9.0.0",
     "axios": "^1.3.5",
+    "dotenv": "^16.0.3",
+    "mongoose": "^7.0.4",
     "puppeteer": "^19.9.1",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.5.5"

src/crawler/crawler.service.ts

@@ -5,11 +5,12 @@ import puppeteer from 'puppeteer';
 import { URL } from 'url';
 import axios from 'axios';
 
+const environment = process.env.NODE_ENV || 'development';
+
 @Injectable()
 export class CrawlerService {
-  async crawl(url: string) {
+  async crawl(url: string): Promise<any> {
-
+    const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
-    const browser = await puppeteer.launch();
     const page = await browser.newPage();
     const domain = this.extractDomain(url);
     await page.goto(url);

src/db/db.service.ts

@@ -10,6 +10,8 @@ export class DbService {
     cssSheetsLocation: string[];
     scriptsSheetsLocation: string[];
   }, collection: string)  {
+
     console.log({data, collection});
+  
   }
 }