kfir/crawling-and-serving
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fixing MONGO BUG

Browse source
This commit is contained in:
Kfir Dayan 2023-04-20 19:49:46 +03:00
parent e04a9dd3b1
commit 0a053ba249
2 changed files with 3 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docker-compose.yaml
View file

@ -21,12 +21,12 @@ services:
- MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME} - MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME}
- MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD} - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}
volumes: volumes:
- dbdata:/data/db - new:/data/db
networks: networks:
- appnet - appnet
networks: networks:
appnet: appnet:
driver: bridge driver: bridge
volumes: volumes:
dbdata: new:
driver: local driver: local

1
src/crawler/crawler.service.ts
View file

@ -59,6 +59,7 @@ export class CrawlerService {
const urls = await page.$$eval('a', links => links.map(link => link.href)); const urls = await page.$$eval('a', links => links.map(link => link.href));
const urlsList = urls.filter((url) => url.startsWith('http')); const urlsList = urls.filter((url) => url.startsWith('http'));
console.log(urlsList); console.log(urlsList);
// URLS //
await browser.close(); await browser.close();
return { return {