kfir/crawling-and-serving
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

trying to fix problem

Browse source
This commit is contained in:
Kfir Dayan 2023-04-19 11:10:15 +03:00
parent c57a13d8d8
commit 75c4f55c01
4 changed files with 149 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -24,4 +24,4 @@ RUN npm install
COPY . . COPY . .
# Start the application # Start the application
CMD ["npm", "run", "start:dev"] CMD ["npm", "run", "start"]

2
README.md
View file

@ -6,9 +6,7 @@ The crawler is a simple crawler that crawls the web and stores the results in a
## Crawler ## ## Crawler ##
### Usage ### ### Usage ###
Post a JSON object to the crawler with the following format: Post a JSON object to the crawler with the following format:
`domain.com/crawl` `domain.com/crawl`

171
package-lock.json generated
View file

@ -15,6 +15,7 @@
"@nestjs/platform-express": "^9.0.0", "@nestjs/platform-express": "^9.0.0",
"axios": "^1.3.5", "axios": "^1.3.5",
"dotenv": "^16.0.3", "dotenv": "^16.0.3",
"minimatch": "^9.0.0",
"mongoose": "^7.0.4", "mongoose": "^7.0.4",
"puppeteer": "^19.9.1", "puppeteer": "^19.9.1",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",
@ -2398,6 +2399,16 @@
"node": "^12.22.0 || ^14.17.0 || >=16.0.0" "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
} }
}, },
"node_modules/@eslint/eslintrc/node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/@eslint/eslintrc/node_modules/debug": { "node_modules/@eslint/eslintrc/node_modules/debug": {
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@ -2415,6 +2426,18 @@
} }
} }
}, },
"node_modules/@eslint/eslintrc/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/@eslint/eslintrc/node_modules/ms": { "node_modules/@eslint/eslintrc/node_modules/ms": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@ -2435,6 +2458,16 @@
"node": ">=10.10.0" "node": ">=10.10.0"
} }
}, },
"node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/@humanwhocodes/config-array/node_modules/debug": { "node_modules/@humanwhocodes/config-array/node_modules/debug": {
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@ -2452,6 +2485,18 @@
} }
} }
}, },
"node_modules/@humanwhocodes/config-array/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/@humanwhocodes/config-array/node_modules/ms": { "node_modules/@humanwhocodes/config-array/node_modules/ms": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@ -2922,15 +2967,6 @@
"node": ">= 12.9.0" "node": ">= 12.9.0"
} }
}, },
"node_modules/@nestjs/cli/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/@nestjs/cli/node_modules/glob": { "node_modules/@nestjs/cli/node_modules/glob": {
"version": "9.3.5", "version": "9.3.5",
"resolved": "https://registry.npmjs.org/glob/-/glob-9.3.5.tgz", "resolved": "https://registry.npmjs.org/glob/-/glob-9.3.5.tgz",
@ -4417,8 +4453,7 @@
"node_modules/balanced-match": { "node_modules/balanced-match": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
"dev": true
}, },
"node_modules/base64-js": { "node_modules/base64-js": {
"version": "1.5.1", "version": "1.5.1",
@ -4507,13 +4542,11 @@
"peer": true "peer": true
}, },
"node_modules/brace-expansion": { "node_modules/brace-expansion": {
"version": "1.1.11", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dev": true,
"dependencies": { "dependencies": {
"balanced-match": "^1.0.0", "balanced-match": "^1.0.0"
"concat-map": "0.0.1"
} }
}, },
"node_modules/braces": { "node_modules/braces": {
@ -4910,7 +4943,7 @@
"node_modules/concat-map": { "node_modules/concat-map": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
"dev": true "dev": true
}, },
"node_modules/concat-stream": { "node_modules/concat-stream": {
@ -5418,6 +5451,16 @@
"node": "^12.22.0 || ^14.17.0 || >=16.0.0" "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
} }
}, },
"node_modules/eslint/node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/eslint/node_modules/debug": { "node_modules/eslint/node_modules/debug": {
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@ -5481,6 +5524,18 @@
"node": ">=10.13.0" "node": ">=10.13.0"
} }
}, },
"node_modules/eslint/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/eslint/node_modules/ms": { "node_modules/eslint/node_modules/ms": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@ -5962,6 +6017,28 @@
"webpack": "^5.11.0" "webpack": "^5.11.0"
} }
}, },
"node_modules/fork-ts-checker-webpack-plugin/node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/form-data": { "node_modules/form-data": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
@ -6178,6 +6255,28 @@
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
"dev": true "dev": true
}, },
"node_modules/glob/node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/glob/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/globals": { "node_modules/globals": {
"version": "13.16.0", "version": "13.16.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-13.16.0.tgz", "resolved": "https://registry.npmjs.org/globals/-/globals-13.16.0.tgz",
@ -7791,15 +7890,17 @@
} }
}, },
"node_modules/minimatch": { "node_modules/minimatch": {
"version": "3.1.2", "version": "9.0.0",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.0.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "integrity": "sha512-0jJj8AvgKqWN05mrwuqi8QYKx1WmYSUoKSxu5Qhs9prezTz10sxAHGNZe9J9cqIJzta8DWsleh2KaVaLl6Ru2w==",
"dev": true,
"dependencies": { "dependencies": {
"brace-expansion": "^1.1.7" "brace-expansion": "^2.0.1"
}, },
"engines": { "engines": {
"node": "*" "node": ">=16 || 14 >=14.17"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
} }
}, },
"node_modules/minimist": { "node_modules/minimist": {
@ -9559,6 +9660,28 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/test-exclude/node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/test-exclude/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/text-table": { "node_modules/text-table": {
"version": "0.2.0", "version": "0.2.0",
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",

1
package.json
View file

@ -25,6 +25,7 @@
"@nestjs/platform-express": "^9.0.0", "@nestjs/platform-express": "^9.0.0",
"axios": "^1.3.5", "axios": "^1.3.5",
"dotenv": "^16.0.3", "dotenv": "^16.0.3",
"minimatch": "^9.0.0",
"mongoose": "^7.0.4", "mongoose": "^7.0.4",
"puppeteer": "^19.9.1", "puppeteer": "^19.9.1",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",