work version , still bugs for auth in mongo, changing vesrion to 5.0.15

master
Kfir Dayan 2023-04-19 14:54:30 +03:00
parent 75c4f55c01
commit 43b598689c
7 changed files with 19 additions and 18 deletions

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
node_modules

View File

@ -1,15 +1,14 @@
FROM ubuntu:20.04 AS base
FROM node:14-alpine AS base
# Set non-interactive mode
ENV DEBIAN_FRONTEND noninteractive
ENV DEBIAN_FRONTEND=noninteractive
ENV NODE_ENV=production
# Install required packages
RUN apt-get update && \
curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
apt-get install -y nodejs chromium-browser
# Install chromium
RUN apk add --no-cache chromium chromium-chromedriver
RUN apt-get install -y npm
# Create a symbolic link for google-chrome
RUN ln -s /usr/bin/chromium-browser /usr/bin/google-chrome
# Set the working directory
WORKDIR /app
@ -18,10 +17,11 @@ WORKDIR /app
COPY package*.json ./
# Install dependencies
RUN npm install
RUN npm install -g npm@9.6.3 && \
npm install
# Copy the source code
COPY . .
# Start the application
CMD ["npm", "run", "start"]
CMD ["npm", "run", "start"]

View File

@ -7,15 +7,15 @@ services:
depends_on:
- mongo
environment:
- MONGO_URL=mongodb://mongo:27017/${DB_NAME}
- MONGO_URL=mongodb://mongo:27017/mydatabase
networks:
- appnet
mongo:
image: mongo
image: mongo:5.0.15
environment:
- MONGO_INITDB_DATABASE=${DB_NAME}
- MONGO_INITDB_ROOT_USERNAME=${DB_USER}
- MONGO_INITDB_ROOT_PASSWORD=${DB_PASSWORD}
- MONGO_INITDB_DATABASE=mydatabase
- MONGO_INITDB_ROOT_USERNAME=akamai
- MONGO_INITDB_ROOT_PASSWORD=password
volumes:
- dbdata:/data/db
networks:

View File

@ -29,10 +29,10 @@
"mongoose": "^7.0.4",
"puppeteer": "^19.9.1",
"reflect-metadata": "^0.1.13",
"@nestjs/cli": "^9.0.0",
"rxjs": "^7.5.5"
},
"devDependencies": {
"@nestjs/cli": "^9.0.0",
"@nestjs/schematics": "^9.0.0",
"@nestjs/testing": "^9.0.0",
"@types/express": "^4.17.13",

View File

@ -19,7 +19,7 @@ export class ApiController {
newSite.save().then((result) => {
console.log("Site saved !", result);
}).catch((err) => {
console.log("Error saving site !", err);
console.log("Error saving site !", err.message);
});
}).catch((err) => {
console.log("** Error crawling ! **", err);

View File

@ -8,7 +8,7 @@ dotenv.config();
@Module({
imports: [
MongooseModule.forRoot(process.env.DB_URI, {
MongooseModule.forRoot(process.env.MONGO_URL, {
useNewUrlParser: true,
useUnifiedTopology: true,
}),

View File

@ -18,7 +18,7 @@ export class CrawlerService {
async crawl(url: string): Promise<Site> {
console.log("start crawl website", url);
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox'] });
const browser = await puppeteer.launch({ executablePath: '/usr/bin/chromium-browser', headless: true, args: ['--no-sandbox'] });
const page = await browser.newPage();
const domain = this.extractDomain(url);
await page.goto(url);