Compare commits
2 commits
6d3ed58526
...
a5799e4b48
Author | SHA1 | Date | |
---|---|---|---|
|
a5799e4b48 | ||
|
661d5e9880 |
5 changed files with 22 additions and 4 deletions
|
@ -1,11 +1,16 @@
|
||||||
import { Body, Controller, Post } from '@nestjs/common';
|
import { Body, Controller, Post } from '@nestjs/common';
|
||||||
import { CrawlerService } from '../crawler/crawler.service';
|
import { CrawlerService } from '../crawler/crawler.service';
|
||||||
|
import { DbService } from '../db/db.service';
|
||||||
|
|
||||||
@Controller('/')
|
@Controller('/')
|
||||||
export class ApiController {
|
export class ApiController {
|
||||||
constructor(private crawlerService: CrawlerService) {}
|
constructor(private crawlerService: CrawlerService, private DbService: DbService) {}
|
||||||
@Post('crawl')
|
@Post('crawl')
|
||||||
async crawl(@Body() body: { url: string }) {
|
async crawl(@Body() body: { url: string }) {
|
||||||
return this.crawlerService.crawl(body.url);
|
const results = this.crawlerService.crawl(body.url);
|
||||||
|
results.then((data) => {
|
||||||
|
console.log(data)
|
||||||
|
this.DbService.insert(data, 'crawler');
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,10 +1,11 @@
|
||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { ApiController } from './api.controller';
|
import { ApiController } from './api.controller';
|
||||||
import { CrawlerService } from '../crawler/crawler.service';
|
import { CrawlerService } from '../crawler/crawler.service';
|
||||||
|
import { DbService } from '../db/db.service';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
controllers: [ApiController],
|
controllers: [ApiController],
|
||||||
providers: [CrawlerService]
|
providers: [CrawlerService, DbService]
|
||||||
|
|
||||||
})
|
})
|
||||||
export class ApiModule {}
|
export class ApiModule {}
|
||||||
|
|
|
@ -1,8 +1,9 @@
|
||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { ApiModule } from './api/api.module';
|
import { ApiModule } from './api/api.module';
|
||||||
import { CrawlerModule } from './crawler/crawler.module';
|
import { CrawlerModule } from './crawler/crawler.module';
|
||||||
|
import { DbModule } from './db/db.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [ApiModule, CrawlerModule]
|
imports: [ApiModule, CrawlerModule, DbModule]
|
||||||
})
|
})
|
||||||
export class AppModule {}
|
export class AppModule {}
|
||||||
|
|
|
@ -42,6 +42,10 @@ export class CrawlerService {
|
||||||
// URLS //
|
// URLS //
|
||||||
const urls = await page.$$eval('a', links => links.map(link => link.href));
|
const urls = await page.$$eval('a', links => links.map(link => link.href));
|
||||||
await browser.close();
|
await browser.close();
|
||||||
|
return {
|
||||||
|
cssSheetsLocation,
|
||||||
|
scriptsSheetsLocation
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async downloadFiles(urls: string[], path: string) {
|
async downloadFiles(urls: string[], path: string) {
|
||||||
|
|
|
@ -5,4 +5,11 @@ export class DbService {
|
||||||
constructor() {
|
constructor() {
|
||||||
console.log(`DbService constructor`);
|
console.log(`DbService constructor`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
insert(data: {
|
||||||
|
cssSheetsLocation: string[];
|
||||||
|
scriptsSheetsLocation: string[];
|
||||||
|
}, collection: string) {
|
||||||
|
console.log({data, collection});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue