From a2df1212b1529696b6450a6aec4caadf6d070aae Mon Sep 17 00:00:00 2001 From: Kfir Dayan Date: Tue, 30 May 2023 17:29:46 +0300 Subject: [PATCH] done with migration file --- app/Http/Controllers/WebCrawlController.php | 87 +++++++++++++++++++ app/Models/User.php | 44 ---------- app/Models/WebCrawl.php | 12 +++ composer.json | 6 +- composer.lock | 2 +- config/database.php | 24 ++--- .../2014_10_12_000000_create_users_table.php | 36 -------- ...12_100000_create_password_resets_table.php | 32 ------- ..._08_19_000000_create_failed_jobs_table.php | 36 -------- ..._05_30_121851_create_web_crawls_table.php} | 19 ++-- routes/api.php | 68 ++------------- 11 files changed, 134 insertions(+), 232 deletions(-) create mode 100644 app/Http/Controllers/WebCrawlController.php delete mode 100644 app/Models/User.php create mode 100644 app/Models/WebCrawl.php delete mode 100644 database/migrations/2014_10_12_000000_create_users_table.php delete mode 100644 database/migrations/2014_10_12_100000_create_password_resets_table.php delete mode 100644 database/migrations/2019_08_19_000000_create_failed_jobs_table.php rename database/migrations/{2019_12_14_000001_create_personal_access_tokens_table.php => 2023_05_30_121851_create_web_crawls_table.php} (50%) diff --git a/app/Http/Controllers/WebCrawlController.php b/app/Http/Controllers/WebCrawlController.php new file mode 100644 index 0000000..563856b --- /dev/null +++ b/app/Http/Controllers/WebCrawlController.php @@ -0,0 +1,87 @@ +json($webCrawl); + } + + /** + * Show the form for creating a new resource. + * + * @return \Illuminate\Http\Response + */ + public function create() + { + // + } + + /** + * Store a newly created resource in storage. + * + * @param \Illuminate\Http\Request $request + * @return \Illuminate\Http\Response + */ + public function store(Request $request) + { + // + } + + /** + * Display the specified resource. + * + * @param \App\Models\WebCrawl $webCrawl + * @return \Illuminate\Http\Response + */ + public function show(WebCrawl $webCrawl) + { + // + } + + /** + * Show the form for editing the specified resource. + * + * @param \App\Models\WebCrawl $webCrawl + * @return \Illuminate\Http\Response + */ + public function edit(WebCrawl $webCrawl) + { + // + } + + /** + * Update the specified resource in storage. + * + * @param \Illuminate\Http\Request $request + * @param \App\Models\WebCrawl $webCrawl + * @return \Illuminate\Http\Response + */ + public function update(Request $request, WebCrawl $webCrawl) + { + // + } + + /** + * Remove the specified resource from storage. + * + * @param \App\Models\WebCrawl $webCrawl + * @return \Illuminate\Http\Response + */ + public function destroy(WebCrawl $webCrawl) + { + // + } +} diff --git a/app/Models/User.php b/app/Models/User.php deleted file mode 100644 index 23b4063..0000000 --- a/app/Models/User.php +++ /dev/null @@ -1,44 +0,0 @@ - - */ - protected $fillable = [ - 'name', - 'email', - 'password', - ]; - - /** - * The attributes that should be hidden for serialization. - * - * @var array - */ - protected $hidden = [ - 'password', - 'remember_token', - ]; - - /** - * The attributes that should be cast. - * - * @var array - */ - protected $casts = [ - 'email_verified_at' => 'datetime', - ]; -} diff --git a/app/Models/WebCrawl.php b/app/Models/WebCrawl.php new file mode 100644 index 0000000..b26e14f --- /dev/null +++ b/app/Models/WebCrawl.php @@ -0,0 +1,12 @@ + [ - 'driver' => 'mongodb', - 'host' => env('DB_HOST', 'localhost'), - 'port' => env('DB_PORT', 27017), - 'database' => env('DB_DATABASE'), - 'username' => env('DB_USERNAME'), - 'password' => env('DB_PASSWORD'), - 'options' => [ - 'database' => env('DB_AUTHENTICATION_DATABASE', 'admin'), - ], - ], + 'connections' => [ - + 'mongodb' => [ + 'driver' => 'mongodb', + 'host' => env('DB_HOST', 'localhost'), + 'port' => env('DB_PORT', 27017), + 'database' => env('DB_DATABASE'), + 'username' => env('DB_USERNAME'), + 'password' => env('DB_PASSWORD'), + 'options' => [ + 'database' => env('DB_AUTHENTICATION_DATABASE', 'admin'), + ], + ], ], /* diff --git a/database/migrations/2014_10_12_000000_create_users_table.php b/database/migrations/2014_10_12_000000_create_users_table.php deleted file mode 100644 index cf6b776..0000000 --- a/database/migrations/2014_10_12_000000_create_users_table.php +++ /dev/null @@ -1,36 +0,0 @@ -id(); - $table->string('name'); - $table->string('email')->unique(); - $table->timestamp('email_verified_at')->nullable(); - $table->string('password'); - $table->rememberToken(); - $table->timestamps(); - }); - } - - /** - * Reverse the migrations. - * - * @return void - */ - public function down() - { - Schema::dropIfExists('users'); - } -}; diff --git a/database/migrations/2014_10_12_100000_create_password_resets_table.php b/database/migrations/2014_10_12_100000_create_password_resets_table.php deleted file mode 100644 index e5f1397..0000000 --- a/database/migrations/2014_10_12_100000_create_password_resets_table.php +++ /dev/null @@ -1,32 +0,0 @@ -string('email')->primary(); - $table->string('token'); - $table->timestamp('created_at')->nullable(); - }); - } - - /** - * Reverse the migrations. - * - * @return void - */ - public function down() - { - Schema::dropIfExists('password_resets'); - } -}; diff --git a/database/migrations/2019_08_19_000000_create_failed_jobs_table.php b/database/migrations/2019_08_19_000000_create_failed_jobs_table.php deleted file mode 100644 index 1719198..0000000 --- a/database/migrations/2019_08_19_000000_create_failed_jobs_table.php +++ /dev/null @@ -1,36 +0,0 @@ -id(); - $table->string('uuid')->unique(); - $table->text('connection'); - $table->text('queue'); - $table->longText('payload'); - $table->longText('exception'); - $table->timestamp('failed_at')->useCurrent(); - }); - } - - /** - * Reverse the migrations. - * - * @return void - */ - public function down() - { - Schema::dropIfExists('failed_jobs'); - } -}; diff --git a/database/migrations/2019_12_14_000001_create_personal_access_tokens_table.php b/database/migrations/2023_05_30_121851_create_web_crawls_table.php similarity index 50% rename from database/migrations/2019_12_14_000001_create_personal_access_tokens_table.php rename to database/migrations/2023_05_30_121851_create_web_crawls_table.php index 6c81fd2..f1e60a5 100644 --- a/database/migrations/2019_12_14_000001_create_personal_access_tokens_table.php +++ b/database/migrations/2023_05_30_121851_create_web_crawls_table.php @@ -13,14 +13,17 @@ */ public function up() { - Schema::create('personal_access_tokens', function (Blueprint $table) { + Schema::create('web_crawls', function (Blueprint $table) { $table->id(); - $table->morphs('tokenable'); - $table->string('name'); - $table->string('token', 64)->unique(); - $table->text('abilities')->nullable(); - $table->timestamp('last_used_at')->nullable(); - $table->timestamp('expires_at')->nullable(); + $table->string('url'); + $table->string('content'); + $table->string('depth'); + $table->string('visited_urls'); + $table->string('status_code'); + $table->string('status'); + $table->string('created_at'); + $table->string('updated_at'); + $table->string('links'); $table->timestamps(); }); } @@ -32,6 +35,6 @@ public function up() */ public function down() { - Schema::dropIfExists('personal_access_tokens'); + Schema::dropIfExists('web_crawls'); } }; diff --git a/routes/api.php b/routes/api.php index f92f2b8..2fcbd3c 100644 --- a/routes/api.php +++ b/routes/api.php @@ -3,69 +3,13 @@ use Illuminate\Http\Request; use Illuminate\Support\Facades\Route; use GuzzleHttp\Client; -use Illuminate\Support\Facades\DB; -use MongoDB\Client as MongoClient; - -Route::get('/crawl', function (Request $request) { - - $url = $request->input('url'); - check_connection_to_mongodb(); - - if (!$url) { - return response()->json([ - 'error' => 'Missing required parameter `url`' - ], 400); - } - $depth = $request->input('depth', 3); // default depth is 3 if not provided - $visitedUrls = []; - crawlWebsite($url, $depth, $visitedUrls); -}); +use App\Http\Controllers\WebCrawlController; -function check_connection_to_mongodb() { - $connection = new MongoClient(); - echo '
';
-    echo "IT WORKS";
-    die;
-    print_r($connection);
-    die;
-}
-function crawlWebsite($url, $depth, &$visitedUrls)
-{
-    // Check if URL has already been visited
-    if (in_array($url, $visitedUrls)) {
-        return;
-    }
 
-    $visitedUrls[] = $url;
+// Route::get('/crawl', function (Request $request) {
+//    // invode WebCrawlController index method
+    
+// });
 
-    // Use GuzzleHttp client to send HTTP requests
-    $client = new Client();
-    $response = $client->get($url);
-
-    // Check if the HTTP response is successful (status code 2xx)
-    if ($response->getStatusCode() >= 200 && $response->getStatusCode() < 300) {
-
-        // echo $response->getBody()->getContents();
-
-        // Insert page info into the database
-        // DB::table('pages')->insert([
-        //     'url' => $url,
-        //     'content' => $response->getBody()->getContents()
-        // ]);
-        // Crawl the links on the page
-        // if ($depth > 0) {
-        //     $body = $response->getBody()->getContents();
-        //     $dom = new DOMDocument();
-        //     @$dom->loadHTML($body);
-
-        //     $links = $dom->getElementsByTagName('a');
-        //     foreach ($links as $link) {
-        //         $href = $link->getAttribute('href');
-        //         if (filter_var($href, FILTER_VALIDATE_URL)) {
-        //             crawlWebsite($href, $depth - 1, $visitedUrls);
-        //         }
-        //     }
-        // }
-    }
-}
+Route::get('/crawl', [WebCrawlController::class, 'index']);
\ No newline at end of file