feat(crawling): cleanup get crawl by id pg query code + remove scraping bee

Author: skeptrunedev

docker-compose-firecrawl.yml

@@ -50,7 +50,6 @@ services:
       - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE}
       - BULL_AUTH_KEY=${BULL_AUTH_KEY}
       - TEST_API_KEY=${TEST_API_KEY}
-      - SCRAPING_BEE_API_KEY=${SCRAPING_BEE_API_KEY}
       - HOST=${HOST:-0.0.0.0}
       - LOGGING_LEVEL=${LOGGING_LEVEL}
     extra_hosts:

server/src/operators/crawl_operator.rs

@@ -462,18 +462,7 @@ pub async fn get_crawl_by_scrape_id_query(
         .await
         .map_err(|e| ServiceError::InternalServerError(e.to_string()))?;
     let request: CrawlRequestPG = crawl_requests_table::crawl_requests
-        .select((
-            crawl_requests_table::id,
-            crawl_requests_table::url,
-            crawl_requests_table::status,
-            crawl_requests_table::crawl_type,
-            crawl_requests_table::next_crawl_at,
-            crawl_requests_table::interval,
-            crawl_requests_table::crawl_options,
-            crawl_requests_table::scrape_id,
-            crawl_requests_table::dataset_id,
-            crawl_requests_table::created_at,
-        ))
+        .select(CrawlRequestPG::as_select())
         .filter(crawl_requests_table::scrape_id.eq(scrape_id))
         .first(&mut conn)
         .await