moved waf into extract.py, fixed tests, added make down/clean

Makefile

@@ -16,6 +16,12 @@ test: up ## Runs poetry tests, ignores ckan load
 up: ## Sets up local docker environment
 	docker compose up -d
 
+down: ## Shuts down local docker instance
+	docker-compose down
+
+clean: ## Cleans docker images
+	docker compose down -v --remove-orphans
+
 lint:  ## Lints wtih ruff
 	ruff .
 

harvester/extract.py

@@ -1,6 +1,8 @@
 import logging
+import os
 
 import requests
+from bs4 import BeautifulSoup
 from requests.exceptions import JSONDecodeError, RequestException
 
 logger = logging.getLogger("harvester")
@@ -26,6 +28,46 @@ def download_dcatus_catalog(url):
         return Exception(e)
 
 
+def traverse_waf(url, files=[], file_ext=".xml", folder="/", filters=[]):
+    # TODO: add exception handling
+    parent = os.path.dirname(url.rstrip("/"))
+
+    res = requests.get(url)
+    if res.status_code == 200:
+        soup = BeautifulSoup(res.content, "html.parser")
+        anchors = soup.find_all("a", href=True)
+
+        folders = []
+        for anchor in anchors:
+            if (
+                anchor["href"].endswith(folder)
+                and not parent.endswith(anchor["href"].rstrip("/"))
+                and anchor["href"] not in filters
+            ):
+                folders.append(os.path.join(url, anchor["href"]))
+
+            if anchor["href"].endswith(file_ext):
+                files.append(os.path.join(url, anchor["href"]))
+
+    for folder in folders:
+        traverse_waf(folder, files=files, filters=filters)
+
+    return files
+
+
+def download_waf(files):
+    output = []
+    for file in files:
+        data = {}
+        data["url"] = file
+        res = requests.get(file)
+        if res.status_code == 200:
+            data["content"] = res.content
+            output.append(data)
+
+    return output
+
+
 def extract(harvest_source) -> list:
     """Extracts all records from a harvest_source"""
     logger.info("Hello from harvester.extract()")

tests/extract/test_waf.py

@@ -1,4 +1,4 @@
-from harvester.extract.waf import traverse_waf, download_waf
+from harvester.extract import download_waf, traverse_waf
 
 
 def test_traverse_waf(get_waf_url):