Merge pull request #58 from GSA/add-manual-harvest-button

Adds support for manual harvest button

.gitignore

@@ -22,7 +22,8 @@ node_modules/
 tmp/
 
 # vscode debugger
-.vscode/
+.vscode/*
+!.vscode/launch.json
 .env
 requirements.txt
 

.vscode/launch.json

@@ -0,0 +1,25 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Remote Attach",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "0.0.0.0",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "."
+                }
+            ],
+            "justMyCode": false,
+            "logToFile": true
+        }
+    ]
+}

Dockerfile

@@ -6,6 +6,12 @@ COPY . /app
 
 RUN pip install --no-cache-dir -r requirements.txt
 
+ARG DEV 
+
+RUN if [ $DEV ]; \
+    then pip install --no-cache-dir -r requirements-dev.txt; \
+    fi
+
 EXPOSE 8080
 
 ENV FLASK_APP=run.py

Makefile

@@ -1,6 +1,10 @@
 pypi-upload: build-dist  ## Uploads new package to PyPi after clean, build
 	poetry publish
 
+deps-update: ## Updates requirements.txt and requirements_dev.txt from pyproject.toml
+	poetry export --without-hashes --without=dev --format=requirements.txt > requirements.txt
+	poetry export --without-hashes --only=dev --format=requirements-dev.txt > requirements-dev.txt 
+
 # pypi-upload-test: build-dist  ## Uploads new package to TEST PyPi after clean, build
 # 	twine upload -r testpypi dist/*	
 
@@ -10,6 +14,9 @@ build-dist: clean-dist  ## Builds new package dist
 build:  ## build Flask app
 	docker compose build app
 
+build-dev:  ## build Flask app w/ dev dependencies
+	docker compose build app --build-arg DEV=True
+
 clean-dist:  ## Cleans dist dir
 	rm -rf dist/*
 
@@ -19,8 +26,11 @@ test: up ## Runs poetry tests, ignores ckan load
 up: ## Sets up local docker environment
 	docker compose up -d
 
+up-debug: ## Sets up local docker environment
+	docker compose -f docker-compose_debug.yml up -d
+
 down: ## Shuts down local docker instance
-	docker-compose down
+	docker compose down
 
 clean: ## Cleans docker images
 	docker compose down -v --remove-orphans

README.md

@@ -23,6 +23,8 @@ This project is using `poetry` to manage this project. Install [here](https://py
 
 Once installed, `poetry install` installs dependencies into a local virtual environment.
 
+We use [Ruff](https://github.com/astral-sh/ruff) to format and lint our Python files. If you use VS Code, you can install the formatter [here](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff).
+
 ## Testing
 
 ### CKAN load testing
@@ -115,10 +117,22 @@ If you followed the instructions for `CKAN load testing` and `Harvester testing`
 3. when there are database DDL changes, use following steps to generate migration scripts and update database:
 
     ```bash
-    docker compose db up
+    docker compose up -d db
     docker compose run app flask db migrate -m "migration description"
     docker compose run app flask db upgrade
     ```
+### Debugging
+*NOTE: To use the VS-Code debugger, you will first need to sacrifice the reloading support for flask*
+
+1. Build new containers with development requirements by running `make build-dev`
+
+2. Launch containers by running `make up-debug`
+
+3. In VS-Code, launch debug process `Python: Remote Attach`
+
+4. Set breakpoints
+
+5. Visit the site at `http://localhost:8080` and invoke the route which contains the code you've set the breakpoint on.
 
 ### Deployment to cloud.gov
 

app/__init__.py

@@ -13,7 +13,8 @@
 
 
 def create_app(testing=False):
-    app = Flask(__name__)
+
+    app = Flask(__name__, static_url_path="", static_folder="static")
 
     if testing:
         app.config["TESTING"] = True

app/forms.py

@@ -1,7 +1,7 @@
 import re
 
 from flask_wtf import FlaskForm
-from wtforms import SelectField, StringField, SubmitField, TextAreaField
+from wtforms import SelectField, StringField, TextAreaField
 from wtforms.validators import URL, DataRequired, ValidationError
 
 
@@ -18,7 +18,7 @@ class HarvestSourceForm(FlaskForm):
     )
     name = StringField("Name", validators=[DataRequired()])
     url = StringField("URL", validators=[DataRequired(), URL()])
-    emails = TextAreaField(
+    notification_emails = TextAreaField(
         "Notification_emails", validators=[DataRequired(), validate_email_list]
     )
     frequency = SelectField(
@@ -35,10 +35,8 @@ class HarvestSourceForm(FlaskForm):
     source_type = SelectField(
         "Source Type", choices=["Datajson", "WAF"], validators=[DataRequired()]
     )
-    submit = SubmitField("Submit")
 
 
 class OrganizationForm(FlaskForm):
     name = StringField("Name", validators=[DataRequired()])
     logo = StringField("Logo", validators=[DataRequired()])
-    submit = SubmitField("Submit")

app/interface.py

@@ -162,12 +162,6 @@ def add_harvest_job(self, job_data):
             self.db.rollback()
             return None
 
-    # for test, will remove later
-    def get_all_harvest_jobs(self):
-        harvest_jobs = self.db.query(HarvestJob).all()
-        harvest_jobs_data = [HarvesterDBInterface._to_dict(job) for job in harvest_jobs]
-        return harvest_jobs_data
-
     def get_harvest_job(self, job_id):
         result = self.db.query(HarvestJob).filter_by(id=job_id).first()
         if result is None:
@@ -223,14 +217,6 @@ def add_harvest_error(self, error_data):
             self.db.rollback()
             return None
 
-    # for test, will remove later
-    def get_all_harvest_errors(self):
-        harvest_errors = self.db.query(HarvestError).all()
-        harvest_errors_data = [
-            HarvesterDBInterface._to_dict(err) for err in harvest_errors
-        ]
-        return harvest_errors_data
-
     def get_harvest_error(self, error_id):
         result = self.db.query(HarvestError).filter_by(id=error_id).first()
         if result is None:
@@ -276,14 +262,6 @@ def add_harvest_records(self, records_data: list) -> bool:
             self.db.rollback()
             return None
 
-    # for test, will remove later
-    def get_all_harvest_records(self):
-        harvest_records = self.db.query(HarvestRecord).all()
-        harvest_records_data = [
-            HarvesterDBInterface._to_dict(err) for err in harvest_records
-        ]
-        return harvest_records_data
-
     def get_harvest_record(self, record_id):
         result = self.db.query(HarvestRecord).filter_by(id=record_id).first()
         if result is None:
@@ -324,3 +302,24 @@ def close(self):
             self.db.remove()
         elif hasattr(self.db, "close"):
             self.db.close()
+
+    ##### TEST INTERFACES BELOW #####
+    ######## TO BE REMOVED ##########
+    def get_all_harvest_jobs(self):
+        harvest_jobs = self.db.query(HarvestJob).all()
+        harvest_jobs_data = [HarvesterDBInterface._to_dict(job) for job in harvest_jobs]
+        return harvest_jobs_data
+
+    def get_all_harvest_records(self):
+        harvest_records = self.db.query(HarvestRecord).all()
+        harvest_records_data = [
+            HarvesterDBInterface._to_dict(err) for err in harvest_records
+        ]
+        return harvest_records_data
+
+    def get_all_harvest_errors(self):
+        harvest_errors = self.db.query(HarvestError).all()
+        harvest_errors_data = [
+            HarvesterDBInterface._to_dict(err) for err in harvest_errors
+        ]
+        return harvest_errors_data

app/models.py

@@ -47,7 +47,13 @@ class HarvestJob(Base):
         db.String(36), db.ForeignKey("harvest_source.id"), nullable=False
     )
     status = db.Column(
-        Enum("new", "in_progress", "complete", name="job_status"),
+        Enum(
+            "in_progress",
+            "complete",
+            "pending",
+            "pending_manual",
+            name="job_status",
+        ),
         nullable=False,
         index=True,
     )