From 2f928f19f59a911de5dc6eb294082715e130a4c7 Mon Sep 17 00:00:00 2001 From: Tyler Burton Date: Wed, 24 Apr 2024 13:06:39 -0500 Subject: [PATCH 01/11] Adds support for debugpy and vscode integration for visual debugger --- .gitignore | 3 +- .vscode/launch.json | 25 ++++++++++++++ Dockerfile | 6 ++++ Makefile | 12 ++++++- README.md | 13 ++++++++ docker-compose.yml | 2 -- docker-compose_debug.yml | 67 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 4 +-- requirements-dev.txt | 11 ++++++ requirements.txt | 72 ++++++++++++++++++++++++++++++++++------ 10 files changed, 198 insertions(+), 17 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 docker-compose_debug.yml create mode 100644 requirements-dev.txt diff --git a/.gitignore b/.gitignore index 074aac09..8c37ca7b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,8 @@ node_modules/ tmp/ # vscode debugger -.vscode/ +.vscode/* +!.vscode/launch.json .env requirements.txt diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..c0bfd27f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,25 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "0.0.0.0", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "justMyCode": false, + "logToFile": true + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7f3d6ab2..01a86913 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,12 @@ COPY . /app RUN pip install --no-cache-dir -r requirements.txt +ARG DEBUG + +RUN if [ $DEBUG ]; \ + then pip install --no-cache-dir -r requirements-dev.txt; \ + fi + EXPOSE 8080 ENV FLASK_APP=run.py diff --git a/Makefile b/Makefile index 73da2a4c..fa8b37a3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ pypi-upload: build-dist ## Uploads new package to PyPi after clean, build poetry publish +deps-update: ## Updates requirements.txt and requirements_dev.txt from pyproject.toml + poetry export --without-hashes --without=dev --format=requirements.txt > requirements.txt + poetry export --without-hashes --only=dev --format=requirements-dev.txt > requirements-dev.txt + # pypi-upload-test: build-dist ## Uploads new package to TEST PyPi after clean, build # twine upload -r testpypi dist/* @@ -10,6 +14,9 @@ build-dist: clean-dist ## Builds new package dist build: ## build Flask app docker compose build app +build-debug: ## build Flask app w/ dev dependencies + docker compose build app --build-arg DEBUG=True + clean-dist: ## Cleans dist dir rm -rf dist/* @@ -19,8 +26,11 @@ test: up ## Runs poetry tests, ignores ckan load up: ## Sets up local docker environment docker compose up -d +up-debug: ## Sets up local docker environment + docker compose -f docker-compose_debug.yml up -d + down: ## Shuts down local docker instance - docker-compose down + docker compose down clean: ## Cleans docker images docker compose down -v --remove-orphans diff --git a/README.md b/README.md index ba28640d..a65d0810 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,20 @@ If you followed the instructions for `CKAN load testing` and `Harvester testing` docker compose run app flask db migrate -m "migration description" docker compose run app flask db upgrade ``` +### Debugging +*NOTE: To use the VS-Code debugger, you will first need to sacrifice the reloading support for flask* +1. Build new containers with development requirements by running `make build-debug` + +2. Launch containers by running `make up-debug` + +3. In VS-Code, launch debug process `Python: Remote Attach` + +4. Set breakpoints + +5. Visit the site at `http://localhost:8080` and invoke the route which contains the code you've set the breakpoint on. + +[(source)](https://medium.com/@lassebenninga/how-to-debug-flask-running-in-docker-compose-in-vs-code-ef37f0f516ee) ### Deployment to cloud.gov #### Database Service Setup diff --git a/docker-compose.yml b/docker-compose.yml index 2e7eeffe..a44399a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: mdtranslator: image: ghcr.io/gsa/mdtranslator:latest diff --git a/docker-compose_debug.yml b/docker-compose_debug.yml new file mode 100644 index 00000000..b1534048 --- /dev/null +++ b/docker-compose_debug.yml @@ -0,0 +1,67 @@ +services: + mdtranslator: + image: ghcr.io/gsa/mdtranslator:latest + ports: + - 3000:3000 + healthcheck: + test: ["CMD", "curl", "-d", "{}", "-X", "POST", "http://localhost:3000/translates"] + interval: 10s + timeout: 10s + retries: 5 + nginx-harvest-source: + image: nginx + volumes: + - ./tests/harvest-sources:/usr/share/nginx/html + - ./tests/nginx.conf:/etc/nginx/conf.d/default.conf + ports: + - 80:80 + localstack-container: + privileged: true + image: localstack/localstack:1.3.1 + ports: + - "4566:4566" + - "8081:8081" + healthcheck: + test: ["CMD", "curl", "--fail", "localhost:4566"] + interval: 2s + timeout: 5s + retries: 5 + environment: + - SERVICES=s3 + - DEBUG=1 + - DATA_DIR=/tmp/localstack/data + - DOCKER_HOST=unix:///var/run/docker.sock + - DEFAULT_REGION=us-east-1 + volumes: + - "./tmp/localstack:/var/lib/localstack" + db: + image: postgres:latest + restart: always + env_file: + - .env + environment: + POSTGRES_USER: ${DATABASE_USER} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD} + POSTGRES_DB: ${DATABASE_NAME} + ports: + - "${DATABASE_PORT}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + + app: + build: . + depends_on: + - db + volumes: + - .:/app + environment: + DATABASE_URI: postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@db:${DATABASE_PORT}/${DATABASE_NAME} + FLASK_APP: run.py + FLASK_ENV: development + ports: + - "8080:8080" + - "5678:5678" + command: python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m flask run --host=0.0.0.0 --port=8080 + +volumes: + postgres_data: \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 1cca4817..c69c4935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ python = ">=3.10" jsonschema = ">=4" python-dotenv = ">=1" deepdiff = ">=6" -pytest = ">=7.3.2" ckanapi = ">=4.7" beautifulsoup4 = "^4.12.2" sansjson = "^0.3.0" @@ -37,9 +36,10 @@ flask-bootstrap = "^3.3.7.1" cloudfoundry-client = "^1.36.0" [tool.poetry.group.dev.dependencies] -pytest = "^7.3.0" +pytest = ">=7.3.2" ruff = "^0.0.261" pytest-cov = "^4.0.0" +debugpy = "^1.8.1" [tool.ruff] # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..aa9d6045 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,11 @@ +colorama==0.4.6 ; python_version >= "3.10" and sys_platform == "win32" +coverage[toml]==7.4.3 ; python_version >= "3.10" +debugpy==1.8.1 ; python_version >= "3.10" +exceptiongroup==1.2.0 ; python_version < "3.11" and python_version >= "3.10" +iniconfig==2.0.0 ; python_version >= "3.10" +packaging==23.2 ; python_version >= "3.10" +pluggy==1.4.0 ; python_version >= "3.10" +pytest-cov==4.1.0 ; python_version >= "3.10" +pytest==7.4.4 ; python_version >= "3.10" +ruff==0.0.261 ; python_version >= "3.10" +tomli==2.0.1 ; python_full_version <= "3.11.0a6" and python_version >= "3.10" diff --git a/requirements.txt b/requirements.txt index 024027d8..1afdea48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,61 @@ -# poetry not supported by cloud.gov, use this file instead for manually push - -flask-migrate==4.0.7 -flask-sqlalchemy==3.1.1 -flask==3.0.2 -psycopg2-binary==2.9.9 -pytest==7.4.4 -python-dotenv==1.0.1 -flask-wtf==1.2.1 -flask_bootstrap -werkzeug>=2.3.8 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file +aiohttp==3.9.3 ; python_version >= "3.10" +aiosignal==1.3.1 ; python_version >= "3.10" +alembic==1.13.1 ; python_version >= "3.10" +async-timeout==4.0.3 ; python_version < "3.11" and python_version >= "3.10" +attrs==23.2.0 ; python_version >= "3.10" +beautifulsoup4==4.12.3 ; python_version >= "3.10" +blinker==1.7.0 ; python_version >= "3.10" +boto3==1.34.54 ; python_version >= "3.10" +botocore==1.34.54 ; python_version >= "3.10" +certifi==2024.2.2 ; python_version >= "3.10" +charset-normalizer==3.3.2 ; python_version >= "3.10" +ckanapi==4.7 ; python_version >= "3.10" +click==8.1.7 ; python_version >= "3.10" +cloudfoundry-client==1.36.0 ; python_version >= "3.10" +colorama==0.4.6 ; python_version >= "3.10" and platform_system == "Windows" +deepdiff==6.7.1 ; python_version >= "3.10" +docopt==0.6.2 ; python_version >= "3.10" +dominate==2.9.1 ; python_version >= "3.10" +flask-bootstrap==3.3.7.1 ; python_version >= "3.10" +flask-migrate==4.0.7 ; python_version >= "3.10" +flask-sqlalchemy==3.1.1 ; python_version >= "3.10" +flask-wtf==1.2.1 ; python_version >= "3.10" +flask==3.0.2 ; python_version >= "3.10" +frozenlist==1.4.1 ; python_version >= "3.10" +greenlet==3.0.3 ; (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.10" +idna==3.6 ; python_version >= "3.10" +itsdangerous==2.1.2 ; python_version >= "3.10" +jinja2==3.1.3 ; python_version >= "3.10" +jmespath==1.0.1 ; python_version >= "3.10" +jsonschema-specifications==2023.12.1 ; python_version >= "3.10" +jsonschema==4.21.1 ; python_version >= "3.10" +mako==1.3.2 ; python_version >= "3.10" +markupsafe==2.1.5 ; python_version >= "3.10" +multidict==6.0.5 ; python_version >= "3.10" +oauth2-client==1.4.2 ; python_version >= "3.10" +ordered-set==4.1.0 ; python_version >= "3.10" +polling2==0.5.0 ; python_version >= "3.10" +protobuf==4.25.3 ; python_version >= "3.10" +psycopg2-binary==2.9.9 ; python_version >= "3.10" +python-dateutil==2.9.0.post0 ; python_version >= "3.10" +python-dotenv==1.0.1 ; python_version >= "3.10" +python-slugify==8.0.4 ; python_version >= "3.10" +pyyaml==6.0.1 ; python_version >= "3.10" +referencing==0.33.0 ; python_version >= "3.10" +requests==2.31.0 ; python_version >= "3.10" +rpds-py==0.18.0 ; python_version >= "3.10" +s3transfer==0.10.0 ; python_version >= "3.10" +sansjson==0.3.0 ; python_version >= "3.10" +setuptools==69.1.1 ; python_version >= "3.10" +simplejson==3.19.2 ; python_version >= "3.10" +six==1.16.0 ; python_version >= "3.10" +soupsieve==2.5 ; python_version >= "3.10" +sqlalchemy==2.0.28 ; python_version >= "3.10" +text-unidecode==1.3 ; python_version >= "3.10" +typing-extensions==4.10.0 ; python_version >= "3.10" +urllib3==2.0.7 ; python_version >= "3.10" +visitor==0.1.3 ; python_version >= "3.10" +websocket-client==1.7.0 ; python_version >= "3.10" +werkzeug==3.0.1 ; python_version >= "3.10" +wtforms==3.1.2 ; python_version >= "3.10" +yarl==1.9.4 ; python_version >= "3.10" From 958878c6459a087f1921e29c11ebd05e0001c0b8 Mon Sep 17 00:00:00 2001 From: Tyler Burton Date: Wed, 24 Apr 2024 13:41:02 -0500 Subject: [PATCH 02/11] updates poetry.lock --- app/__init__.py | 1 + app/forms.py | 5 +- app/interface.py | 2 +- app/routes.py | 54 +++++--- app/templates/harvest_source.html | 71 ----------- app/templates/index.html | 203 +++++++++++++++--------------- app/templates/source_form.html | 51 -------- poetry.lock | 40 +++++- 8 files changed, 174 insertions(+), 253 deletions(-) delete mode 100644 app/templates/harvest_source.html delete mode 100644 app/templates/source_form.html diff --git a/app/__init__.py b/app/__init__.py index 9dcc4682..d5ddf6f3 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -10,6 +10,7 @@ DATABASE_URI = os.getenv('DATABASE_URI') def create_app(testing=False): + app = Flask(__name__) if testing: diff --git a/app/forms.py b/app/forms.py index dcb6fc1f..e50c3cc4 100644 --- a/app/forms.py +++ b/app/forms.py @@ -14,7 +14,7 @@ class HarvestSourceForm(FlaskForm): choices=[], validators=[DataRequired()]) name = StringField('Name', validators=[DataRequired()]) url = StringField('URL', validators=[DataRequired(), URL()]) - emails = TextAreaField('Notification_emails', + notification_emails = TextAreaField('Notification_emails', validators=[DataRequired(), validate_email_list]) frequency = SelectField('Frequency', choices=['Manual', 'Daily', 'Weekly', 'Biweekly','Monthly'], @@ -27,8 +27,7 @@ class HarvestSourceForm(FlaskForm): source_type = SelectField('Source Type', choices=['Datajson', 'WAF'], validators=[DataRequired()]) - submit = SubmitField('Submit') - + class OrganizationForm(FlaskForm): name = StringField('Name', validators=[DataRequired()]) logo = StringField('Logo', validators=[DataRequired()]) diff --git a/app/interface.py b/app/interface.py index 025e016c..475c6702 100644 --- a/app/interface.py +++ b/app/interface.py @@ -129,7 +129,7 @@ def update_harvest_source(self, source_id, updates): print(f"Warning: non-existing field '{key}' in HarvestSource") self.db.commit() - return self._to_dict(source) + return source except NoResultFound: self.db.rollback() diff --git a/app/routes.py b/app/routes.py index 6b908231..d348c8f0 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,10 +1,23 @@ -from flask import Blueprint, request, render_template, jsonify +from flask import Blueprint, request, render_template, jsonify, flash, redirect from .interface import HarvesterDBInterface from .forms import HarvestSourceForm, OrganizationForm mod = Blueprint("harvest", __name__) db = HarvesterDBInterface() +# convenience method to extract form data +def make_new_source(form): + return { + "name": form.name.data, + "notification_emails": form.notification_emails.data.replace('\r\n', ', '), + "frequency": form.frequency.data, + "user_requested_frequency": form.frequency.data, + "url": form.url.data, + "schema_type": form.schema_type.data, + "source_type": form.source_type.data, + "organization_id": form.organization_id.data + } + @mod.route("/", methods=["GET"]) def index(): return render_template("index.html") @@ -68,36 +81,43 @@ def add_harvest_source(): return jsonify({"error": "Failed to add harvest source."}), 400 else: if form.validate_on_submit(): - new_source = { - "name": form.name.data, - "notification_emails": form.emails.data.replace('\r\n', ', '), - "frequency": form.frequency.data, - "user_requested_frequency": form.frequency.data, - "url": form.url.data, - "schema_type": form.schema_type.data, - "source_type": form.source_type.data, - "organization_id": form.organization_id.data - } + new_source = make_new_source(form) source=db.add_harvest_source(new_source) if source: - return f"Added new source with ID: {source.id}" + flash(f"Updated source with ID: {source.id}") else: - return "Failed to add harvest source." - return render_template("source_form.html", form=form, choices=organization_choices) + flash("Failed to add harvest source.") + return redirect('/') + return render_template("data_form.html", form=form, action="Add", data_type="Harvest Source", button="Submit") # test interface, will remove later @mod.route("/get_harvest_source", methods=["GET"]) def get_all_harvest_sources(): source = db.get_all_harvest_sources() org = db.get_all_organizations() - return render_template("harvest_source.html", sources=source, organizations=org) + return render_template("get_harvest_source.html", sources=source, organizations=org) @mod.route("/harvest_source/", methods=["GET"]) -@mod.route("/harvest_source/", methods=["GET"]) +@mod.route("/harvest_source/", methods=["GET", "POST"]) def get_harvest_source(source_id=None): if source_id: source = db.get_harvest_source(source_id) - return jsonify(source) if source else ("Not Found", 404) + organizations = db.get_all_organizations() + organization_choices = [(str(org["id"]), f'{org["name"]} - {org["id"]}') + for org in organizations] + form = HarvestSourceForm(data=source) + form.organization_id.choices = organization_choices + if form.validate_on_submit(): + new_source_data = make_new_source(form) + print(new_source_data) + source=db.update_harvest_source(source_id, new_source_data) + print(source) + if source: + flash(f"Updated source with ID: {source.id}") + else: + flash("Failed to add harvest source.") + return redirect('/') + return render_template("data_form.html", form=form, action="Edit", data_type="Harvest Source", button="Update") organization_id = request.args.get("organization_id") if organization_id: diff --git a/app/templates/harvest_source.html b/app/templates/harvest_source.html deleted file mode 100644 index 56d821a9..00000000 --- a/app/templates/harvest_source.html +++ /dev/null @@ -1,71 +0,0 @@ - - - - - Harvest Sources - - - - - -
-

Harvest Sources

- -
- - -
- Get Source - /harvest_source/ -
- -
- - -
- Get Organization Sources - /harvest_source/?organization_id= -
- - - - diff --git a/app/templates/index.html b/app/templates/index.html index 3542181f..09b217ac 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -1,104 +1,99 @@ -# flake8: noqa - - - - - Flask routes - - - -
-

Harvest Actions

-
    -
  • - Add Organization [Form]
    • -
  • - Add Harvest Source [Form]
    • - -
  • - Get Harvest Source by id or org
    • - -
  • Add Job/Record/Error:
    • - - -
  • GET:
    • - - -
  • UPDATE:
    • - - -
  • DELETE:
    • - - -
  • Reference for testing:
    • -
  • - Get All Organizations
    • -
  • - Get All Harvest Sources
    • -
  • - Get All Harvest Jobs
    • -
  • - Get All Harvest Records (for testing)
    • -
  • - Get All Harvest Errors (for testing)
    • -
-
- - - +{% extends 'base.html' %} + +{% block title %} +Flask routes +{% endblock %} + + +{% block content %} +

Harvest Actions

+ +
    +
  • + Add Organization [Form]
    • +
  • + Add Harvest Source [Form]
    • + +
  • + Get Harvest Source by id or org
    • + +
  • Add Job/Record/Error:
    • + + +
  • GET:
    • + + +
  • UPDATE:
    • + + +
  • DELETE:
    • + + +
  • Reference for testing:
    • +
  • + Get All Organizations
    • +
  • + Get All Harvest Sources
    • +
  • + Get All Harvest Jobs
    • +
  • + Get All Harvest Records (for testing)
    • +
  • + Get All Harvest Errors (for testing)
    • +
+ +{% endblock %} \ No newline at end of file diff --git a/app/templates/source_form.html b/app/templates/source_form.html deleted file mode 100644 index bd79814f..00000000 --- a/app/templates/source_form.html +++ /dev/null @@ -1,51 +0,0 @@ - - - - - Add Harvest Source - - - -
-

Add Harvest Source

-
- {{ form.hidden_tag() }} -
- {{ form.organization_id.label(class_='form-control-label') }}: - {{ form.organization_id(class_='form-control') }} -
-
- {{ form.name.label(class_='form-control-label') }}: - {{ form.name(class_='form-control') }} -
-
- {{ form.emails.label(class_='form-control-label') }}: - {{ form.emails(class_='form-control') }} -
-
- {{ form.url.label(class_='form-control-label') }}: - {{ form.url(class_='form-control') }} -
-
- {{ form.frequency.label(class_='form-control-label') }}: - {{ form.frequency(class_='form-control') }} -
-
- {{ form.user_requested_frequency.label(class_='form-control-label') }}: - {{ form.user_requested_frequency(class_='form-control') }} -
-
- {{ form.schema_type.label(class_='form-control-label') }}: - {{ form.schema_type(class_='form-control') }} -
-
- {{ form.source_type.label(class_='form-control-label') }}: - {{ form.source_type(class_='form-control') }} -
- -
-
- - - - diff --git a/poetry.lock b/poetry.lock index 21945ba2..092c8d95 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -469,6 +469,37 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "debugpy" +version = "1.8.1" +description = "An implementation of the Debug Adapter Protocol for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "debugpy-1.8.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:3bda0f1e943d386cc7a0e71bfa59f4137909e2ed947fb3946c506e113000f741"}, + {file = "debugpy-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dda73bf69ea479c8577a0448f8c707691152e6c4de7f0c4dec5a4bc11dee516e"}, + {file = "debugpy-1.8.1-cp310-cp310-win32.whl", hash = "sha256:3a79c6f62adef994b2dbe9fc2cc9cc3864a23575b6e387339ab739873bea53d0"}, + {file = "debugpy-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:7eb7bd2b56ea3bedb009616d9e2f64aab8fc7000d481faec3cd26c98a964bcdd"}, + {file = "debugpy-1.8.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:016a9fcfc2c6b57f939673c874310d8581d51a0fe0858e7fac4e240c5eb743cb"}, + {file = "debugpy-1.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd97ed11a4c7f6d042d320ce03d83b20c3fb40da892f994bc041bbc415d7a099"}, + {file = "debugpy-1.8.1-cp311-cp311-win32.whl", hash = "sha256:0de56aba8249c28a300bdb0672a9b94785074eb82eb672db66c8144fff673146"}, + {file = "debugpy-1.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:1a9fe0829c2b854757b4fd0a338d93bc17249a3bf69ecf765c61d4c522bb92a8"}, + {file = "debugpy-1.8.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3ebb70ba1a6524d19fa7bb122f44b74170c447d5746a503e36adc244a20ac539"}, + {file = "debugpy-1.8.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2e658a9630f27534e63922ebf655a6ab60c370f4d2fc5c02a5b19baf4410ace"}, + {file = "debugpy-1.8.1-cp312-cp312-win32.whl", hash = "sha256:caad2846e21188797a1f17fc09c31b84c7c3c23baf2516fed5b40b378515bbf0"}, + {file = "debugpy-1.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:edcc9f58ec0fd121a25bc950d4578df47428d72e1a0d66c07403b04eb93bcf98"}, + {file = "debugpy-1.8.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:7a3afa222f6fd3d9dfecd52729bc2e12c93e22a7491405a0ecbf9e1d32d45b39"}, + {file = "debugpy-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d915a18f0597ef685e88bb35e5d7ab968964b7befefe1aaea1eb5b2640b586c7"}, + {file = "debugpy-1.8.1-cp38-cp38-win32.whl", hash = "sha256:92116039b5500633cc8d44ecc187abe2dfa9b90f7a82bbf81d079fcdd506bae9"}, + {file = "debugpy-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:e38beb7992b5afd9d5244e96ad5fa9135e94993b0c551ceebf3fe1a5d9beb234"}, + {file = "debugpy-1.8.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:bfb20cb57486c8e4793d41996652e5a6a885b4d9175dd369045dad59eaacea42"}, + {file = "debugpy-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd3fdd3f67a7e576dd869c184c5dd71d9aaa36ded271939da352880c012e703"}, + {file = "debugpy-1.8.1-cp39-cp39-win32.whl", hash = "sha256:58911e8521ca0c785ac7a0539f1e77e0ce2df753f786188f382229278b4cdf23"}, + {file = "debugpy-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:6df9aa9599eb05ca179fb0b810282255202a66835c6efb1d112d21ecb830ddd3"}, + {file = "debugpy-1.8.1-py2.py3-none-any.whl", hash = "sha256:28acbe2241222b87e255260c76741e1fbf04fdc3b6d094fcf57b6c6f75ce1242"}, + {file = "debugpy-1.8.1.zip", hash = "sha256:f696d6be15be87aef621917585f9bb94b1dc9e8aced570db1b8a6fc14e8f9b42"}, +] + [[package]] name = "deepdiff" version = "6.7.1" @@ -501,7 +532,6 @@ files = [ name = "dominate" version = "2.9.1" description = "Dominate is a Python library for creating and manipulating HTML documents using an elegant DOM API." -category = "main" optional = false python-versions = ">=3.4" files = [ @@ -549,7 +579,6 @@ dotenv = ["python-dotenv"] name = "flask-bootstrap" version = "3.3.7.1" description = "An extension that includes Bootstrap in your project, without any boilerplate code." -category = "main" optional = false python-versions = "*" files = [ @@ -1764,7 +1793,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} typing-extensions = ">=4.6.0" [package.extras] @@ -1846,7 +1875,6 @@ zstd = ["zstandard (>=0.18.0)"] name = "visitor" version = "0.1.3" description = "A tiny pythonic visitor implementation." -category = "main" optional = false python-versions = "*" files = [ @@ -2009,4 +2037,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10" -content-hash = "caee85df1023e73c48fa835d0f7b3d39fc645d0cb3e1c01b4f717f7d8d4f23f3" +content-hash = "8c6ddf9ceeea5e2eb3cc2fe82ed8fd015ee67fed518f5f70777134d26fabb0ba" From 1dcf08a0c38d38f61bd986651ecdbda36c2cfc6c Mon Sep 17 00:00:00 2001 From: Tyler Burton Date: Wed, 24 Apr 2024 13:49:10 -0500 Subject: [PATCH 03/11] touch --- app/templates/base.html | 33 ++++++++++++ app/templates/data_form.html | 16 ++++++ app/templates/get_harvest_source.html | 77 +++++++++++++++++++++++++++ harvester/harvest.py | 4 +- 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 app/templates/base.html create mode 100644 app/templates/data_form.html create mode 100644 app/templates/get_harvest_source.html diff --git a/app/templates/base.html b/app/templates/base.html new file mode 100644 index 00000000..f7ce7dc9 --- /dev/null +++ b/app/templates/base.html @@ -0,0 +1,33 @@ + + + + + {% block title %} + {{action}} {{data_type}} + {% endblock %} + + + +
+ +
+
+ {% for message in get_flashed_messages() %} +
+ + {{ message }} +
+ {% endfor %} + +
{% block content %}{% endblock %}
+
+ + + + + \ No newline at end of file diff --git a/app/templates/data_form.html b/app/templates/data_form.html new file mode 100644 index 00000000..35f467a9 --- /dev/null +++ b/app/templates/data_form.html @@ -0,0 +1,16 @@ +{% extends 'base.html' %} + +{% block content %} +

{{action}} {{data_type}}

+
+ {{ form.hidden_tag() }} + {% for field in form if field.widget.input_type != 'hidden' %} +
+ {{field.label(class_='form-control-label')}} + {{field(class_='form-control')}} +
+ {% endfor %} + + +
+{% endblock %} \ No newline at end of file diff --git a/app/templates/get_harvest_source.html b/app/templates/get_harvest_source.html new file mode 100644 index 00000000..e2a8a771 --- /dev/null +++ b/app/templates/get_harvest_source.html @@ -0,0 +1,77 @@ + + + + + Harvest Sources + + + + + +
+

Harvest Sources

+ {% for message in get_flashed_messages() %} +
+ + {{ message }} +
+ {% endfor %} + +
+ + +
+ Get Source + /harvest_source/ +
+ +
+ + +
+ Get Organization Sources + /harvest_source/?organization_id= +
+ + + + diff --git a/harvester/harvest.py b/harvester/harvest.py index 32a158b5..acff5b2e 100644 --- a/harvester/harvest.py +++ b/harvester/harvest.py @@ -288,7 +288,7 @@ def get_ckan_records_as_id_hash(self) -> None: logger.info("retrieving and preparing ckan records") try: self.ckan_to_id_hash(self.get_ckan_records(results=[])) - except Exception as e: # ruff: noqa: E841 + except Exception as e: # noqa: E841 # TODO: do something with 'e' raise ExtractCKANSourceException( f"{self.title} {self.url} failed to extract ckan records. exiting.", @@ -307,7 +307,7 @@ def get_harvest_records_as_id_hash(self) -> None: self.harvest_to_id_hash( self.download_waf(self.traverse_waf(self.url, **self.waf_config)) ) - except Exception as e: # ruff: noqa: E841 + except Exception as e: # noqa: E841 raise ExtractHarvestSourceException( f"{self.title} {self.url} failed to extract harvest source. exiting", self.job_id, From 91c1e9e802eb2b0c0bbfa87206a4788cdfe744ca Mon Sep 17 00:00:00 2001 From: Tyler Burton Date: Wed, 24 Apr 2024 17:04:42 -0500 Subject: [PATCH 04/11] standardizes more templates; adds edit org template --- app/__init__.py | 4 +- app/forms.py | 3 +- app/interface.py | 2 +- app/routes.py | 46 ++++++--- app/static/css/styles.css | 11 +++ app/static/styles.css | 0 app/templates/base.html | 8 +- app/templates/data_form.html | 6 +- app/templates/get_harvest_source.html | 128 +++++++++++--------------- app/templates/get_orgs.html | 36 ++++++++ app/templates/index.html | 4 +- 11 files changed, 155 insertions(+), 93 deletions(-) create mode 100644 app/static/css/styles.css delete mode 100644 app/static/styles.css create mode 100644 app/templates/get_orgs.html diff --git a/app/__init__.py b/app/__init__.py index d5ddf6f3..1dad3453 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -11,7 +11,9 @@ def create_app(testing=False): - app = Flask(__name__) + app = Flask(__name__, + static_url_path="", + static_folder="static") if testing: app.config['TESTING'] = True diff --git a/app/forms.py b/app/forms.py index e50c3cc4..9dcf2f44 100644 --- a/app/forms.py +++ b/app/forms.py @@ -30,5 +30,4 @@ class HarvestSourceForm(FlaskForm): class OrganizationForm(FlaskForm): name = StringField('Name', validators=[DataRequired()]) - logo = StringField('Logo', validators=[DataRequired()]) - submit = SubmitField('Submit') \ No newline at end of file + logo = StringField('Logo', validators=[DataRequired()]) \ No newline at end of file diff --git a/app/interface.py b/app/interface.py index 475c6702..839f9850 100644 --- a/app/interface.py +++ b/app/interface.py @@ -67,7 +67,7 @@ def update_organization(self, org_id, updates): print(f"Warning: non-existing field '{key}' in organization") self.db.commit() - return self._to_dict(org) + return org except NoResultFound: self.db.rollback() diff --git a/app/routes.py b/app/routes.py index d348c8f0..88cbdbc8 100644 --- a/app/routes.py +++ b/app/routes.py @@ -18,6 +18,12 @@ def make_new_source(form): "organization_id": form.organization_id.data } +def make_new_org(form): + return { + "name": form.name.data, + "logo": form.logo.data + } + @mod.route("/", methods=["GET"]) def index(): return render_template("index.html") @@ -40,25 +46,35 @@ def add_organization(): } org=db.add_organization(new_org) if org: - return f"Added new organization with ID: {org.id}" + flash(f"Added new organization with ID: {org.id}") else: - return "Failed to add organization." - return render_template("org_form.html", form=form) + flash("Failed to add organization.") + return redirect('/') + return render_template("data_form.html", form=form, action="Add", data_type="Organization", button="Submit") @mod.route("/organization", methods=["GET"]) -@mod.route("/organization/", methods=["GET"]) +@mod.route("/organization/", methods=["GET", "POST"]) def get_organization(org_id=None): if org_id: org = db.get_organization(org_id) - return jsonify(org) if org else ("Not Found", 404) + form = OrganizationForm(data=org) + if form.validate_on_submit(): + new_org_data = make_new_org(form) + org = db.update_organization(org_id, new_org_data) + if org: + flash(f"Updated org with ID: ${org.id}") + else: + flash("Failed to update organization.") + return redirect('/') + return render_template("data_form.html", form=form, action="Edit", data_type="Organization", button="Update") else: org = db.get_all_organizations() return org -@mod.route("/organization/", methods=["PUT"]) -def update_organization(org_id): - result = db.update_organization(org_id, request.json) - return result +# @mod.route("/organization/", methods=["PUT"]) +# def update_organization(org_id): +# result = db.update_organization(org_id, request.json) +# return result @mod.route("/organization/", methods=["DELETE"]) def delete_organization(org_id): @@ -97,6 +113,12 @@ def get_all_harvest_sources(): org = db.get_all_organizations() return render_template("get_harvest_source.html", sources=source, organizations=org) +# test interface, will remove later +@mod.route("/get_organizations", methods=["GET"]) +def get_all_organizations(): + org = db.get_all_organizations() + return render_template("get_orgs.html", organizations=org) + @mod.route("/harvest_source/", methods=["GET"]) @mod.route("/harvest_source/", methods=["GET", "POST"]) def get_harvest_source(source_id=None): @@ -109,13 +131,11 @@ def get_harvest_source(source_id=None): form.organization_id.choices = organization_choices if form.validate_on_submit(): new_source_data = make_new_source(form) - print(new_source_data) - source=db.update_harvest_source(source_id, new_source_data) - print(source) + source = db.update_harvest_source(source_id, new_source_data) if source: flash(f"Updated source with ID: {source.id}") else: - flash("Failed to add harvest source.") + flash("Failed to update harvest source.") return redirect('/') return render_template("data_form.html", form=form, action="Edit", data_type="Harvest Source", button="Update") diff --git a/app/static/css/styles.css b/app/static/css/styles.css new file mode 100644 index 00000000..65e356c1 --- /dev/null +++ b/app/static/css/styles.css @@ -0,0 +1,11 @@ +ul.menu { + list-style-type: none; +} + +ul.menu > li { + display: inline-block; +} + +ul.menu > li::after { + content: ' |' +} \ No newline at end of file diff --git a/app/static/styles.css b/app/static/styles.css deleted file mode 100644 index e69de29b..00000000 diff --git a/app/templates/base.html b/app/templates/base.html index f7ce7dc9..967cf231 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -1,10 +1,11 @@ - + {% block title %} {{action}} {{data_type}} {% endblock %} + @@ -12,7 +13,8 @@ @@ -29,5 +31,7 @@ + {% block scripts %} + {% endblock %} \ No newline at end of file diff --git a/app/templates/data_form.html b/app/templates/data_form.html index 35f467a9..c0949e26 100644 --- a/app/templates/data_form.html +++ b/app/templates/data_form.html @@ -1,7 +1,11 @@ {% extends 'base.html' %} +{% block title %} +{{action}} {{data_type}} +{% endblock %} + {% block content %} -

{{action}} {{data_type}}

+

{{action}} {{data_type}}

{{ form.hidden_tag() }} {% for field in form if field.widget.input_type != 'hidden' %} diff --git a/app/templates/get_harvest_source.html b/app/templates/get_harvest_source.html index e2a8a771..0d36ae7e 100644 --- a/app/templates/get_harvest_source.html +++ b/app/templates/get_harvest_source.html @@ -1,77 +1,61 @@ - - - - - Harvest Sources - - - - - -
-

Harvest Sources

- {% for message in get_flashed_messages() %} -
- - {{ message }} -
+{% extends 'base.html' %} + +{% block title %} +Get Harvest Sources +{% endblock %} + +{% block content %} +
+ + - - {% for source in sources %} - - {% endfor %} - -
- Get Source - /harvest_source/ -
- -
- - -
- Get Organization Sources - /harvest_source/?organization_id= -
+ + +Get Source +/harvest_source/ +
+ +
+ + +
+Get Organization Sources +/harvest_source/?organization_id= + +{% endblock %} + +{% block scripts %} + +{% endblock %} - - - diff --git a/app/templates/get_orgs.html b/app/templates/get_orgs.html new file mode 100644 index 00000000..693d5ced --- /dev/null +++ b/app/templates/get_orgs.html @@ -0,0 +1,36 @@ +{% extends 'base.html' %} + +{% block title %} +Get Organizations +{% endblock %} + +{% block content %} +
+ + +
+Get Organization +/organization + +{% endblock %} + +{% block scripts %} + +{% endblock %} + diff --git a/app/templates/index.html b/app/templates/index.html index 09b217ac..ca07dcbe 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -4,7 +4,6 @@ Flask routes {% endblock %} - {% block content %}

Harvest Actions

@@ -17,6 +16,9 @@

Harvest Actions

  • Get Harvest Source by id or org
    • +
  • + Get Organizations
    • +
  • Add Job/Record/Error: