Merge pull request #100 from huridocs/ner-extractor

Add gliner date parser and the flair NER extractors

.dockerignore

@@ -1,7 +1,7 @@
 /venv/
 /.venv/
 .git
-/docker_volume/
+/models_data/
 /src/tenant_test/
 /huggingface/
 /data/

.github/workflows/test.yml

@@ -37,8 +37,11 @@ jobs:
           url: http://localhost:5056
           method: GET
           expected-status: 200
-          timeout: 60000
+          timeout: 240000
           interval: 500
 
+      - name: Wait for queues
+        run: make wait_for_queues
+
       - name: Test with unittest
         run: make test

.gitignore

@@ -15,7 +15,7 @@ core
 src/.pytest_cache
 outputs
 /performance_results/mistakes/
-/docker_volume/
+/models_data/
 /huggingface/
 /.ruff_cache/
 /data/

Dockerfile

@@ -2,7 +2,7 @@ FROM pytorch/pytorch
 
 RUN apt-get update && apt-get -y -q --no-install-recommends install libgomp1 pdftohtml
 RUN apt-get -y install git
-RUN mkdir -p /app/src /app/docker_volume
+RUN mkdir -p /app/src /app/models_data
 
 RUN addgroup --system python && adduser --system --group python
 RUN chown -R python:python /app
@@ -20,11 +20,9 @@ WORKDIR /app
 COPY ./src ./src
 
 ENV PYTHONPATH "${PYTHONPATH}:/app/src"
-ENV NLTK_DATA=/app/docker_volume/cache/nltk_data
-ENV HF_DATASETS_CACHE=/app/docker_volume/cache/HF
-ENV HF_HOME=/app/docker_volume/cache/HF_home
-ENV TRANSFORMERS_CACHE=/app/docker_volume/cache/Transformers
-ENV HF_HOME=/app/docker_volume/cache/Transformers
+ENV NLTK_DATA=/app/models_data/cache/nltk_data
+ENV HF_DATASETS_CACHE=/app/models_data/cache/HF
+ENV HF_HOME=/app/models_data/cache/HF_home
 ENV TRANSFORMERS_VERBOSITY=error
 ENV TRANSFORMERS_NO_ADVISORY_WARNINGS=1
 ENV CUDA_VISIBLE_DEVICES=0

Makefile

@@ -18,6 +18,9 @@ check_format:
 test:
 	. .venv/bin/activate; command cd src; command python -m pytest
 
+wait_for_queues:
+	. .venv/bin/activate; command cd src/scripts; command python wait_for_queues.py
+
 remove_docker_containers:
 	docker compose ps -q | xargs docker rm
 
@@ -46,9 +49,6 @@ stop:
 delete_queues:
 	. .venv/bin/activate; python scripts/delete_queues.py
 
-download_models:
-	. .venv/bin/activate; command cd src; python download_models.py
-
 gpu:
 	. .venv/bin/activate; command cd src; python is_gpu_available.py
 

README.md

@@ -266,7 +266,7 @@ If the service is running, the end point definitions can be founded in the follo
 
 The end points code can be founded inside the file `app.py`.
 
-The errors are reported to the file `docker_volume/service.log`, if the configuration is not changed (
+The errors are reported to the file `models_data/service.log`, if the configuration is not changed (
 see [Get service logs](#get-service-logs))
 
 ## Queue processor

docker-compose.yml

@@ -9,7 +9,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     depends_on:
       - pdf_metadata_extraction_worker
     environment:
@@ -29,7 +29,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     shm_size: '11gb'
     depends_on:
       - mongo_metadata_extraction

gpu-docker-compose.yml

@@ -9,7 +9,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     depends_on:
       - mongo_metadata_extraction
       - pdf_metadata_extraction_worker
@@ -34,7 +34,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     shm_size: '11gb'
     depends_on:
       - mongo_metadata_extraction

local-docker-compose.yml

@@ -9,7 +9,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     depends_on:
       - mongo_metadata_extraction
       - pdf_metadata_extraction_worker
@@ -27,7 +27,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     shm_size: '11gb'
     depends_on:
       - mongo_metadata_extraction

local-gpu-docker-compose.yml

@@ -9,7 +9,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     depends_on:
       - mongo_metadata_extraction
       - pdf_metadata_extraction_worker
@@ -34,7 +34,7 @@ services:
       context: .
       dockerfile: Dockerfile
     volumes:
-      - data:/app/docker_volume
+      - data:/app/models_data
     shm_size: '11gb'
     depends_on:
       - mongo_metadata_extraction