Initial ingest logic

Author: AndriyMulyar

.gitignore

@@ -0,0 +1,108 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+app/__pycache__/
+*.py[cod]
+*$py.class
+*.pyc
+
+# C extensions
+*.so
+
+# VS Code
+.vscode/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

README.md

@@ -5,12 +5,17 @@ An open-source datalake to ingest, organize and efficiently store all data contr
 ### Architecture
 The core datalake architecture is a simple HTTP API (written in FastAPI) that ingests JSON in a fixed schema, performs some integrity checking and stores it. This JSON is transformed into storage efficient Arrow/Parquet files and stored in a target filesystem. A light-weight index of the entire parquet filesystem is maintained with DuckDB.
 
+#### Data formats
+- Data is stored on disk in parquet files in subdirectories organized by day. These parquet files have a standardized schema allowing for easy manipulation in any programming language.
+- 
+
 ### Open sourcing the data.
 Nomic AI will provide automatic snapshots of this raw parquet data.
 You will be able to interact with the snapshots:
 - In automatic [Atlas](https://atlas.nomic.ai/) maps over its raw, cleaned and curated form.
 - Through highly-processed downloads where the data has been curated, de-duplicated and cleaned for LLM training/finetuning.
 
+
 ### Data Privacy
 
 

api/.gitignore

@@ -0,0 +1,108 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+app/__pycache__/
+*.py[cod]
+*$py.class
+*.pyc
+
+# C extensions
+*.so
+
+# VS Code
+.vscode/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

api/Dockerfile.buildkit

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1.0.0-experimental
+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8
+
+ARG image_version=gpt4all_datalake-0.0.1
+ENV IMAGE_VERSION=$image_version
+
+# Put first so anytime this file changes other cached layers are invalidated.
+COPY api/requirements.txt /requirements.txt
+
+RUN pip install --upgrade pip
+
+# Run various pip install commands.
+RUN --mount=type=ssh pip install -r /requirements.txt && \
+  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
+
+
+# Finally, copy app and client.
+COPY api/app /app

api/app/__init__.py

@@ -0,0 +1,7 @@
+from api_v1.routes import health, ingest
+from fastapi import APIRouter
+
+router = APIRouter()
+
+router.include_router(health.router)
+router.include_router(ingest.router)

api/app/api_v1/api.py

@@ -0,0 +1,30 @@
+import logging
+
+from fastapi import HTTPException
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+
+from api_v1.settings import settings
+
+log = logging.getLogger(__name__)
+
+startup_msg_fmt = """
+ GPT4All Datalake started with {settings.app_environment} with version {settings.image_version}.
+"""
+
+
+async def on_http_error(request: Request, exc: HTTPException):
+    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
+
+
+async def on_startup(app):
+    startup_msg = startup_msg_fmt.format(settings=settings)
+    log.info(startup_msg)
+
+
+def startup_event_handler(app):
+    async def start_app() -> None:
+        await on_startup(app)
+
+    return start_app
+

api/app/api_v1/events.py

@@ -0,0 +1 @@
+from .models import *

api/app/api_v1/models/__init__.py

@@ -0,0 +1,36 @@
+from enum import Enum
+
+from typing import List, Optional
+from pydantic import BaseModel, Field
+
+
+class SuccessResponse(BaseModel):
+    result = 'ok'
+
+
+class Rating(str, Enum):
+    positive = "positive"
+    negative = "negative"
+
+class IngestMetadata(BaseModel):
+    source: Optional[str] = Field(None, description='The source contributing the ingest data.', example='gpt4all-chat')
+    submitter_id: str = Field(None, description='An identifier for the entity that submitted the ingest', example='EliteHacker#42')
+
+class ChatItem(BaseModel):
+    content: str = Field(..., description='The textual contents of the chat turn', example='Hello, how can I assist you today?')
+    role: Optional[str] = Field(None, description='The role of the entity that generated content.', example='assistant')
+    rating: Optional[Rating] = Field(None, description='A rating of the chat item', example='negative')
+    edited_content: Optional[str] = Field(None, description='An optional edited version of the content.', example='Hello, how may I assist you today?')
+
+
+class ChatIngestRequest(IngestMetadata):
+    agent_id: str = Field(..., description='An identifier for the entity in the conversation', example='gpt4all-j-v1.2-jazzy')
+    conversation: List[ChatItem] = Field(..., description='The conversation history.',
+                                         example=[{'content': 'Hello, how can I assist you today?',
+                                                   'role': 'assistant',
+                                                   'rating': 'negative',
+                                                   'edited_content': 'Hello, how may I assist you today?'},
+                                                  {'content': 'Write me python code to contribute data to the GPT4All Datalake!', 'role': 'user'}])
+
+class ChatIngestResponse(BaseModel):
+    ingest_id: str = Field(..., description='The id of the ingest', example='d920b363-abab-4d19-a5b6-89d182115f82')