feat: Initial indexing and scraping versions

Author: clemlesne

.devcontainer/devcontainer.json

@@ -0,0 +1,50 @@
+{
+  "name": "Development",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bookworm",
+  "forwardPorts": [3000, 8000],
+  "init": true,
+  "remoteEnv": {
+    "HOMEBREW_AUTO_UPDATE_SECS": "604800"
+  },
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "configureZshAsDefaultShell": true
+    },
+    "ghcr.io/devcontainers/features/azure-cli:1": {},
+    "ghcr.io/jlaundry/devcontainer-features/azure-functions-core-tools:1": {},
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+  },
+  "onCreateCommand": "cd /workspaces/scrape-it-now && python3 -m venv .venv",
+  "updateContentCommand": "cd /workspaces/scrape-it-now && . .venv/bin/activate && make upgrade install",
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "python.defaultInterpreterPath": ".venv/bin/python"
+      },
+      "extensions": [
+        "bierner.markdown-mermaid",
+        "bradlc.vscode-tailwindcss",
+        "DavidAnson.vscode-markdownlint",
+        "EditorConfig.EditorConfig",
+        "github.vscode-github-actions",
+        "mechatroner.rainbow-csv",
+        "mikestead.dotenv",
+        "ms-azuretools.vscode-azurefunctions",
+        "ms-azuretools.vscode-bicep",
+        "ms-python.black-formatter",
+        "ms-python.debugpy",
+        "ms-python.isort",
+        "ms-python.pylint",
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "ms-toolsai.jupyter",
+        "ms-vscode.azurecli",
+        "ms-vscode.makefile-tools",
+        "qwtel.sqlite-viewer",
+        "redhat.vscode-yaml",
+        "visualstudioexptteam.vscodeintellicode",
+        "wholroyd.jinja"
+      ]
+    }
+  }
+}

.editorconfig

@@ -0,0 +1,18 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.py]
+indent_size = 4
+
+[Makefile]
+indent_size = 4
+indent_style = tab

.env.example

@@ -0,0 +1,18 @@
+# OpenAI
+AZURE_OPENAI_API_KEY=xxx
+AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=gpt-4o-2024-05-13
+AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=text-embedding-3-large-1
+AZURE_OPENAI_EMBEDDING_DIMENSIONS=3072
+AZURE_OPENAI_ENDPOINT=https://xxx.openai.azure.com
+
+# AI Search
+AZURE_SEARCH_API_KEY=xxx
+AZURE_SEARCH_ENDPOINT=https://xxx.search.windows.net
+AZURE_SEARCH_INDEX_NAME=learn
+
+# Blob Storage
+AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=xxx;AccountKey=xxx;EndpointSuffix=core.windows.net
+AZURE_STORAGE_CONTAINER=learn-scraping
+
+# Application Insights
+APPLICATIONINSIGHTS_CONNECTION_STRING=xxx

.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: ""
+    target-branch: develop
+    schedule:
+      interval: weekly
+  - package-ecosystem: pip
+    directory: ""
+    target-branch: develop
+    schedule:
+      interval: weekly
+  - package-ecosystem: "devcontainers"
+    directory: ""
+    target-branch: develop
+    schedule:
+      interval: weekly

.github/workflows/codeql.yml

@@ -0,0 +1,42 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - develop
+      - main
+  pull_request:
+    branches:
+      - develop
+      - main
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+      security-events: write
+    runs-on: ubuntu-22.04
+    timeout-minutes: 360
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - build-mode: none
+            language: python
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          build-mode: ${{ matrix.build-mode }}
+          languages: ${{ matrix.language }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"