Deploy to ZIP download

Author: cybermaggedon

grafana/dashboards/dashboard.json

@@ -0,0 +1,17 @@
+
+apiVersion: 1
+
+providers:
+
+  - name: 'trustgraph.ai'
+    orgId: 1
+    folder: 'TrustGraph'
+    folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228'
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false
+

grafana/provisioning/dashboard.yml

@@ -0,0 +1,21 @@
+apiVersion: 1
+
+prune: true
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    orgId: 1
+    # <string> Sets a custom UID to reference this
+    # data source in other parts of the configuration.
+    # If not specified, Grafana generates one.
+    uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129'
+
+    url: http://prometheus:9090
+
+    basicAuth: false
+    withCredentials: false
+    isDefault: true
+    editable: true
+

grafana/provisioning/datasource.yml

@@ -11,10 +11,13 @@
   },
   "dependencies": {
     "@emotion/styled": "^11.13.0",
+    "@mui/icons-material": "^6.1.2",
     "@mui/material": "^6.0.1",
     "@mui/styled-engine": "^6.0.1",
+    "jszip": "^3.10.1",
     "react": "^18.3.1",
-    "react-dom": "^18.3.1"
+    "react-dom": "^18.3.1",
+    "zustand": "^5.0.0-rc.2"
   },
   "devDependencies": {
     "@eslint/js": "^9.9.0",

package-lock.json

@@ -0,0 +1,147 @@
+global:
+
+  scrape_interval:     15s # By default, scrape targets every 15 seconds.
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+    monitor: 'trustgraph'
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+
+  # The job name is added as a label `job=<job_name>` to any timeseries
+  # scraped from this config.
+
+  - job_name: 'pdf-decoder'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'pdf-decoder:8000'
+
+  - job_name: 'chunker'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'chunker:8000'
+
+
+  - job_name: 'vectorize'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'vectorize:8000'
+
+
+  - job_name: 'embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'embeddings:8000'
+
+
+  - job_name: 'kg-extract-definitions'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'kg-extract-definitions:8000'
+
+
+  - job_name: 'kg-extract-topics'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'kg-extract-topics:8000'
+
+
+  - job_name: 'kg-extract-relationships'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'kg-extract-relationships:8000'
+
+
+  - job_name: 'metering'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'metering:8000'
+
+
+  - job_name: 'metering-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'metering-rag:8000'
+
+
+  - job_name: 'store-graph-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'store-graph-embeddings:8000'
+
+
+  - job_name: 'store-triples'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'store-triples:8000'
+
+
+  - job_name: 'text-completion'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'text-completion:8000'
+
+
+  - job_name: 'text-completion-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'text-completion-rag:8000'
+
+
+  - job_name: 'graph-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'graph-rag:8000'
+
+
+  - job_name: 'prompt'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'prompt:8000'
+
+
+  - job_name: 'prompt-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'prompt-rag:8000'
+
+
+  - job_name: 'query-graph-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'query-graph-embeddings:8000'
+
+
+  - job_name: 'query-triples'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'query-triples:8000'
+
+
+  - job_name: 'pulsar'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'pulsar:8080'
+

package.json

@@ -6,15 +6,19 @@ import logging
 import os
 import json
 import yaml
+import zipfile
+from io import BytesIO
 
 logger = logging.getLogger("service")
 logger.setLevel(logging.INFO)
 
 class Generator:
 
-    def __init__(self, config, base="./templates/"):
+    def __init__(self, config, base="./templates/", version="0.0.0"):
+
         self.jsonnet_base = base
         self.config = config
+        self.version = f"\"{version}\"".encode("utf-8")
 
     def process(self, config):
 
@@ -23,25 +27,46 @@ class Generator:
 
     def load(self, dir, filename):
 
-        logger.info("Request jsonnet: %s %s", dir, filename)
+        logger.debug("Request jsonnet: %s %s", dir, filename)
 
         if filename == "config.json" and dir == "":
-            print("config.json...")
             path = os.path.join(".", dir, filename)
             return str(path), self.config
 
+        if filename == "version.jsonnet" and dir == "./templates/values/":
+            path = os.path.join(".", dir, filename)
+            return str(path), self.version
+
+        if dir:
+            candidates = [
+                os.path.join(".", dir, filename),
+                os.path.join(".", filename)
+            ]
+        else:
+            candidates = [
+                os.path.join(".", filename)
+            ]
+
         try:
-            if dir:
-                path = os.path.join(".", dir, filename)
-            else:
-                path = os.path.join(self.jsonnet_base, filename)
 
-            logger.debug("Try: %s", path)
-            with open(path, "rb") as f:
-                logger.debug("Loaded: %s", path)
-                return str(path), f.read()
+            if filename == "vertexai/private.json":
+
+                return candidates[0], private_json.encode("utf-8")
 
+            for c in candidates:
+                logger.debug("Try: %s", c)
+
+                if os.path.isfile(c):
+                    with open(c, "rb") as f:
+                        logger.debug("Loading: %s", c)
+                        return str(c), f.read()
+
+            raise RuntimeError(
+                f"Could not load file={filename} dir={dir}"
+            )
+                
         except:
+
             path = os.path.join(self.jsonnet_base, filename)
             logger.debug("Try: %s", path)
             with open(path, "rb") as f:
@@ -98,23 +123,71 @@ class Api:
 
         return web.HTTPNotFound()
 
+    def process(
+            self, config, version="0.11.19", platform="docker-compose",
+    ):
+
+        config = config.encode("utf-8")
+
+        gen = Generator(config, version=version)
+
+
+        with open(f"./templates/config-to-{platform}.jsonnet", "r") as f:
+            wrapper = f.read()
+
+        processed = gen.process(wrapper)
+
+        return processed
+
     async def generate(self, request):
 
         print("Generate...")
+
         config = await request.text()
 
         print(config)
 
-        config = config.encode("utf-8")
+        processed = self.process(config)
+        y = yaml.dump(processed)
 
-        gen = Generator(config)
+        mem = BytesIO()
 
-        with open("./templates/config-loader.jsonnet", "r") as f:
-            wrapper = f.read()
-        processed = gen.process(wrapper)
+        with zipfile.ZipFile(mem, mode='w') as out:
+
+            def output(name, content):
+                logger.info(f"Adding {name}...")
+                out.writestr(name, content)
+
+            fname = "docker-compose.yaml"
+
+            output(fname, y)
+
+            # Grafana config
+            with open("grafana/dashboards/dashboard.json") as f:
+                output(
+                    "grafana/dashboards/dashboard.json",
+                    f.read()
+                )
+
+            with open("grafana/provisioning/dashboard.yml") as f:
+                output(
+                    "grafana/provisioning/dashboard.yml",
+                    f.read()
+                )
+
+            with open("grafana/provisioning/datasource.yml") as f:
+                output(
+                    "grafana/provisioning/datasource.yml",
+                    f.read()
+                )
+
+            # Prometheus config
+            with open("prometheus/prometheus.yml") as f:
+                output("prometheus/prometheus.yml", f.read())
 
         return web.Response(
-            text=yaml.dump(processed), content_type = "text/plain"
+            body=mem.getvalue(),
+            content_type = "application/octet-stream"
         )
 
     def run(self):