ROB-1258 Live-refresh git-synced skills via first-class skill_repos

helm/holmes/templates/_helpers.tpl

@@ -76,3 +76,34 @@ Usage: {{- include "holmes.commonLabels" . | nindent 4 }}
 {{- end }}
 {{- end }}
 {{- end }}
+
+{{/*
+Checksum of the Secret DATA behind every skillRepos credential, for the
+pod-template annotation: env comes from these Secrets via secretKeyRef, which
+does not restart pods when the data rotates, so the rollout must be triggered
+here. Same lookup pattern as holmes.kubernetesRemediationMcp.authTokenChecksum;
+during `helm template` (no cluster) lookup returns nothing and the checksum is
+stable, exactly like that precedent.
+*/}}
+{{- define "holmes.skillRepoSecretsChecksum" -}}
+{{- $parts := list -}}
+{{- range .Values.skillRepos -}}
+{{- with .tokenSecret -}}
+{{- $existing := lookup "v1" "Secret" $.Release.Namespace .name -}}
+{{- if and $existing $existing.data (hasKey $existing.data .key) -}}
+{{- $parts = append $parts (dict "name" .name "key" .key "value" (index $existing.data .key)) -}}
+{{- end -}}
+{{- end -}}
+{{- with .githubApp -}}
+{{- with .privateKeySecret -}}
+{{- $existing := lookup "v1" "Secret" $.Release.Namespace .name -}}
+{{- if and $existing $existing.data (hasKey $existing.data .key) -}}
+{{- $parts = append $parts (dict "name" .name "key" .key "value" (index $existing.data .key)) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- /* Structured records, not joined values: concatenation would hash two
+       adjacent secrets whose contents merely shifted a boundary identically. */ -}}
+{{- toJson $parts | sha256sum -}}
+{{- end -}}

helm/holmes/templates/holmes.yaml

@@ -39,7 +39,13 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}
         # checksum annotation triggering pod reload when .Values.toolsets changes by helm upgrade
-        checksum/toolset-config: {{ list .Values.toolsets .Values.modelList .Values.mcp_servers .Values.customSkillPaths .Values.customSkills | toYaml | sha256sum }}
+        checksum/toolset-config: {{ list .Values.toolsets .Values.modelList .Values.mcp_servers .Values.customSkillPaths .Values.customSkills .Values.skillRepos | toYaml | sha256sum }}
+        {{- if .Values.skillRepos }}
+        # Rolls the pod when a skill repo credential Secret's DATA rotates —
+        # the token env vars come from secretKeyRef, which a Secret update
+        # alone does not refresh.
+        checksum/skill-repo-secrets: {{ include "holmes.skillRepoSecretsChecksum" . }}
+        {{- end }}
         {{- if and .Values.mcpAddons.kubernetesRemediation.enabled .Values.mcpAddons.kubernetesRemediation.auth.enabled }}
         # Roll this pod when the remediation MCP auth token rotates — the token
         # is read from a Secret into env, so a content change alone would not
@@ -194,6 +200,46 @@ spec:
           - name: CUSTOM_SKILL_PATHS
             value: {{ join "," $skillPaths | quote }}
           {{- end }}
+          {{- if .Values.skillRepos }}
+          {{- /* One pass per repo: build its SKILL_REPOS entry and emit its
+                 secret-backed env vars together, so the env var names and the
+                 token_env / github_app_private_key_env fields referencing them
+                 cannot drift apart. */}}
+          {{- $skillRepos := list }}
+          {{- range $i, $repo := .Values.skillRepos }}
+          {{- if and $repo.tokenSecret $repo.githubApp }}
+          {{- fail (printf "skillRepos[%d]: set either tokenSecret or githubApp, not both" $i) }}
+          {{- end }}
+          {{- $entry := dict "url" (required "skillRepos entries need a url" $repo.url) }}
+          {{- with $repo.name }}{{- $entry = merge $entry (dict "name" .) }}{{- end }}
+          {{- with $repo.branch }}{{- $entry = merge $entry (dict "branch" .) }}{{- end }}
+          {{- with $repo.subPath }}{{- $entry = merge $entry (dict "sub_path" .) }}{{- end }}
+          {{- with $repo.username }}{{- $entry = merge $entry (dict "username" .) }}{{- end }}
+          {{- with $repo.tokenSecret }}
+          {{- $entry = merge $entry (dict "token_env" (printf "SKILL_REPO_TOKEN_%d" $i)) }}
+          - name: SKILL_REPO_TOKEN_{{ $i }}
+            valueFrom:
+              secretKeyRef:
+                name: {{ required "skillRepos tokenSecret needs a name" .name }}
+                key: {{ required "skillRepos tokenSecret needs a key" .key }}
+          {{- end }}
+          {{- with $repo.githubApp }}
+          {{- $entry = merge $entry (dict
+                "github_app_id" (toString (required "skillRepos githubApp needs appId" .appId))
+                "github_app_installation_id" (toString (required "skillRepos githubApp needs installationId" .installationId))
+                "github_app_private_key_env" (printf "SKILL_REPO_GH_APP_KEY_%d" $i)) }}
+          {{- with .apiUrl }}{{- $entry = merge $entry (dict "github_api_url" .) }}{{- end }}
+          - name: SKILL_REPO_GH_APP_KEY_{{ $i }}
+            valueFrom:
+              secretKeyRef:
+                name: {{ required "skillRepos githubApp privateKeySecret needs a name" (required "skillRepos githubApp needs privateKeySecret" .privateKeySecret).name }}
+                key: {{ required "skillRepos githubApp privateKeySecret needs a key" .privateKeySecret.key }}
+          {{- end }}
+          {{- $skillRepos = append $skillRepos $entry }}
+          {{- end }}
+          - name: SKILL_REPOS
+            value: {{ toJson $skillRepos | quote }}
+          {{- end }}
           {{- if .Values.additionalEnvVars -}}
           {{ toYaml .Values.additionalEnvVars | nindent 10 }}
           {{- end }}

helm/holmes/values.yaml

@@ -168,6 +168,32 @@ customSkills: {}
   #     ## Goal
   #     ...
 
+# Git repositories to sync skills from. Holmes clones each repo at startup and
+# re-pulls it periodically (every TOOLSET_STATUS_REFRESH_INTERVAL_SECONDS,
+# default 5 minutes), so pushed skill changes reach a running agent without a
+# pod restart. The chart sets SKILL_REPOS in the Holmes pod from this list and,
+# for entries with tokenSecret, injects the token env var from that Secret.
+# After rotating a credential Secret, run `helm upgrade`: the chart checksums
+# the Secret data into the pod template, so the upgrade rolls the pod with the
+# new credential. A Secret edit alone does not reach running pods (env comes
+# from secretKeyRef); without an upgrade, restart the Deployment yourself.
+skillRepos: []
+  # - name: team-skills            # optional; defaults to the repo name from the URL
+  #   url: https://github.com/acme/holmes-skills.git
+  #   branch: main                 # optional; defaults to the repo's default branch
+  #   subPath: skills              # optional; subdirectory holding the SKILL.md dirs
+  #   username: oauth2             # optional; "x-token-auth" for Bitbucket tokens
+  #   tokenSecret:                 # omit for public repos / GitHub App auth
+  #     name: holmes-skills-git-credentials
+  #     key: token
+  #   githubApp:                   # alternative to tokenSecret: GitHub App auth.
+  #     appId: "123456"            # Holmes mints short-lived installation tokens
+  #     installationId: "7890"     # from the App's private key on every sync.
+  #     privateKeySecret:
+  #       name: holmes-github-app
+  #       key: GITHUB_APP_PRIVATE_KEY
+  #     apiUrl: https://api.github.com   # optional; override for GitHub Enterprise
+
 resources:
   requests:
     cpu: 100m

holmes/config.py

@@ -24,6 +24,11 @@
 from holmes.core.tools_utils.tool_executor import ToolExecutor
 from holmes.core.toolset_manager import ToolsetManager
 from holmes.core.transformers.llm_summarize import LLMSummarizeTransformer
+from holmes.plugins.skills.git_skill_repos import (
+    GitSkillRepo,
+    GitSkillRepoManager,
+    parse_skill_repos_env,
+)
 from holmes.plugins.skills.skill_loader import (
     SkillCatalog,
     load_skill_catalog,
@@ -39,6 +44,7 @@
     from holmes.plugins.sources.pagerduty import PagerDutySource
     from holmes.plugins.sources.prometheus.plugin import AlertManagerSource
 
+from holmes.common.env_vars import TOOLSET_STATUS_REFRESH_INTERVAL_SECONDS
 from holmes.core.config import config_path_dir
 from holmes.core.oauth_utils import eager_load_oauth_tools, preload_oauth_tokens, set_oauth_dal
 from holmes.core.supabase_dal import SupabaseDal
@@ -133,6 +139,10 @@ class Config(RobustaBaseConfig):
     opsgenie_query: Optional[str] = None
 
     custom_skill_paths: List[Union[str, FilePath]] = []
+    # Git repositories to sync skills from (re-pulled periodically by the server's
+    # refresh loop; synced once per run in the CLI). Their checkouts are appended
+    # to the effective skill paths -- see all_skill_paths.
+    skill_repos: List[GitSkillRepo] = []
 
     # custom_toolsets is passed from config file, and be used to override built-in toolsets, provides 'stable' customized toolset.
     # The status of custom toolsets can be cached.
@@ -160,6 +170,7 @@ class Config(RobustaBaseConfig):
 
     # TODO: Separate those fields to facade class, this shouldn't be part of the config.
     _toolset_manager: Optional[ToolsetManager] = PrivateAttr(None)
+    _skill_repo_manager: Optional[GitSkillRepoManager] = PrivateAttr(None)
     _llm_model_registry: Optional[LLMModelRegistry] = PrivateAttr(None)
     _dal: Optional[SupabaseDal] = PrivateAttr(None)
     _config_file_path: Optional[Path] = PrivateAttr(None)
@@ -183,12 +194,35 @@ def toolset_manager(self) -> ToolsetManager:
                 mcp_servers=self.mcp_servers,
                 custom_toolsets=self.custom_toolsets,
                 custom_toolsets_from_cli=self.custom_toolsets_from_cli,
-                custom_skill_paths=self.custom_skill_paths,
+                custom_skill_paths=self.all_skill_paths,
                 config_file_path=self._config_file_path,
                 additional_toolsets=self.additional_toolsets,
             )
         return self._toolset_manager
 
+    @property
+    def skill_repo_manager(self) -> GitSkillRepoManager:
+        if not self._skill_repo_manager:
+            # The manager rate-limits itself to the refresh cadence, so callers
+            # (the server refresh loop included) can invoke sync() freely.
+            self._skill_repo_manager = GitSkillRepoManager(
+                self.skill_repos,
+                min_sync_interval_seconds=TOOLSET_STATUS_REFRESH_INTERVAL_SECONDS,
+            )
+        return self._skill_repo_manager
+
+    @property
+    def all_skill_paths(self) -> List[Union[str, FilePath]]:
+        """Every path skills load from: configured paths plus git-repo checkouts.
+
+        The repo paths are `current` symlinks that keep pointing at the newest
+        checkout across syncs, so this list is stable even though the content
+        behind it moves. Accessing it clones the repos on first use.
+        """
+        paths: List[Union[str, FilePath]] = list(self.custom_skill_paths)
+        paths.extend(self.skill_repo_manager.skill_paths())
+        return paths
+
     @property
     def dal(self) -> SupabaseDal:
         if not self._dal:
@@ -256,7 +290,7 @@ def load_from_file(cls, config_file: Optional[Path], **kwargs) -> "Config":
         return result
 
     def _apply_env_fallbacks(self) -> None:
-        """Apply MODEL and CUSTOM_SKILL_PATHS when absent after YAML load/reload."""
+        """Apply MODEL, CUSTOM_SKILL_PATHS and SKILL_REPOS when absent after YAML load/reload."""
         if self.model is None:
             model_from_env = os.environ.get("MODEL")
             if model_from_env and model_from_env.strip():
@@ -268,6 +302,11 @@ def _apply_env_fallbacks(self) -> None:
             if skill_paths:
                 self.custom_skill_paths = skill_paths
 
+        if not self.skill_repos:
+            skill_repos = parse_skill_repos_env()
+            if skill_repos:
+                self.skill_repos = skill_repos
+
     @classmethod
     def load_from_env(cls):
         kwargs = {}
@@ -296,14 +335,14 @@ def load_from_env(cls):
             val = os.getenv(field_name.upper(), None)
             if val is not None:
                 kwargs[field_name] = val
-        skill_paths = _parse_custom_skill_paths_env()
-        if skill_paths:
-            kwargs["custom_skill_paths"] = skill_paths
         kwargs["cluster_name"] = Config.__get_cluster_name()
         if kwargs["cluster_name"] and not os.environ.get("CLUSTER_NAME"):
             os.environ["CLUSTER_NAME"] = kwargs["cluster_name"]
         kwargs["should_try_robusta_ai"] = True
         result = cls(**kwargs)
+        # CUSTOM_SKILL_PATHS / SKILL_REPOS share one env-fallback path with
+        # load_from_file, so the two loaders cannot drift.
+        result._apply_env_fallbacks()
         if "model" in kwargs:
             result._model_source = "via $MODEL"
         result.log_useful_info()
@@ -351,7 +390,7 @@ def get_skill_catalog(
         hierarchy = self.dal.get_skill_hierarchy_config()
         return load_skill_catalog(
             dal=self.dal,
-            custom_skill_paths=self.custom_skill_paths,
+            custom_skill_paths=self.all_skill_paths,
             user_id=user_id,
             hierarchy=hierarchy,
             alert_name=alert_name,
@@ -547,8 +586,15 @@ def reload_toolsets(self) -> dict:
                 self.mcp_servers = fresh.mcp_servers
                 self.custom_toolsets = fresh.custom_toolsets
                 self.custom_skill_paths = fresh.custom_skill_paths
+                previous_skill_repos = self.skill_repos
+                self.skill_repos = fresh.skill_repos
                 self.additional_toolsets = fresh.additional_toolsets
                 self._apply_env_fallbacks()
+                # Keep the manager when the repo config is unchanged: it holds
+                # the synced state and cached GitHub App tokens, and dropping it
+                # would force a full network re-sync on the next request.
+                if self.skill_repos != previous_skill_repos:
+                    self._skill_repo_manager = None
             self._toolset_manager = None
             self._cached_tool_executor = None
             self._cached_executor_key = None

holmes/core/toolset_manager.py

@@ -150,8 +150,16 @@ def _list_all_toolsets(
         # Extract search paths from custom skill paths
         additional_search_paths = None
         if self.custom_skill_paths:
+            # Absolutize without resolving symlinks: a git-synced skill repo is
+            # published through a `current` symlink that flips on every update,
+            # and resolving it here would pin this long-lived toolset to one
+            # commit's checkout. Scans resolve at scan time instead.
+            # dirname only for an explicit SKILL.md file path -- a path that
+            # does not exist yet (a repo whose first sync failed) must pass
+            # through unchanged so scans report it unreadable rather than
+            # walking its parent directory.
             additional_search_paths = [
-                str(Path(p).resolve()) if Path(p).is_dir() else os.path.dirname(os.path.abspath(str(p)))
+                os.path.dirname(os.path.abspath(str(p))) if Path(p).is_file() else os.path.abspath(str(p))
                 for p in self.custom_skill_paths
             ]