adding segment based tools (#33)

kdr · web-flow · commit 9f64985ca331 · 2025-09-26T16:28:47.000-07:00
changes
- bumps version of cloudglue-js sdk
- bumps package version
- introduces two new tools (segment video chapters and segment video
camera shots) which use new segments api
- updates readme with description of tools
diff --git a/README.md b/README.md
@@ -109,6 +109,10 @@ The following Cloudglue tools are available to LLMs through this MCP server:
 
 - **`get_video_metadata`**: Get comprehensive technical metadata about a Cloudglue video file including duration, resolution, file size, processing status, and computed statistics. Use this when you need video specifications, file details, or processing information rather than content analysis. Different from content-focused tools like describe_video.
 
+- **`segment_video_camera_shots`**: Segment videos into camera shots with intelligent cost optimization. Automatically checks for existing shot segmentation jobs before creating new ones. Returns timestamps and metadata for each camera shot detected. Supports Cloudglue URLs and direct HTTP video URLs. Note: YouTube URLs are not supported for segmentation.
+
+- **`segment_video_chapters`**: Segment videos into chapters with intelligent cost optimization. Automatically checks for existing chapter segmentation jobs before creating new ones. Returns timestamps and descriptions for each chapter detected. Supports Cloudglue URLs and direct HTTP video URLs. Note: YouTube URLs are not supported for segmentation.
+
 ### **Collection Analysis**
 
 - **`retrieve_summaries`**: Bulk retrieve video summaries and titles from a collection to quickly understand its content and themes. Works with both rich-transcripts and media-descriptions collections. Perfect for getting a high-level overview of what's in a collection, identifying common topics, or determining if a collection contains relevant content for a specific query. Use this as your first step when analyzing a collection - it's more efficient than retrieving full descriptions and helps you determine if you need more detailed information. Only proceed to retrieve_descriptions if you need the full multimodal context for specific videos identified through the summaries. For targeted content discovery, consider using search_video_summaries or search_video_moments instead of browsing through all summaries. **Pagination guidance**: For comprehensive collection analysis, paginate through all summaries (check `has_more` and increment `offset` by `limit`) to ensure complete coverage. Use larger limits (25-50) for efficient bulk analysis, smaller limits (5-10) for targeted exploration.
@@ -147,7 +151,7 @@ The following Cloudglue tools are available to LLMs through this MCP server:
 ### **When to Use Which Tool**
 
 - **Start exploring**: Use `list_collections` and `list_videos` to explore available content
-- **For single videos**: Use `describe_video` or `extract_video_entities`
+- **For single videos**: Use `describe_video`, `extract_video_entities`, `segment_video_camera_shots`, or `segment_video_chapters`
 - **For collection overview**: Always start with `retrieve_summaries` to efficiently understand what's in a collection
 - **For detailed analysis**: Only use `retrieve_descriptions` for specific videos that need full multimodal context, identified through summaries
 - **For structured data**: Use `retrieve_entities` for bulk entity extraction
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@aviaryhq/cloudglue-mcp-server",
   "type": "module",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "main": "index.js",
   "bin": {
     "cloudglue-mcp-server": "build/index.js"
@@ -24,7 +24,7 @@
   "license": "Elastic-2.0",
   "description": "Cloudglue MCP Server",
   "dependencies": {
-    "@aviaryhq/cloudglue-js": "^0.2.1",
+    "@aviaryhq/cloudglue-js": "^0.3.0",
     "@modelcontextprotocol/sdk": "^1.18.0",
     "dotenv": "^16.4.7",
     "shx": "^0.4.0",
diff --git a/src/index.ts b/src/index.ts
@@ -16,6 +16,8 @@ import { registerSearchVideoMoments } from "./tools/search-video-moments.js";
 import { registerSearchVideoSummaries } from "./tools/search-video-summaries.js";
 import { registerGetVideoMetadata } from "./tools/get-video-metadata.js";
 import { registerRetrieveSummaries } from "./tools/retrieve-summaries.js";
+import { registerSegmentVideoCameraShots } from "./tools/segment-video-camera-shots.js";
+import { registerSegmentVideoChapters } from "./tools/segment-video-chapters.js";
 
 // Parse command line arguments
 const { values: args } = parseArgs({
@@ -61,6 +63,8 @@ registerRetrieveEntities(server, cgClient);
 registerSearchVideoMoments(server, cgClient);
 registerSearchVideoSummaries(server, cgClient);
 registerGetVideoMetadata(server, cgClient);
+registerSegmentVideoCameraShots(server, cgClient);
+registerSegmentVideoChapters(server, cgClient);
 
 // Run server
 async function main() {
diff --git a/src/tools/segment-video-camera-shots.ts b/src/tools/segment-video-camera-shots.ts
@@ -0,0 +1,127 @@
+import { z } from "zod";
+import { CloudGlue } from "@aviaryhq/cloudglue-js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+
+export const schema = {
+  url: z
+    .string()
+    .describe("Video URL to segment into camera shots. Supports Cloudglue URLs (cloudglue://files/file-id) or direct HTTP video URLs. For Cloudglue URLs, use the file ID from list_videos. Note: YouTube URLs are not supported for segmentation."),
+};
+
+// Helper function to format time in seconds to HH:MM:SS format
+function formatTime(seconds: number): string {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor((seconds % 3600) / 60);
+  const secs = Math.floor(seconds % 60);
+  
+  if (hours > 0) {
+    return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
+  } else {
+    return `${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
+  }
+}
+
+export function registerSegmentVideoCameraShots(
+  server: McpServer,
+  cgClient: CloudGlue,
+) {
+  server.tool(
+    "segment_video_camera_shots",
+    "Segment videos into camera shots with intelligent cost optimization. Automatically checks for existing shot segmentation jobs before creating new ones. Returns timestamps and metadata for each camera shot detected. Supports Cloudglue URLs and direct HTTP video URLs. Note: YouTube URLs are not supported for segmentation.",
+    schema,
+    async ({ url }) => {
+      // Check if it's a YouTube URL (not supported)
+      if (url.includes('youtube.com') || url.includes('youtu.be')) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: "Error: YouTube URLs are not supported for video segmentation. Please use Cloudglue URLs or direct HTTP video URLs instead.",
+            },
+          ],
+        };
+      }
+
+      // Step 1: Check for existing shot segmentation jobs for this URL
+      try {
+        const existingJobs = await cgClient.segments.listSegmentJobs({ 
+          criteria: "shot",
+          url: url,
+          status: "completed",
+          limit: 1
+        });
+        
+        if (existingJobs.data && existingJobs.data.length > 0) {
+          const job = existingJobs.data[0];
+          if (job.segments && job.segments.length > 0) {
+            const segmentsText = job.segments.map((segment, index) => {
+              const startTime = formatTime(segment.start_time);
+              const endTime = formatTime(segment.end_time);
+              const duration = (segment.end_time - segment.start_time).toFixed(1);
+              return `Shot ${index + 1}: ${startTime} - ${endTime} (${duration}s)`;
+            }).join('\n');
+
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `Found existing camera shot segmentation:\n\n${segmentsText}\n\nTotal shots: ${job.segments.length}`,
+                },
+              ],
+            };
+          }
+        }
+      } catch (error) {
+        // Continue to create new job if listing fails
+      }
+
+      // Step 2: Create new shot segmentation job
+      try {
+        const segmentJob = await cgClient.segments.createSegmentJob({
+          url: url,
+          criteria: "shot",
+        });
+
+        // Wait for completion using SDK's waitForReady method
+        const completedJob = await cgClient.segments.waitForReady(segmentJob.job_id);
+
+        if (completedJob.status === "completed" && completedJob.segments) {
+          const segmentsText = completedJob.segments.map((segment, index) => {
+            const startTime = formatTime(segment.start_time);
+            const endTime = formatTime(segment.end_time);
+            const duration = (segment.end_time - segment.start_time).toFixed(1);
+            return `Shot ${index + 1}: ${startTime} - ${endTime} (${duration}s)`;
+          }).join('\n');
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: `New camera shot segmentation created:\n\n${segmentsText}\n\nTotal shots: ${completedJob.segments.length}`,
+              },
+            ],
+          };
+        }
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: "Error: Failed to create camera shot segmentation - job did not complete successfully",
+            },
+          ],
+        };
+
+      } catch (error) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Error creating camera shot segmentation: ${error instanceof Error ? error.message : 'Unknown error'}`,
+            },
+          ],
+        };
+      }
+    },
+  );
+}
diff --git a/src/tools/segment-video-chapters.ts b/src/tools/segment-video-chapters.ts
@@ -0,0 +1,135 @@
+import { z } from "zod";
+import { CloudGlue } from "@aviaryhq/cloudglue-js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+
+export const schema = {
+  url: z
+    .string()
+    .describe("Video URL to segment into chapters. Supports Cloudglue URLs (cloudglue://files/file-id) or direct HTTP video URLs. For Cloudglue URLs, use the file ID from list_videos. Note: YouTube URLs are not supported for segmentation."),
+  prompt: z
+    .string()
+    .describe("Custom prompt to guide chapter detection. Describe what types of chapters or segments you want to identify. Examples: 'Identify main topics and transitions', 'Find scene changes and key moments', 'Segment by speaker changes and topics'. Leave empty to use default chapter detection.")
+    .optional(),
+};
+
+// Helper function to format time in seconds to HH:MM:SS format
+function formatTime(seconds: number): string {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor((seconds % 3600) / 60);
+  const secs = Math.floor(seconds % 60);
+  
+  if (hours > 0) {
+    return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
+  } else {
+    return `${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
+  }
+}
+
+export function registerSegmentVideoChapters(
+  server: McpServer,
+  cgClient: CloudGlue,
+) {
+  server.tool(
+    "segment_video_chapters",
+    "Segment videos into chapters with intelligent cost optimization. Automatically checks for existing chapter segmentation jobs before creating new ones. Returns timestamps and descriptions for each chapter detected. Supports Cloudglue URLs and direct HTTP video URLs. Note: YouTube URLs are not supported for segmentation.",
+    schema,
+    async ({ url, prompt }) => {
+      // Check if it's a YouTube URL (not supported)
+      if (url.includes('youtube.com') || url.includes('youtu.be')) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: "Error: YouTube URLs are not supported for video segmentation. Please use Cloudglue URLs or direct HTTP video URLs instead.",
+            },
+          ],
+        };
+      }
+
+      // Step 1: Check for existing narrative segmentation jobs for this URL
+      try {
+        const existingJobs = await cgClient.segments.listSegmentJobs({ 
+          criteria: "narrative",
+          url: url,
+          status: "completed",
+          limit: 1
+        });
+        
+        if (existingJobs.data && existingJobs.data.length > 0) {
+          const job = existingJobs.data[0];
+          if (job.segments && job.segments.length > 0) {
+            const chaptersText = job.segments.map((segment, index) => {
+              const startTime = formatTime(segment.start_time);
+              const description = segment.description || `Chapter ${index + 1}`;
+              return `Chapter ${index + 1}: ${startTime} - ${description}`;
+            }).join('\n');
+
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `Found existing chapter segmentation:\n\n${chaptersText}\n\nTotal chapters: ${job.segments.length}`,
+                },
+              ],
+            };
+          }
+        }
+      } catch (error) {
+        // Continue to create new job if listing fails
+      }
+
+      // Step 2: Create new narrative segmentation job
+      try {
+        const narrativeConfig: any = {};
+        if (prompt) {
+          narrativeConfig.prompt = prompt;
+        }
+
+        const segmentJob = await cgClient.segments.createSegmentJob({
+          url: url,
+          criteria: "narrative",
+          narrative_config: narrativeConfig,
+        });
+
+        // Wait for completion using SDK's waitForReady method
+        const completedJob = await cgClient.segments.waitForReady(segmentJob.job_id);
+
+        if (completedJob.status === "completed" && completedJob.segments) {
+          const chaptersText = completedJob.segments.map((segment, index) => {
+            const startTime = formatTime(segment.start_time);
+            const description = segment.description || `Chapter ${index + 1}`;
+            return `Chapter ${index + 1}: ${startTime} - ${description}`;
+          }).join('\n');
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: `New chapter segmentation created:\n\n${chaptersText}\n\nTotal chapters: ${completedJob.segments.length}`,
+              },
+            ],
+          };
+        }
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: "Error: Failed to create chapter segmentation - job did not complete successfully",
+            },
+          ],
+        };
+
+      } catch (error) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Error creating chapter segmentation: ${error instanceof Error ? error.message : 'Unknown error'}`,
+            },
+          ],
+        };
+      }
+    },
+  );
+}
