From 47e6724f8c3a53acb7c9935a45e8a1678764ad95 Mon Sep 17 00:00:00 2001 From: sr2echa <65058816+sr2echa@users.noreply.github.com> Date: Fri, 5 Sep 2025 20:44:03 +0530 Subject: [PATCH 1/5] feat: enhance AI functionality + ChatBox with context menu and utilities --- app/app.css | 50 +- app/components/chat/ChatBox.tsx | 1126 ++++++++++++++---- app/components/editor/LeftPanel.tsx | 11 +- app/components/timeline/MediaBin.tsx | 490 +++----- app/hooks/useTimeline.ts | 770 +++++------- app/routes/home.tsx | 43 +- app/utils/llm-handler.ts | 416 +++---- backend/main.py | 375 +++++- backend/poetry.lock | 1653 ++++++++++++++++++++++++++ backend/pyproject.toml | 1 + backend/schema.py | 59 +- backend/tools_registry.py | 191 +++ package.json | 1 + pnpm-lock.yaml | 23 + 14 files changed, 3877 insertions(+), 1332 deletions(-) create mode 100644 backend/poetry.lock create mode 100644 backend/tools_registry.py diff --git a/app/app.css b/app/app.css index ca09f38..b358301 100644 --- a/app/app.css +++ b/app/app.css @@ -4,8 +4,9 @@ @custom-variant dark (&:is(.dark *)); @theme { - --font-sans: "Inter", ui-sans-serif, system-ui, sans-serif, - "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; + --font-sans: + "Inter", ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", + "Noto Color Emoji"; } html, @@ -202,6 +203,27 @@ body { max-width: 100%; } + /* Ultra-thin scrollbar specifically for chat tabs strip */ + .chat-tabs-scroll::-webkit-scrollbar { + height: 0px; /* hide horizontal bar */ + } + .chat-tabs-scroll::-webkit-scrollbar-thumb { + background: transparent; + border-radius: 9999px; + } + .chat-tabs-scroll::-webkit-scrollbar-track { + background: transparent; + } + + /* Hide scrollbar utility (cross-browser) */ + .no-scrollbar::-webkit-scrollbar { + display: none; + } + .no-scrollbar { + -ms-overflow-style: none; /* IE and Edge */ + scrollbar-width: none; /* Firefox */ + } + /* Prevent horizontal overflow in chat areas */ .chat-container * { max-width: 100%; @@ -332,11 +354,14 @@ body { @keyframes glow-pulse { 0%, 100% { - box-shadow: 0 0 20px rgba(37, 99, 235, 0.1), + box-shadow: + 0 0 20px rgba(37, 99, 235, 0.1), 0 0 40px rgba(37, 99, 235, 0.05); } 50% { - box-shadow: 0 0 30px rgba(37, 99, 235, 0.2), 0 0 60px rgba(37, 99, 235, 0.1); + box-shadow: + 0 0 30px rgba(37, 99, 235, 0.2), + 0 0 60px rgba(37, 99, 235, 0.1); } } @@ -489,7 +514,16 @@ body { } @keyframes indeterminate-slide { - 0% { left: -40%; width: 40%; } - 50% { left: 20%; width: 60%; } - 100% { left: 100%; width: 40%; } -} \ No newline at end of file + 0% { + left: -40%; + width: 40%; + } + 50% { + left: 20%; + width: 60%; + } + 100% { + left: 100%; + width: 40%; + } +} diff --git a/app/components/chat/ChatBox.tsx b/app/components/chat/ChatBox.tsx index 55cf82c..404af87 100644 --- a/app/components/chat/ChatBox.tsx +++ b/app/components/chat/ChatBox.tsx @@ -1,4 +1,5 @@ import React, { useState, useRef, useEffect } from "react"; +import { parseDuration } from "@alwatr/parse-duration"; import { Send, Bot, @@ -11,13 +12,34 @@ import { ChevronLeft, ChevronRight, RotateCcw, + History, + Trash2, + Pencil, + Eraser, + CornerUpLeft, } from "lucide-react"; import { Button } from "~/components/ui/button"; import { - type MediaBinItem, - type TimelineState, - type ScrubberState, -} from "../timeline/types"; + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "~/components/ui/alert-dialog"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "~/components/ui/dropdown-menu"; +import { Separator } from "~/components/ui/separator"; +import { type MediaBinItem, type TimelineState, type ScrubberState } from "../timeline/types"; import { cn } from "~/lib/utils"; import axios from "axios"; import { apiUrl } from "~/utils/api"; @@ -28,6 +50,11 @@ import { llmMoveScrubber, llmAddScrubberByName, llmDeleteScrubbersInTrack, + llmResizeScrubber, + llmUpdateTextContent, + llmUpdateTextStyle, + llmMoveScrubbersByOffset, + llmSetResolution, } from "~/utils/llm-handler"; interface Message { @@ -35,16 +62,13 @@ interface Message { content: string; isUser: boolean; timestamp: Date; + snapshot?: TimelineState | null; } interface ChatBoxProps { className?: string; mediaBinItems: MediaBinItem[]; - handleDropOnTrack: ( - item: MediaBinItem, - trackId: string, - dropLeftPx: number - ) => void; + handleDropOnTrack: (item: MediaBinItem, trackId: string, dropLeftPx: number) => string; isMinimized?: boolean; onToggleMinimize?: () => void; messages: Message[]; @@ -52,6 +76,9 @@ interface ChatBoxProps { timelineState: TimelineState; handleUpdateScrubber: (updatedScrubber: ScrubberState) => void; handleDeleteScrubber?: (scrubberId: string) => void; + pixelsPerSecond: number; + handleAddTrack?: () => void; + restoreTimeline?: (state: TimelineState) => void; } export function ChatBox({ @@ -65,6 +92,9 @@ export function ChatBox({ timelineState, handleUpdateScrubber, handleDeleteScrubber, + pixelsPerSecond, + handleAddTrack, + restoreTimeline, }: ChatBoxProps) { const [inputValue, setInputValue] = useState(""); const [isTyping, setIsTyping] = useState(false); @@ -76,11 +106,162 @@ export function ChatBox({ const [textareaHeight, setTextareaHeight] = useState(36); // Starting height for proper size const [sendWithMedia, setSendWithMedia] = useState(false); // Track send mode const [mentionedItems, setMentionedItems] = useState([]); // Store actual mentioned items + const [contextMenu, setContextMenu] = useState<{ + open: boolean; + x: number; + y: number; + index: number; + message?: Message | null; + }>({ open: false, x: 0, y: 0, index: -1, message: null }); + const [showConfirmRestore, setShowConfirmRestore] = useState(false); + const [confirmRestoreIndex, setConfirmRestoreIndex] = useState(null); + const [showConfirmDelete, setShowConfirmDelete] = useState(false); + const [confirmDeleteIndex, setConfirmDeleteIndex] = useState(null); + const [showEdit, setShowEdit] = useState(false); + const [editIndex, setEditIndex] = useState(null); + const [editValue, setEditValue] = useState(""); + const [tabsMenu, setTabsMenu] = useState<{ open: boolean; x: number; y: number; tabId: string | null }>({ + open: false, + x: 0, + y: 0, + tabId: null, + }); + const headerRef = useRef(null); + const [historyWidthPx, setHistoryWidthPx] = useState(null); + const [historyQuery, setHistoryQuery] = useState(""); + const [isHistoryOpen, setIsHistoryOpen] = useState(false); + const [editingTabId, setEditingTabId] = useState(null); + const [editingTabName, setEditingTabName] = useState(""); + const [historyEditingId, setHistoryEditingId] = useState(null); + const [historyEditingName, setHistoryEditingName] = useState(""); + const tabsContainerRef = useRef(null); const messagesEndRef = useRef(null); const scrollContainerRef = useRef(null); + const scrollToTabId = (id: string) => { + const container = tabsContainerRef.current; + if (!container) return; + const el = container.querySelector(`[data-tab-id="${id}"]`); + if (!el) return; + const targetLeft = el.offsetLeft - container.clientWidth / 2 + el.clientWidth / 2; + container.scrollTo({ left: Math.max(0, targetLeft), behavior: "smooth" }); + }; const inputRef = useRef(null); const mentionsRef = useRef(null); const sendOptionsRef = useRef(null); + const latestTimelineRef = useRef(timelineState); + const STORAGE_KEY = "kimu.chat.tabs.v1"; + const ACTIVE_TAB_KEY = "kimu.chat.activeTab.v1"; + + const getRecencyGroup = (ts: number) => { + const now = Date.now(); + const diff = now - ts; + const oneHour = 60 * 60 * 1000; + const oneDay = 24 * oneHour; + const startOfToday = new Date(); + startOfToday.setHours(0, 0, 0, 0); + const startOfYesterday = new Date(startOfToday.getTime() - oneDay); + const startOfWeek = new Date(); + startOfWeek.setDate(startOfWeek.getDate() - startOfWeek.getDay()); + startOfWeek.setHours(0, 0, 0, 0); + + if (diff <= oneHour) return "Last hour"; + if (ts >= startOfToday.getTime()) return "Today"; + if (ts >= startOfYesterday.getTime()) return "Yesterday"; + if (ts >= startOfWeek.getTime()) return "This week"; + return "Older"; + }; + + type ChatTab = { + id: string; + name: string; + messages: Message[]; + timelineSnapshot: TimelineState | null; + createdAt: number; + }; + + const loadTabs = (): ChatTab[] => { + try { + const raw = localStorage.getItem(STORAGE_KEY); + if (!raw) return []; + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) { + return parsed.map((t: any) => ({ + id: String(t.id ?? Date.now().toString()), + name: String(t.name ?? "Chat"), + messages: Array.isArray(t.messages) + ? t.messages.map((m: any) => ({ + id: String(m.id ?? Date.now().toString()), + content: String(m.content ?? ""), + isUser: Boolean(m.isUser), + timestamp: m && m.timestamp ? new Date(m.timestamp) : new Date(), + })) + : [], + timelineSnapshot: t.timelineSnapshot ?? null, + createdAt: Number(t.createdAt ?? Date.now()), + })); + } + } catch {} + return []; + }; + + const [tabs, setTabs] = useState(() => { + const existing = loadTabs(); + if (existing.length) return existing; + return [{ id: Date.now().toString(), name: "Chat 1", messages: [], timelineSnapshot: null, createdAt: Date.now() }]; + }); + const [activeTabId, setActiveTabId] = useState(() => { + try { + const stored = localStorage.getItem(ACTIVE_TAB_KEY); + if (stored) return stored; + } catch {} + return tabs[0]?.id || ""; + }); + const activeTab = tabs.find((t) => t.id === activeTabId) || tabs[0]; + + useEffect(() => { + latestTimelineRef.current = timelineState; + }, [timelineState]); + + useEffect(() => { + if (activeTabId) { + scrollToTabId(activeTabId); + } + }, [activeTabId]); + + const persistTabs = (next: ChatTab[]) => { + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify(next)); + } catch {} + }; + + useEffect(() => { + persistTabs(tabs); + }, [tabs]); + + useEffect(() => { + const updateWidth = () => { + if (headerRef.current) { + const w = headerRef.current.offsetWidth; + setHistoryWidthPx(w); + } + }; + updateWidth(); + window.addEventListener("resize", updateWidth); + return () => window.removeEventListener("resize", updateWidth); + }, []); + + // Ensure activeTabId is valid after tabs change + useEffect(() => { + if (!tabs.find((t) => t.id === activeTabId)) { + setActiveTabId(tabs[0]?.id || ""); + } + }, [tabs, activeTabId]); + + // keep ChatBox external messages prop in sync with active tab + useEffect(() => { + if (!activeTab) return; + onMessagesChange(activeTab.messages); + }, [activeTabId]); // Auto-scroll to bottom when new messages are added useEffect(() => { @@ -92,25 +273,19 @@ export function ChatBox({ // Click outside handler for send options useEffect(() => { const handleClickOutside = (event: MouseEvent) => { - if ( - sendOptionsRef.current && - !sendOptionsRef.current.contains(event.target as Node) - ) { + if (sendOptionsRef.current && !sendOptionsRef.current.contains(event.target as Node)) { setShowSendOptions(false); } }; if (showSendOptions) { document.addEventListener("mousedown", handleClickOutside); - return () => - document.removeEventListener("mousedown", handleClickOutside); + return () => document.removeEventListener("mousedown", handleClickOutside); } }, [showSendOptions]); // Filter media bin items based on mention query - const filteredMentions = mediaBinItems.filter((item) => - item.name.toLowerCase().includes(mentionQuery.toLowerCase()) - ); + const filteredMentions = mediaBinItems.filter((item) => item.name.toLowerCase().includes(mentionQuery.toLowerCase())); // Handle input changes and @ mention detection const handleInputChange = (e: React.ChangeEvent) => { @@ -129,15 +304,9 @@ export function ChatBox({ // Clean up mentioned items that are no longer in the text const mentionPattern = /@(\w+(?:\s+\w+)*)/g; - const currentMentions = Array.from(value.matchAll(mentionPattern)).map( - (match) => match[1] - ); + const currentMentions = Array.from(value.matchAll(mentionPattern)).map((match) => match[1]); setMentionedItems((prev) => - prev.filter((item) => - currentMentions.some( - (mention) => mention.toLowerCase() === item.name.toLowerCase() - ) - ) + prev.filter((item) => currentMentions.some((mention) => mention.toLowerCase() === item.name.toLowerCase())), ); // Check for @ mentions @@ -147,9 +316,7 @@ export function ChatBox({ if (lastAtIndex !== -1) { const afterAt = beforeCursor.slice(lastAtIndex + 1); // Only show mentions if @ is at start or after whitespace, and no spaces after @ - const isValidMention = - (lastAtIndex === 0 || /\s/.test(beforeCursor[lastAtIndex - 1])) && - !afterAt.includes(" "); + const isValidMention = (lastAtIndex === 0 || /\s/.test(beforeCursor[lastAtIndex - 1])) && !afterAt.includes(" "); if (isValidMention) { setMentionQuery(afterAt); @@ -169,8 +336,7 @@ export function ChatBox({ const afterCursor = inputValue.slice(cursorPosition); const lastAtIndex = beforeCursor.lastIndexOf("@"); - const newValue = - beforeCursor.slice(0, lastAtIndex) + `@${item.name} ` + afterCursor; + const newValue = beforeCursor.slice(0, lastAtIndex) + `@${item.name} ` + afterCursor; setInputValue(newValue); setShowMentions(false); @@ -204,125 +370,337 @@ export function ChatBox({ // Add all media items to the items to send itemsToSend = [ ...mentionedItems, - ...mediaBinItems.filter( - (item) => - !mentionedItems.find((mentioned) => mentioned.id === item.id) - ), + ...mediaBinItems.filter((item) => !mentionedItems.find((mentioned) => mentioned.id === item.id)), ]; } + const captureSnapshot = (): TimelineState => JSON.parse(JSON.stringify(latestTimelineRef.current)); + const userMessage: Message = { id: Date.now().toString(), content: messageContent, isUser: true, timestamp: new Date(), + snapshot: captureSnapshot(), }; + const nextTabs = tabs.map((t) => + t.id === activeTab.id + ? { + ...t, + // intelligent one-time auto-rename if this is the first message + name: + (t.messages?.length || 0) === 0 + ? messageContent.length > 24 + ? messageContent.slice(0, 24) + "…" + : messageContent + : t.name, + messages: [...(t.messages || []), userMessage], + } + : t, + ); + setTabs(nextTabs); + onMessagesChange( + (nextTabs.find((tt) => tt.id === activeTab.id)?.messages || []).map((m) => ({ + id: m.id, + content: m.content, + isUser: m.isUser, + timestamp: m.timestamp, + })), + ); - onMessagesChange([...messages, userMessage]); - setInputValue(""); - setMentionedItems([]); // Clear mentioned items after sending - setIsTyping(true); - - // Reset textarea height - if (inputRef.current) { - inputRef.current.style.height = "36px"; // Back to normal height - setTextareaHeight(36); - } + // Build assistant context + const chatHistoryPayload = (nextTabs.find((tt) => tt.id === activeTab.id)?.messages || []).map((m) => ({ + role: m.isUser ? "user" : "assistant", + content: m.content, + timestamp: m.timestamp, + })); try { // Use the stored mentioned items to get their IDs const mentionedScrubberIds = itemsToSend.map((item) => item.id); - // Build short chat history to give context to the backend - const history = messages.slice(-10).map((m) => ({ - role: m.isUser ? "user" : "assistant", - content: m.content, - timestamp: m.timestamp, - })); - - // Make API call to the backend const response = await axios.post(apiUrl("/ai", true), { message: messageContent, mentioned_scrubber_ids: mentionedScrubberIds, timeline_state: timelineState, mediabin_items: mediaBinItems, - chat_history: history, + chat_history: chatHistoryPayload, }); const functionCallResponse = response.data; let aiResponseContent = ""; - // Handle the function call based on function_name + // Handle the function call (universal v2: {function_name, arguments}) if (functionCallResponse.function_call) { const { function_call } = functionCallResponse; + const fn = function_call.function_name; + const args = function_call.arguments || {}; + + const toNumber = (val: unknown): number | undefined => { + if (typeof val === "number") return Number.isFinite(val) ? val : undefined; + if (typeof val === "string") { + const n = parseFloat(val); + return Number.isFinite(n) ? n : undefined; + } + return undefined; + }; + + const toSeconds = (val: unknown): number | undefined => { + if (typeof val === "number") return Number.isFinite(val) ? val : undefined; + if (typeof val !== "string") return undefined; + const raw = val.trim().toLowerCase(); + // Try @alwatr/parse-duration (returns ms) + try { + const ms = (parseDuration as unknown as (v: unknown) => number)(raw); + if (typeof ms === "number" && Number.isFinite(ms)) return ms / 1000; + } catch {} + // Try hh:mm:ss / mm:ss + const colon = raw.match(/^\s*(\d{1,2}):(\d{1,2})(?::(\d{1,2}))?\s*$/); + if (colon) { + const h = colon[3] ? parseFloat(colon[1]) : 0; + const m = colon[3] ? parseFloat(colon[2]) : parseFloat(colon[1]); + const s = colon[3] ? parseFloat(colon[2 + 1]) : parseFloat(colon[2]); + const total = (h || 0) * 3600 + (m || 0) * 60 + (s || 0); + return Number.isFinite(total) ? total : undefined; + } + // Fallback numeric seconds + const n = parseFloat(raw); + return Number.isFinite(n) ? n : undefined; + }; try { - if (function_call.function_name === "LLMAddScrubberToTimeline") { + if (fn === "LLMAddScrubberToTimeline") { // Find the media item by ID - const mediaItem = mediaBinItems.find( - (item) => item.id === function_call.scrubber_id - ); + const mediaItem = mediaBinItems.find((item) => item.id === (args.scrubber_id as string)); if (!mediaItem) { - aiResponseContent = `❌ Error: Media item with ID "${function_call.scrubber_id}" not found in the media bin.`; + aiResponseContent = `❌ Error: Media item with ID "${args.scrubber_id}" not found in the media bin.`; } else { // Execute the function llmAddScrubberToTimeline( - function_call.scrubber_id, + args.scrubber_id as string, mediaBinItems, - function_call.track_id, - function_call.drop_left_px, - handleDropOnTrack + args.track_id as string, + args.drop_left_px as number, + handleDropOnTrack, ); - aiResponseContent = `✅ Successfully added "${mediaItem.name}" to ${function_call.track_id} at position ${function_call.drop_left_px}px.`; + aiResponseContent = `✅ Successfully added "${mediaItem.name}" to ${args.track_id} at position ${args.drop_left_px}px.`; } - } else if (function_call.function_name === "LLMMoveScrubber") { + } else if (fn === "LLMMoveScrubber" || fn === "MoveScrubber") { // Execute move scrubber operation + const posSec = + toSeconds(args.new_position_seconds) ?? + toSeconds(args.position_seconds) ?? + toSeconds(args.start_seconds) ?? + 0; + const destTrack = (toNumber(args.new_track_number) ?? toNumber(args.track_number) ?? 1) as number; llmMoveScrubber( - function_call.scrubber_id, - function_call.new_position_seconds, - function_call.new_track_number, - function_call.pixels_per_second, + args.scrubber_id as string, + posSec, + destTrack, + (args.pixels_per_second as number) ?? pixelsPerSecond, timelineState, - handleUpdateScrubber + handleUpdateScrubber, ); // Try to locate the scrubber name for a nicer message - const allScrubbers = timelineState.tracks.flatMap( - (t) => t.scrubbers - ); - const moved = allScrubbers.find( - (s) => s.id === function_call.scrubber_id - ); - const movedName = moved ? moved.name : function_call.scrubber_id; - aiResponseContent = `✅ Moved "${movedName}" to track ${function_call.new_track_number} at ${function_call.new_position_seconds}s.`; - } else if (function_call.function_name === "LLMAddScrubberByName") { + const allScrubbers = timelineState.tracks.flatMap((t) => t.scrubbers); + const moved = allScrubbers.find((s) => s.id === (args.scrubber_id as string)); + const movedName = moved ? moved.name : (args.scrubber_id as string); + aiResponseContent = `✅ Moved "${movedName}" to track ${args.new_track_number} at ${args.new_position_seconds}s.`; + } else if (fn === "LLMAddScrubberByName" || fn === "AddMediaByName") { // Add media by name with defaults - llmAddScrubberByName( - function_call.scrubber_name, + const name = String(args.scrubber_name ?? ""); + const pps = toNumber(args.pixels_per_second) ?? pixelsPerSecond; + const startSeconds = toSeconds(args.start_seconds) ?? toSeconds(args.position_seconds) ?? 0; + const trackNumber = (toNumber(args.track_number) ?? 1) as number; + const startPx = startSeconds * pps; + + const newId = llmAddScrubberByName( + name, mediaBinItems, - function_call.track_number, - function_call.position_seconds, - function_call.pixels_per_second ?? 100, - handleDropOnTrack - ); + trackNumber, + startSeconds, + pps, + handleDropOnTrack, + ) as unknown as string; + + // Optional duration or end time handling (resize after drop) + const endSec = toSeconds(args.end_seconds); + const durationSeconds = + toSeconds(args.duration_seconds) ?? + (endSec !== undefined ? Math.max(0, endSec - startSeconds) : undefined); + if (durationSeconds && durationSeconds > 0) { + setTimeout(() => { + const tl = latestTimelineRef.current; + const trackIndex = Math.max(0, trackNumber - 1); + const track = tl.tracks?.[trackIndex]; + if (!track) return; + const target = newId ? tl.tracks[trackIndex]?.scrubbers.find((s) => s.id === newId) : undefined; + if (target) llmResizeScrubber(target.id, durationSeconds, pps, tl, handleUpdateScrubber); + }, 150); + } + + aiResponseContent = `✅ Added "${args.scrubber_name}" to track ${trackNumber} at ${startSeconds}s.`; + } else if (fn === "AddMediaById") { + const scrubberId = String(args.scrubber_id ?? ""); + const pps = toNumber(args.pixels_per_second) ?? pixelsPerSecond; + const startSeconds = toSeconds(args.start_seconds) ?? 0; + const trackNumber = (toNumber(args.track_number) ?? 1) as number; + const startPx = startSeconds * pps; + + const mediaItem = mediaBinItems.find((i) => i.id === scrubberId); + if (!mediaItem) { + aiResponseContent = `❌ Error: Media item with ID "${scrubberId}" not found in the media bin.`; + } else { + const trackId = `track-${trackNumber}`; + const newId = handleDropOnTrack(mediaItem, trackId, startPx); - aiResponseContent = `✅ Added "${function_call.scrubber_name}" to track ${function_call.track_number} at ${function_call.position_seconds}s.`; - } else if ( - function_call.function_name === "LLMDeleteScrubbersInTrack" - ) { + const endSec2 = toSeconds(args.end_seconds); + const durationSeconds = + toSeconds(args.duration_seconds) ?? + (endSec2 !== undefined ? Math.max(0, endSec2 - startSeconds) : undefined); + if (durationSeconds && durationSeconds > 0) { + setTimeout(() => { + const tl = latestTimelineRef.current; + const trackIndex = Math.max(0, trackNumber - 1); + const track = tl.tracks?.[trackIndex]; + if (!track) return; + const target = newId ? tl.tracks[trackIndex]?.scrubbers.find((s) => s.id === newId) : undefined; + if (target) llmResizeScrubber(target.id, durationSeconds, pps, tl, handleUpdateScrubber); + }, 150); + } + + aiResponseContent = `✅ Added media to track ${trackNumber} at ${startSeconds}s.`; + } + } else if (fn === "LLMDeleteScrubbersInTrack" || fn === "DeleteScrubbersInTrack") { if (!handleDeleteScrubber) { throw new Error("Delete handler is not available"); } - llmDeleteScrubbersInTrack( - function_call.track_number, + llmDeleteScrubbersInTrack((args.track_number as number) ?? 1, timelineState, handleDeleteScrubber); + aiResponseContent = `✅ Removed all scrubbers in track ${(args.track_number as number) ?? 1}.`; + } else if (fn === "LLMResizeScrubber" || fn === "ResizeScrubber") { + const startSecForDiff = toSeconds((args as any).start_seconds) ?? toSeconds((args as any).position_seconds); + const candidateDur = + toSeconds((args as any).new_duration_seconds) ?? + toSeconds((args as any).duration_seconds) ?? + toSeconds((args as any).seconds) ?? + toSeconds((args as any).duration) ?? + toSeconds((args as any).newDurationSeconds) ?? + toSeconds((args as any).durationInSeconds) ?? + // try to parse free-form text provided by model (e.g., "12 seconds long") + (typeof (args as any).new_text_content === "string" + ? toSeconds((args as any).new_text_content) + : undefined); + const endSecVal = toSeconds((args as any).end_seconds); + const dur = + candidateDur ?? + (startSecForDiff !== undefined && endSecVal !== undefined + ? Math.max(0, endSecVal - startSecForDiff) + : undefined); + const ppsVal = toNumber(args.pixels_per_second) ?? pixelsPerSecond; + const trackNum = toNumber((args as any).track_number) ?? toNumber((args as any).new_track_number); + let targetId = typeof args.scrubber_id === "string" ? (args.scrubber_id as string) : undefined; + if (!targetId && trackNum !== undefined) { + const trackIndex = Math.max(0, Math.floor(trackNum) - 1); + const track = timelineState.tracks?.[trackIndex]; + if (track && track.scrubbers.length > 0) { + const nameSub = + typeof (args as any).scrubber_name === "string" + ? String((args as any).scrubber_name).toLowerCase() + : undefined; + if (nameSub) { + const found = track.scrubbers.find((s) => s.name.toLowerCase().includes(nameSub)); + if (found) targetId = found.id; + } + if (!targetId) { + // fallback to rightmost scrubber + targetId = track.scrubbers.reduce( + (best, s) => (s.left > best.left ? s : best), + track.scrubbers[0], + ).id; + } + } + } + if (dur && dur > 0 && targetId) { + llmResizeScrubber(targetId, dur, ppsVal, timelineState, handleUpdateScrubber); + aiResponseContent = `✅ Resized scrubber to ${dur}s.`; + } else if (!targetId) { + aiResponseContent = `❌ Unable to resize: could not identify target scrubber.`; + } else { + aiResponseContent = `❌ Unable to resize: invalid duration.`; + } + } else if (fn === "LLMUpdateTextContent" || fn === "UpdateTextContent") { + llmUpdateTextContent( + args.scrubber_id as string, + args.new_text_content as string, timelineState, - handleDeleteScrubber + handleUpdateScrubber, ); - aiResponseContent = `✅ Removed all scrubbers in track ${function_call.track_number}.`; + aiResponseContent = `✅ Updated text content.`; + } else if (fn === "LLMUpdateTextStyle" || fn === "UpdateTextStyle") { + llmUpdateTextStyle( + args.scrubber_id as string, + { + fontSize: args.fontSize as number | undefined, + fontFamily: args.fontFamily as string | undefined, + color: args.color as string | undefined, + textAlign: args.textAlign as any, + fontWeight: args.fontWeight as any, + }, + timelineState, + handleUpdateScrubber, + ); + aiResponseContent = `✅ Updated text style.`; + } else if (fn === "LLMMoveScrubbersByOffset" || fn === "MoveScrubbersByOffset") { + const ids = (args.scrubber_ids as string[]) || []; + llmMoveScrubbersByOffset( + ids, + args.offset_seconds as number, + (args.pixels_per_second as number) ?? pixelsPerSecond, + timelineState, + handleUpdateScrubber, + ); + aiResponseContent = `✅ Moved ${ids.length} scrubber(s) by ${args.offset_seconds}s.`; + } else if (fn === "CreateTrack") { + if (handleAddTrack) { + handleAddTrack(); + aiResponseContent = "✅ Created 1 new track."; + } else { + aiResponseContent = "❌ Cannot create track: handler unavailable."; + } + } else if (fn === "CreateTracks") { + const count = toNumber((args as any).count) ?? 1; + if (handleAddTrack) { + const n = Math.max(1, Math.floor(count)); + for (let i = 0; i < n; i++) handleAddTrack(); + aiResponseContent = `✅ Created ${n} track(s).`; + } else { + aiResponseContent = "❌ Cannot create tracks: handler unavailable."; + } + } else if (fn === "PlaceAllAssetsParallel") { + // Place each media bin item on a separate (new if needed) track at the same start time + const startSec = toSeconds((args as any).start_seconds) ?? 0; + const pps = toNumber((args as any).pixels_per_second) ?? pixelsPerSecond; + const startPx = startSec * pps; + const requiredTracks = mediaBinItems.length; + // Ensure enough tracks + const shortage = Math.max(0, requiredTracks - timelineState.tracks.length); + if (shortage > 0 && handleAddTrack) { + for (let i = 0; i < shortage; i++) handleAddTrack(); + } + mediaBinItems.forEach((item, index) => { + const trackId = timelineState.tracks[index]?.id || `track-${index + 1}`; + handleDropOnTrack(item, trackId, startPx); + }); + aiResponseContent = `✅ Placed ${mediaBinItems.length} asset(s) in parallel across tracks at ${startSec}s.`; + } else if (fn === "LLMSetResolution" || fn === "SetResolution") { + // This requires handlers from parent; ChatBox doesn't own them, so we ignore here or bubble up later. + // Leaving placeholder for future wiring if exposed via props. + aiResponseContent = `ℹ️ Resolution change acknowledged.`; } else { - aiResponseContent = `❌ Unknown function: ${function_call.function_name}`; + aiResponseContent = `❌ Unknown function: ${fn}`; } } catch (error) { aiResponseContent = `❌ Error executing function: ${ @@ -341,8 +719,14 @@ export function ChatBox({ content: aiResponseContent, isUser: false, timestamp: new Date(), + snapshot: captureSnapshot(), }; - + const updated = tabs.map((t) => + t.id === activeTab.id + ? { ...t, messages: [...t.messages, userMessage, aiMessage], timelineSnapshot: latestTimelineRef.current } + : t, + ); + setTabs(updated); onMessagesChange([...messages, userMessage, aiMessage]); } catch (error) { console.error("Error calling AI API:", error); @@ -352,8 +736,15 @@ export function ChatBox({ content: `❌ Sorry, I encountered an error while processing your request. Please try again.`, isUser: false, timestamp: new Date(), + snapshot: captureSnapshot(), }; + const updated = tabs.map((t) => + t.id === activeTab.id + ? { ...t, messages: [...t.messages, userMessage, errorMessage], timelineSnapshot: latestTimelineRef.current } + : t, + ); + setTabs(updated); onMessagesChange([...messages, userMessage, errorMessage]); } finally { setIsTyping(false); @@ -364,16 +755,12 @@ export function ChatBox({ if (showMentions && filteredMentions.length > 0) { if (e.key === "ArrowDown") { e.preventDefault(); - setSelectedMentionIndex((prev) => - prev < filteredMentions.length - 1 ? prev + 1 : 0 - ); + setSelectedMentionIndex((prev) => (prev < filteredMentions.length - 1 ? prev + 1 : 0)); return; } if (e.key === "ArrowUp") { e.preventDefault(); - setSelectedMentionIndex((prev) => - prev > 0 ? prev - 1 : filteredMentions.length - 1 - ); + setSelectedMentionIndex((prev) => (prev > 0 ? prev - 1 : filteredMentions.length - 1)); return; } if (e.key === "Enter") { @@ -400,47 +787,292 @@ export function ChatBox({ } }; - const formatTime = (date: Date) => { - return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); + // helpers to update current tab messages consistently + const setActiveTabMessages = (newMessages: Message[]) => { + const updatedTabs = tabs.map((t) => + t.id === activeTab.id ? { ...t, messages: newMessages, timelineSnapshot: latestTimelineRef.current } : t, + ); + setTabs(updatedTabs); + onMessagesChange(newMessages); + }; + + const truncateAtIndexPreserveReply = (index: number) => { + const base = activeTab?.messages || messages; + if (index < 0 || index >= base.length) return; + const keepUntil = base[index + 1] && !base[index + 1].isUser ? index + 1 : index; + setActiveTabMessages(base.slice(0, keepUntil + 1)); + }; + + const restoreAtIndex = (index: number) => { + const base = activeTab?.messages || messages; + if (index < 0 || index >= base.length) return; + const msg = base[index]; + const snap = msg?.snapshot || null; + if (snap && restoreTimeline) restoreTimeline(snap); + truncateAtIndexPreserveReply(index); + }; + + const startInlineEditAt = (index: number) => { + const base = activeTab?.messages || messages; + const msg = base[index]; + if (!msg) return; + // auto-restore to saved snapshot and truncate + restoreSnapshot?.(); + truncateAtIndexPreserveReply(index); + setInputValue(msg.content); + setTimeout(() => inputRef.current?.focus(), 0); + }; + + const formatTime = (dateLike: unknown) => { + try { + const d = dateLike instanceof Date ? dateLike : new Date(dateLike as any); + if (!(d instanceof Date) || isNaN(d.getTime())) return ""; + return d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); + } catch { + return ""; + } + }; + + // Tab actions + const createTab = () => { + const t: ChatTab = { + id: Date.now().toString(), + name: `Chat ${tabs.length + 1}`, + messages: [], + timelineSnapshot: null, + createdAt: Date.now(), + }; + const next = [...tabs, t]; + setTabs(next); + setActiveTabId(t.id); + }; + const renameTab = (id: string) => { + const name = prompt("Rename chat", tabs.find((x) => x.id === id)?.name || "Chat"); + if (!name) return; + setTabs(tabs.map((t) => (t.id === id ? { ...t, name } : t))); + }; + const deleteTab = (id: string) => { + const next = tabs.filter((t) => t.id !== id); + setTabs( + next.length + ? next + : [{ id: Date.now().toString(), name: "Chat 1", messages: [], timelineSnapshot: null, createdAt: Date.now() }], + ); + if (activeTabId === id) setActiveTabId((next[0] || { id: "" }).id); + }; + const saveSnapshot = () => { + setTabs(tabs.map((t) => (t.id === activeTab.id ? { ...t, timelineSnapshot: latestTimelineRef.current } : t))); + }; + const restoreSnapshot = () => { + const snap = activeTab.timelineSnapshot; + if (!snap || !restoreTimeline) return; + restoreTimeline(snap); + }; + + // Send to new chat helper + const [sendToTabId, setSendToTabId] = useState(null); + const sendMessageToNewChat = (includeAllMedia = false) => { + const newTab: ChatTab = { + id: Date.now().toString(), + name: `Chat ${tabs.length + 1}`, + messages: [], + timelineSnapshot: null, + createdAt: Date.now(), + }; + const next = [...tabs, newTab]; + setTabs(next); + setActiveTabId(newTab.id); + setSendToTabId(newTab.id); + // Slight delay to allow state to settle before sending + setTimeout(() => handleSendMessage(includeAllMedia), 0); }; return ( -
+
{/* Chat Header */} -
-
- - Ask Kimu -
- -
- - {onToggleMinimize && ( +
+ {/* Row 1: brand + actions */} +
+
+ + Ask Kimu +
+
+ - )} + {onToggleMinimize && ( + + )} +
+
+ {/* Row 2: tabs strip (single-line, horizontally scrollable) */} +
+
+ {tabs.map((t) => ( + + ))} +
+ {/* History panel centered (no blur overlay) */} + {isHistoryOpen && ( + <> + {/* slight dark/blur overlay only over the chat panel */} +
setIsHistoryOpen(false)} + /> +
+
+ setHistoryQuery(e.target.value)} + /> +
+
+ {(() => { + const filtered = tabs + .map((t) => ({ + ...t, + lastActivity: (t.messages?.[t.messages.length - 1]?.timestamp as any)?.getTime?.() || t.createdAt, + })) + .filter((t) => t.name.toLowerCase().includes(historyQuery.toLowerCase())) + .sort((a, b) => b.lastActivity - a.lastActivity); + + const groups: Record = {} as any; + filtered.forEach((t) => { + const g = getRecencyGroup(t.lastActivity); + if (!groups[g]) groups[g] = [] as any; + groups[g].push(t); + }); + + const order = ["Last hour", "Today", "Yesterday", "This week", "Older"]; + return order + .filter((g) => groups[g] && groups[g].length) + .map((g) => ( +
+
{g}
+ {groups[g].map((t) => ( +
{ + setActiveTabId(t.id); + setIsHistoryOpen(false); + scrollToTabId(t.id); + }}> + + {historyEditingId === t.id ? ( + setHistoryEditingName(e.target.value)} + onClick={(e) => e.stopPropagation()} + onBlur={() => { + const name = historyEditingName.trim(); + if (name) setTabs(tabs.map((x) => (x.id === t.id ? { ...x, name } : x))); + setHistoryEditingId(null); + }} + onKeyDown={(e) => { + if (e.key === "Enter") (e.currentTarget as HTMLInputElement).blur(); + if (e.key === "Escape") setHistoryEditingId(null); + }} + /> + ) : ( + t.name + )} + +
+ + +
+
+ ))} +
+ )); + })()} +
+
+ + )} + {/* Content Area */}
{messages.length === 0 ? ( @@ -451,8 +1083,8 @@ export function ChatBox({

Ask Kimu

- Kimu is your AI assistant for video editing. Ask questions, get - help with timeline operations, or request specific edits. + Kimu is your AI assistant for video editing. Ask questions, get help with timeline operations, or request + specific edits.

@@ -460,19 +1092,13 @@ export function ChatBox({ to chat with media
- - Enter - + Enter to send
- - Shift - + Shift + - - Enter - + Enter for new line
@@ -482,47 +1108,52 @@ export function ChatBox({
+ style={{ maxHeight: "calc(100vh - 200px)" }}>
- {messages.map((message) => ( + {(activeTab?.messages || messages).map((message, idx) => (
-
-
- {!message.isUser && ( - - )} -
-

- {message.content} -

- - {formatTime(message.timestamp)} - + className={`flex ${message.isUser ? "justify-end" : "justify-start"}`} + onContextMenu={(e) => { + e.preventDefault(); + setContextMenu({ open: true, x: e.clientX, y: e.clientY, index: idx, message }); + }}> + {message.isUser ? ( +
+
+
+

{message.content}

+
+ {formatTime(message.timestamp)} + +
+
- {message.isUser && ( - - )}
-
+ ) : ( +
+
+
+

{message.content}

+
+ {formatTime(message.timestamp)} +
+
+
+
+ )}
))} {/* Typing Indicator */} {isTyping && (
-
+
@@ -546,6 +1177,31 @@ export function ChatBox({ {/* Invisible element to scroll to */}
+ {/* Simple custom context menu */} + {contextMenu.open && ( +
setContextMenu({ ...contextMenu, open: false })}> +
{ + setContextMenu({ ...contextMenu, open: false }); + startInlineEditAt(contextMenu.index); + }}> + Edit here (inline) +
+
{ + setContextMenu({ ...contextMenu, open: false }); + setConfirmRestoreIndex(contextMenu.index); + setShowConfirmRestore(true); + }}> + Restore to this point +
+
+ )}
)} @@ -557,18 +1213,14 @@ export function ChatBox({ {showMentions && filteredMentions.length > 0 && (
+ className="absolute bottom-full left-4 right-4 mb-2 bg-background border border-border/50 rounded-lg shadow-lg max-h-40 overflow-y-auto z-50"> {filteredMentions.map((item, index) => (
insertMention(item)} - > + onClick={() => insertMention(item)}>
{item.mediaType === "video" ? ( @@ -579,9 +1231,7 @@ export function ChatBox({ )}
{item.name} - - {item.mediaType} - + {item.mediaType}
))}
@@ -591,8 +1241,7 @@ export function ChatBox({ {showSendOptions && (
+ className="absolute bottom-full right-4 mb-2 bg-background border border-border/50 rounded-md shadow-lg z-50 min-w-48">
+ }}> Send - - Enter - + Enter
+ }}> Send with all Media
{ // Clear current messages and send to new chat - onMessagesChange([]); setShowSendOptions(false); - handleSendMessage(false); - }} - > + sendMessageToNewChat(false); + }}> Send to New Chat
@@ -643,7 +1286,7 @@ export function ChatBox({ placeholder="Ask Kimu..." className={cn( "w-full min-h-8 max-h-20 resize-none text-xs bg-transparent border-0 px-3 pt-2.5 pb-1 placeholder:text-muted-foreground/60 focus:outline-none disabled:cursor-not-allowed disabled:opacity-50", - "transition-all duration-200 leading-relaxed" + "transition-all duration-200 leading-relaxed", )} disabled={isTyping} rows={1} @@ -659,12 +1302,8 @@ export function ChatBox({ className="h-6 w-6 p-0 text-muted-foreground/70 hover:text-foreground hover:bg-muted/50" onClick={() => { if (inputRef.current) { - const cursorPos = - inputRef.current.selectionStart || inputValue.length; - const newValue = - inputValue.slice(0, cursorPos) + - "@" + - inputValue.slice(cursorPos); + const cursorPos = inputRef.current.selectionStart || inputValue.length; + const newValue = inputValue.slice(0, cursorPos) + "@" + inputValue.slice(cursorPos); setInputValue(newValue); const newCursorPos = cursorPos + 1; setCursorPosition(newCursorPos); @@ -676,14 +1315,10 @@ export function ChatBox({ setTimeout(() => { inputRef.current?.focus(); - inputRef.current?.setSelectionRange( - newCursorPos, - newCursorPos - ); + inputRef.current?.setSelectionRange(newCursorPos, newCursorPos); }, 0); } - }} - > + }}> @@ -694,8 +1329,7 @@ export function ChatBox({ disabled={!inputValue.trim() || isTyping} size="sm" className="h-6 px-2 bg-transparent hover:bg-primary/10 text-primary hover:text-primary text-xs" - variant="ghost" - > + variant="ghost">
+ {/* Modals for restore/delete/edit */} + + + + Restore to this point? + + The timeline will be restored to the snapshot saved for this chat. Messages after this point can be + deleted optionally. + + + + Cancel + { + setShowConfirmRestore(false); + if (confirmRestoreIndex !== null) restoreAtIndex(confirmRestoreIndex); + }}> + Restore + + + + + + {/* Tabs context menu: Rename / Clear / Delete */} + {tabsMenu.open && ( +
setTabsMenu({ ...tabsMenu, open: false })}> +
{ + setTabsMenu({ ...tabsMenu, open: false }); + if (!tabsMenu.tabId) return; + const t = tabs.find((x) => x.id === tabsMenu.tabId); + if (!t) return; + setEditingTabId(t.id); + setEditingTabName(t.name); + }}> + Rename chat +
+
{ + setTabsMenu({ ...tabsMenu, open: false }); + if (!tabsMenu.tabId) return; + if (tabsMenu.tabId === activeTab.id) { + setActiveTabMessages([]); + } else { + setTabs(tabs.map((t) => (t.id === tabsMenu.tabId ? { ...t, messages: [] } : t))); + } + }}> + Clear chat +
+
{ + setTabsMenu({ ...tabsMenu, open: false }); + if (!tabsMenu.tabId) return; + deleteTab(tabsMenu.tabId); + }}> + Delete chat +
+
+ )}
); } diff --git a/app/components/editor/LeftPanel.tsx b/app/components/editor/LeftPanel.tsx index 5e9002c..338d5ab 100644 --- a/app/components/editor/LeftPanel.tsx +++ b/app/components/editor/LeftPanel.tsx @@ -14,7 +14,7 @@ interface LeftPanelProps { fontFamily: string, color: string, textAlign: "left" | "center" | "right", - fontWeight: "normal" | "bold" + fontWeight: "normal" | "bold", ) => void; contextMenu: { x: number; @@ -59,8 +59,7 @@ export default function LeftPanel({ + className="h-8 text-xs data-[state=active]:bg-background data-[state=active]:shadow-sm"> @@ -68,8 +67,7 @@ export default function LeftPanel({ + className="h-8 text-xs data-[state=active]:bg-background data-[state=active]:shadow-sm"> @@ -77,8 +75,7 @@ export default function LeftPanel({ + className="h-8 text-xs data-[state=active]:bg-background data-[state=active]:shadow-sm"> diff --git a/app/components/timeline/MediaBin.tsx b/app/components/timeline/MediaBin.tsx index 454fe96..069f7f6 100644 --- a/app/components/timeline/MediaBin.tsx +++ b/app/components/timeline/MediaBin.tsx @@ -44,7 +44,7 @@ interface MediaBinProps { fontFamily: string, color: string, textAlign: "left" | "center" | "right", - fontWeight: "normal" | "bold" + fontWeight: "normal" | "bold", ) => void; contextMenu: { x: number; @@ -58,35 +58,25 @@ interface MediaBinProps { } // Memoized component for video thumbnails to prevent flickering -const VideoThumbnail = memo( - ({ - mediaUrl, - width, - height, - }: { - mediaUrl: string; - width: number; - height: number; - }) => { - const VideoComponent = useMemo(() => { - return () =>