From 3f874e5b0cd6c73566b43fd8713435cc6366454a Mon Sep 17 00:00:00 2001 From: TheMaddax Date: Sun, 26 Jan 2025 09:07:22 -0600 Subject: [PATCH] Improved prompts and appearance --- src-tauri/src/lib.rs | 161 +++++++++++++++++++++++++------ src/App.tsx | 27 ++++-- src/components/Settings.tsx | 34 ++++++- src/components/VideoTimeline.tsx | 82 ++++++++++++++-- 4 files changed, 256 insertions(+), 48 deletions(-) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 4f56d38..db4245e 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -40,11 +40,18 @@ pub struct MergeArgs { pub resize_to_720p: bool, } +#[derive(Debug, Deserialize)] +pub struct SelectedFrame { + pub path: String, + pub time: u32, + pub base64_image: String, +} + #[derive(Debug, Deserialize)] pub struct TranscriptionArgs { pub video_path: String, pub subtitle_path: String, - pub frame_time: u32, + pub selected_frames: Vec, pub api_key: String, } @@ -236,45 +243,145 @@ pub async fn process_transcription_impl(args: TranscriptionArgs) -> Result = line.split(',').collect(); + if parts.len() >= 10 { + let start_time = parts[1].trim(); + // Combine all remaining parts as the text, skipping style and positioning info + let text = parts[9..].join(",").trim().to_string(); + if !text.is_empty() { + subtitle_lines.push((start_time.to_string(), text)); + } + } + } + } + } else { + // SRT format parsing + let mut i = 0; + while i < lines.len() { + if i + 3 <= lines.len() && lines[i + 1].contains("-->") { + let timestamp_parts: Vec<&str> = lines[i + 1].split(" --> ").collect(); + if timestamp_parts.len() == 2 { + let start_time = timestamp_parts[0].trim(); + let mut text = lines[i + 2].trim().to_string(); + + // Collect multi-line subtitle text + let mut j = i + 3; + while j < lines.len() && !lines[j].trim().is_empty() && !lines[j].contains("-->") { + text.push_str(" "); + text.push_str(lines[j].trim()); + j += 1; + } + + subtitle_lines.push((start_time.to_string(), text)); + i = j; + } + } + i += 1; + } + } -Format the description in clear, concise paragraphs that would be helpful for DeafBlind readers to understand the visual context. Focus on spatial relationships and important visual details that contribute to understanding the scene. Use maximum of 200 words and do not include your own thoughts-- just provide the visual description alone."; + // Sort subtitles by timestamp + subtitle_lines.sort_by(|a, b| a.0.cmp(&b.0)); - let subtitle_prompt = format!("Convert the following subtitle content into a natural, flowing narrative that includes: -1. Speaker identification when there are multiple speakers and if there is a single speaker, no need to provide speaker identification. -2. Speaking manner and tone where relevant (e.g., \"warmly\", \"enthusiastically\") -3. Clear paragraph breaks between different speakers or topics -4. Natural transition words to connect dialogue -5. Integration of any important non-verbal cues from the original subtitles + // Format scene descriptions with timestamps in MM:SS format + let scene_descriptions = frame_descriptions.iter() + .map(|(time, desc)| { + let minutes = time / 60; + let seconds = time % 60; + format!("[{:02}:{:02}] {}", minutes, seconds, desc) + }) + .collect::>() + .join("\n\n"); -Subtitle content: -{} + // Format subtitle content with timestamps + let formatted_subtitles = subtitle_lines.iter() + .map(|(time, text)| format!("[{}] {}", time, text)) + .collect::>() + .join("\n"); -Format the text as a professional transcript, maintaining chronological flow while making it read naturally. I want you to just provide the visual description alone.", subtitle_content); + let subtitle_prompt = format!(r#"I will provide you with timestamped scene descriptions and subtitle content. Create a chronological narrative that combines these elements with these strict rules: - let (visual_description, subtitle_narrative) = tokio::join!( - query_gemini(&args.api_key, visual_prompt, Some(frame_path.to_str().unwrap())), - query_gemini(&args.api_key, &subtitle_prompt, None) - ); + Scene Descriptions: + {} - // Get the results, handling any errors - let visual_description = visual_description?; - let subtitle_narrative = subtitle_narrative?; + Subtitle Content with Timestamps: + {} + + Required Format Rules: + 1. Use ONLY the exact dialogue lines from the subtitle content - do not add, modify, or create new dialogue but feel free to turn them into paragraphs provided that content of exact words are not modified. + 2. For the first scene description, MOVE it to the beginning of the narration. For the rest of the scenes, ensure that they are located according to their exact timestamps when they occur in the dialogue flow. + 3. Keep all dialogue in strict chronological order and to be quoted verbatim + 4. For multiple speakers, add speaker identification before their lines until the next speaker's dialogue lines. If there is only one speaker, do not add a speaker identifier. + 5. Include speaking manner (e.g., "warmly", "firmly") only when clearly implied by the subtitle text + 6. Use paragraph breaks to separate different scenes if there is more than one scene + 7. Ensure that there are no timestamps in the output including for the scenes + + Critical Requirements: + - Do not invent or modify any dialogue + - Only use the provided subtitle text verbatim + - Scene descriptions should be inserted exactly where their timestamps occur except for the first scene which is to start at the beginning of the narration + - Maintain precise chronological order of all elements + - Keep your thoughts and ideas to yourself + + Format the output as a flowing narrative that preserves all original dialogue while weaving in the visual context at the appropriate moments as like it's a novel."#, + scene_descriptions, + formatted_subtitles + ); + + // Process the subtitle content with scene descriptions + let subtitle_narrative = query_gemini(&args.api_key, &subtitle_prompt, None).await?; + + // Combine all visual descriptions + let visual_description = frame_descriptions.into_iter() + .map(|(time, desc)| format!("At timestamp {}:\n{}", time, desc)) + .collect::>() + .join("\n\n"); println!("\n=== Final Transcription Result ==="); println!("\nVisual Description:"); diff --git a/src/App.tsx b/src/App.tsx index 236ee92..36fd32a 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -32,9 +32,16 @@ function App() { const [resizeTo720p, setResizeTo720p] = useState(true); const [geminiEnabled, setGeminiEnabled] = useState(false); const [apiKey, setApiKey] = useState('AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc'); - const [selectedTime, setSelectedTime] = useState(10); + const [selectedFrames, setSelectedFrames] = useState([]); const [transcription, setTranscription] = useState(null); const [thumbnails, setThumbnails] = useState([]); + + interface SelectedFrame { + path: string; + time: number; + base64Image: string; + description?: string; + } const [isSettingsOpen, setIsSettingsOpen] = useState(false); useEffect(() => { @@ -97,7 +104,7 @@ function App() { addBlackBar, resizeTo720p, geminiEnabled, - frameTime: selectedTime + selectedFrames: selectedFrames }); // Process transcription first if Gemini is enabled @@ -108,7 +115,11 @@ function App() { args: { video_path: videoFile.path, subtitle_path: subtitleFile.path, - frame_time: selectedTime, + selected_frames: selectedFrames.map(frame => ({ + path: frame.path, + time: frame.time, + base64_image: frame.base64Image + })), api_key: apiKey } }); @@ -226,11 +237,11 @@ function App() { /> {geminiEnabled && videoFile && ( - + )}
diff --git a/src/components/Settings.tsx b/src/components/Settings.tsx index d754d95..4f2d123 100644 --- a/src/components/Settings.tsx +++ b/src/components/Settings.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import { invoke } from '@tauri-apps/api/core'; interface SettingsProps { @@ -13,9 +13,31 @@ const DEFAULT_API_KEY = 'AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc'; export const Settings: React.FC = ({ isOpen, onClose, apiKey, onApiKeyChange }) => { const [testing, setTesting] = useState(false); const [testResult, setTestResult] = useState(null); + const [isEditing, setIsEditing] = useState(false); + const [displayValue, setDisplayValue] = useState('*'.repeat(apiKey.length)); + + useEffect(() => { + if (!isEditing) { + setDisplayValue('*'.repeat(apiKey.length)); + } + }, [apiKey, isEditing]); const handleApiKeyChange = (e: React.ChangeEvent) => { - onApiKeyChange(e.target.value); + const newValue = e.target.value; + onApiKeyChange(newValue); + if (!isEditing) { + setDisplayValue('*'.repeat(newValue.length)); + } + }; + + const handleFocus = () => { + setIsEditing(true); + setDisplayValue(apiKey); + }; + + const handleBlur = () => { + setIsEditing(false); + setDisplayValue('*'.repeat(apiKey.length)); }; const resetToDefault = () => { @@ -60,10 +82,14 @@ export const Settings: React.FC = ({ isOpen, onClose, apiKey, onA
diff --git a/src/components/VideoTimeline.tsx b/src/components/VideoTimeline.tsx index 1f60fe8..5ff1a66 100644 --- a/src/components/VideoTimeline.tsx +++ b/src/components/VideoTimeline.tsx @@ -3,7 +3,7 @@ import { invoke } from '@tauri-apps/api/core'; interface VideoTimelineProps { videoPath: string | null; - onTimeSelect: (time: number) => void; + onFramesSelect: (frames: SelectedFrame[]) => void; thumbnails: Thumbnail[]; } @@ -12,8 +12,15 @@ interface Thumbnail { time: number; } -export const VideoTimeline: React.FC = ({ videoPath, onTimeSelect, thumbnails }) => { - const [selectedTime, setSelectedTime] = useState(10); // Default 10 seconds +interface SelectedFrame { + path: string; + time: number; + base64Image: string; + description?: string; +} + +export const VideoTimeline: React.FC = ({ videoPath, onFramesSelect, thumbnails }) => { + const [selectedFrames, setSelectedFrames] = useState([]); const [duration, setDuration] = useState(0); const [thumbnailUrls, setThumbnailUrls] = useState<{ [key: string]: string }>({}); @@ -58,23 +65,60 @@ export const VideoTimeline: React.FC = ({ videoPath, onTimeS
- Selected: {formatTime(selectedTime)} + {selectedFrames.length} frames selected
+
+
+ Click thumbnails to select frames for scene descriptions. Selected frames will be analyzed in chronological order. +
+
+
{thumbnails.map((thumb, index) => (
Math.abs(f.time - thumb.time) < 2) + ? 'ring-2 ring-blue-500' + : 'hover:ring-2 hover:ring-blue-300' }`} - onClick={() => { - setSelectedTime(thumb.time); - onTimeSelect(thumb.time); + onClick={async () => { + const isSelected = selectedFrames.some( + f => Math.abs(f.time - thumb.time) < 2 + ); + + if (isSelected) { + // Remove from selection + const newFrames = selectedFrames.filter( + f => Math.abs(f.time - thumb.time) >= 2 + ); + setSelectedFrames(newFrames); + onFramesSelect(newFrames); + } else { + // Add to selection + try { + const base64 = await invoke('read_thumbnail', { + path: thumb.path + }); + const newFrame: SelectedFrame = { + path: thumb.path, + time: thumb.time, + base64Image: base64 + }; + const newFrames = [...selectedFrames, newFrame].sort( + (a, b) => a.time - b.time + ); + setSelectedFrames(newFrames); + onFramesSelect(newFrames); + } catch (error) { + console.error('Error loading thumbnail:', error); + } + } }} > {thumbnailUrls[thumb.path] && ( @@ -87,9 +131,29 @@ export const VideoTimeline: React.FC = ({ videoPath, onTimeS
{formatTime(thumb.time)}
+ {selectedFrames.some(f => Math.abs(f.time - thumb.time) < 2) && ( +
+ {selectedFrames.findIndex(f => Math.abs(f.time - thumb.time) < 2) + 1} +
+ )}
))}
+ + {selectedFrames.length > 0 && ( +
+
+ Selected Frames Order: +
+
+ {selectedFrames.map((frame, index) => ( +
+ {index + 1}. {formatTime(frame.time)} +
+ ))} +
+
+ )}
); };