Improved prompts and appearance

src-tauri/src/lib.rs

@@ -40,11 +40,18 @@ pub struct MergeArgs {
     pub resize_to_720p: bool,
 }
 
+#[derive(Debug, Deserialize)]
+pub struct SelectedFrame {
+    pub path: String,
+    pub time: u32,
+    pub base64_image: String,
+}
+
 #[derive(Debug, Deserialize)]
 pub struct TranscriptionArgs {
     pub video_path: String,
     pub subtitle_path: String,
-    pub frame_time: u32,
+    pub selected_frames: Vec<SelectedFrame>,
     pub api_key: String,
 }
 
@@ -236,45 +243,145 @@ pub async fn process_transcription_impl(args: TranscriptionArgs) -> Result<Trans
     println!("\n=== Starting Transcription Process ===");
     println!("Video Path: {}", args.video_path);
     println!("Subtitle Path: {}", args.subtitle_path);
-    println!("Frame Time: {}", args.frame_time);
+    println!("Number of selected frames: {}", args.selected_frames.len());
 
     let temp_dir = TempDir::new().map_err(|e| Error::IoError(e))?;
-    let frame_path = temp_dir.path().join("frame.jpg");
-    println!("\nExtracting frame to: {:?}", frame_path);
-    extract_frame(&args.video_path, args.frame_time, frame_path.to_str().unwrap()).await?;
+    
+    // Process each frame and collect descriptions
+    let mut frame_descriptions = Vec::new();
+    for (index, frame) in args.selected_frames.iter().enumerate() {
+        println!("\nProcessing frame {} at time {}", index + 1, frame.time);
+        
+        // Save base64 image to temp file
+        let frame_path = temp_dir.path().join(format!("frame_{}.jpg", frame.time));
+        let image_data = base64::decode(&frame.base64_image)
+            .map_err(|e| Error::ImageError(format!("Failed to decode base64 image: {}", e)))?;
+        fs::write(&frame_path, image_data)
+            .map_err(|e| Error::IoError(e))?;
 
-    println!("\nReading subtitle content...");
+        let visual_prompt = format!("Analyze this video frame from timestamp {} and provide a visual description focusing on:
+1. The visual setup and environment
+2. The people present, their appearance, and positioning
+3. Any relevant visual context or background details
+4. Do not include details about the individuals' hands
+
+Format the description in clear, concise paragraphs that would be helpful for DeafBlind readers to understand the visual context. Focus on spatial relationships and important visual details that contribute to understanding the scene. Use maximum of 200 words and do not include your own thoughts-- just provide the visual description alone.", frame.time);
+
+        let description = query_gemini(&args.api_key, &visual_prompt, Some(frame_path.to_str().unwrap())).await?;
+        frame_descriptions.push((frame.time, description));
+    }
+
+    println!("\nReading and parsing subtitle content...");
     let subtitle_content = fs::read_to_string(&args.subtitle_path)
         .map_err(|e| Error::IoError(e))?;
     println!("Subtitle content length: {} characters", subtitle_content.len());
 
-    let visual_prompt = "Analyze this video frame and provide a visual description focusing on:
-1. The visual setup and environment
-2. The people present, their appearance, and positioning
-3. Any relevant visual context or background details
+    // Parse subtitle timestamps and content, handling both SRT and ASS formats
+    let mut subtitle_lines = Vec::new();
+    let lines: Vec<&str> = subtitle_content.lines().collect();
+    
+    // Check if it's an ASS file
+    if args.subtitle_path.to_lowercase().ends_with(".ass") {
+        let mut in_events = false;
+        for line in lines {
+            if line.starts_with("[Events]") {
+                in_events = true;
+                continue;
+            }
+            if in_events && line.starts_with("Dialogue:") {
+                let parts: Vec<&str> = line.split(',').collect();
+                if parts.len() >= 10 {
+                    let start_time = parts[1].trim();
+                    // Combine all remaining parts as the text, skipping style and positioning info
+                    let text = parts[9..].join(",").trim().to_string();
+                    if !text.is_empty() {
+                        subtitle_lines.push((start_time.to_string(), text));
+                    }
+                }
+            }
+        }
+    } else {
+        // SRT format parsing
+        let mut i = 0;
+        while i < lines.len() {
+            if i + 3 <= lines.len() && lines[i + 1].contains("-->") {
+                let timestamp_parts: Vec<&str> = lines[i + 1].split(" --> ").collect();
+                if timestamp_parts.len() == 2 {
+                    let start_time = timestamp_parts[0].trim();
+                    let mut text = lines[i + 2].trim().to_string();
+                    
+                    // Collect multi-line subtitle text
+                    let mut j = i + 3;
+                    while j < lines.len() && !lines[j].trim().is_empty() && !lines[j].contains("-->") {
+                        text.push_str(" ");
+                        text.push_str(lines[j].trim());
+                        j += 1;
+                    }
+                    
+                    subtitle_lines.push((start_time.to_string(), text));
+                    i = j;
+                }
+            }
+            i += 1;
+        }
+    }
 
-Format the description in clear, concise paragraphs that would be helpful for DeafBlind readers to understand the visual context. Focus on spatial relationships and important visual details that contribute to understanding the scene. Use maximum of 200 words and do not include your own thoughts-- just provide the visual description alone.";
+    // Sort subtitles by timestamp
+    subtitle_lines.sort_by(|a, b| a.0.cmp(&b.0));
 
-    let subtitle_prompt = format!("Convert the following subtitle content into a natural, flowing narrative that includes:
-1. Speaker identification when there are multiple speakers and if there is a single speaker, no need to provide speaker identification.
-2. Speaking manner and tone where relevant (e.g., \"warmly\", \"enthusiastically\")
-3. Clear paragraph breaks between different speakers or topics
-4. Natural transition words to connect dialogue
-5. Integration of any important non-verbal cues from the original subtitles
+    // Format scene descriptions with timestamps in MM:SS format
+    let scene_descriptions = frame_descriptions.iter()
+        .map(|(time, desc)| {
+            let minutes = time / 60;
+            let seconds = time % 60;
+            format!("[{:02}:{:02}] {}", minutes, seconds, desc)
+        })
+        .collect::<Vec<_>>()
+        .join("\n\n");
 
-Subtitle content:
-{}
+    // Format subtitle content with timestamps
+    let formatted_subtitles = subtitle_lines.iter()
+        .map(|(time, text)| format!("[{}] {}", time, text))
+        .collect::<Vec<_>>()
+        .join("\n");
 
-Format the text as a professional transcript, maintaining chronological flow while making it read naturally.  I want you to just provide the visual description alone.", subtitle_content);
+    let subtitle_prompt = format!(r#"I will provide you with timestamped scene descriptions and subtitle content. Create a chronological narrative that combines these elements with these strict rules:
 
-    let (visual_description, subtitle_narrative) = tokio::join!(
-        query_gemini(&args.api_key, visual_prompt, Some(frame_path.to_str().unwrap())),
-        query_gemini(&args.api_key, &subtitle_prompt, None)
-    );
+        Scene Descriptions:
+        {}
 
-    // Get the results, handling any errors
-    let visual_description = visual_description?;
-    let subtitle_narrative = subtitle_narrative?;
+        Subtitle Content with Timestamps:
+        {}
+
+        Required Format Rules:
+        1. Use ONLY the exact dialogue lines from the subtitle content - do not add, modify, or create new dialogue but feel free to turn them into paragraphs provided that content of exact words are not modified.
+        2. For the first scene description, MOVE it to the beginning of the narration. For the rest of the scenes, ensure that they are located according to their exact timestamps when they occur in the dialogue flow.
+        3. Keep all dialogue in strict chronological order and to be quoted verbatim
+        4. For multiple speakers, add speaker identification before their lines until the next speaker's dialogue lines. If there is only one speaker, do not add a speaker identifier.
+        5. Include speaking manner (e.g., "warmly", "firmly") only when clearly implied by the subtitle text
+        6. Use paragraph breaks to separate different scenes if there is more than one scene
+        7. Ensure that there are no timestamps in the output including for the scenes
+        
+        Critical Requirements:
+        - Do not invent or modify any dialogue
+        - Only use the provided subtitle text verbatim
+        - Scene descriptions should be inserted exactly where their timestamps occur except for the first scene which is to start at the beginning of the narration
+        - Maintain precise chronological order of all elements
+        - Keep your thoughts and ideas to yourself
+        
+        Format the output as a flowing narrative that preserves all original dialogue while weaving in the visual context at the appropriate moments as like it's a novel."#, 
+            scene_descriptions, 
+            formatted_subtitles
+        );
+        
+    // Process the subtitle content with scene descriptions
+    let subtitle_narrative = query_gemini(&args.api_key, &subtitle_prompt, None).await?;
+
+    // Combine all visual descriptions
+    let visual_description = frame_descriptions.into_iter()
+        .map(|(time, desc)| format!("At timestamp {}:\n{}", time, desc))
+        .collect::<Vec<_>>()
+        .join("\n\n");
 
     println!("\n=== Final Transcription Result ===");
     println!("\nVisual Description:");

src/App.tsx

@@ -32,9 +32,16 @@ function App() {
   const [resizeTo720p, setResizeTo720p] = useState(true);
   const [geminiEnabled, setGeminiEnabled] = useState(false);
   const [apiKey, setApiKey] = useState('AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc');
-  const [selectedTime, setSelectedTime] = useState<number>(10);
+  const [selectedFrames, setSelectedFrames] = useState<SelectedFrame[]>([]);
   const [transcription, setTranscription] = useState<TranscriptionState | null>(null);
   const [thumbnails, setThumbnails] = useState<Thumbnail[]>([]);
+
+  interface SelectedFrame {
+    path: string;
+    time: number;
+    base64Image: string;
+    description?: string;
+  }
   const [isSettingsOpen, setIsSettingsOpen] = useState(false);
 
   useEffect(() => {
@@ -97,7 +104,7 @@ function App() {
         addBlackBar,
         resizeTo720p,
         geminiEnabled,
-        frameTime: selectedTime
+          selectedFrames: selectedFrames
       });
 
       // Process transcription first if Gemini is enabled
@@ -108,7 +115,11 @@ function App() {
             args: {
               video_path: videoFile.path,
               subtitle_path: subtitleFile.path,
-              frame_time: selectedTime,
+              selected_frames: selectedFrames.map(frame => ({
+              path: frame.path,
+              time: frame.time,
+              base64_image: frame.base64Image
+            })),
               api_key: apiKey
             }
           });
@@ -226,11 +237,11 @@ function App() {
           />
 
           {geminiEnabled && videoFile && (
-            <VideoTimeline
-              videoPath={videoFile.path}
-              onTimeSelect={setSelectedTime}
-              thumbnails={thumbnails}
-            />
+          <VideoTimeline
+            videoPath={videoFile.path}
+            onFramesSelect={setSelectedFrames}
+            thumbnails={thumbnails}
+          />
           )}
 
           <div className="space-y-3">

src/components/Settings.tsx

@@ -1,4 +1,4 @@
-import React, { useState } from 'react';
+import React, { useState, useEffect } from 'react';
 import { invoke } from '@tauri-apps/api/core';
 
 interface SettingsProps {
@@ -13,9 +13,31 @@ const DEFAULT_API_KEY = 'AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc';
 export const Settings: React.FC<SettingsProps> = ({ isOpen, onClose, apiKey, onApiKeyChange }) => {
   const [testing, setTesting] = useState(false);
   const [testResult, setTestResult] = useState<string | null>(null);
+  const [isEditing, setIsEditing] = useState(false);
+  const [displayValue, setDisplayValue] = useState('*'.repeat(apiKey.length));
+
+  useEffect(() => {
+    if (!isEditing) {
+      setDisplayValue('*'.repeat(apiKey.length));
+    }
+  }, [apiKey, isEditing]);
 
   const handleApiKeyChange = (e: React.ChangeEvent<HTMLInputElement>) => {
-    onApiKeyChange(e.target.value);
+    const newValue = e.target.value;
+    onApiKeyChange(newValue);
+    if (!isEditing) {
+      setDisplayValue('*'.repeat(newValue.length));
+    }
+  };
+
+  const handleFocus = () => {
+    setIsEditing(true);
+    setDisplayValue(apiKey);
+  };
+
+  const handleBlur = () => {
+    setIsEditing(false);
+    setDisplayValue('*'.repeat(apiKey.length));
   };
 
   const resetToDefault = () => {
@@ -60,10 +82,14 @@ export const Settings: React.FC<SettingsProps> = ({ isOpen, onClose, apiKey, onA
             <input
               type="text"
               id="apiKey"
-              value={apiKey}
+              value={isEditing ? apiKey : displayValue}
               onChange={handleApiKeyChange}
-              className="w-full px-3 py-2 rounded-md border border-gray-300 focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
+              onFocus={handleFocus}
+              onBlur={handleBlur}
+              className="w-full px-3 py-2 rounded-md border border-gray-300 focus:ring-blue-500 focus:border-blue-500 sm:text-sm font-mono"
               placeholder="Enter your Gemini API key"
+              spellCheck={false}
+              autoComplete="off"
             />
           </div>
 

src/components/VideoTimeline.tsx

@@ -3,7 +3,7 @@ import { invoke } from '@tauri-apps/api/core';
 
 interface VideoTimelineProps {
   videoPath: string | null;
-  onTimeSelect: (time: number) => void;
+  onFramesSelect: (frames: SelectedFrame[]) => void;
   thumbnails: Thumbnail[];
 }
 
@@ -12,8 +12,15 @@ interface Thumbnail {
   time: number;
 }
 
-export const VideoTimeline: React.FC<VideoTimelineProps> = ({ videoPath, onTimeSelect, thumbnails }) => {
-  const [selectedTime, setSelectedTime] = useState<number>(10); // Default 10 seconds
+interface SelectedFrame {
+  path: string;
+  time: number;
+  base64Image: string;
+  description?: string;
+}
+
+export const VideoTimeline: React.FC<VideoTimelineProps> = ({ videoPath, onFramesSelect, thumbnails }) => {
+  const [selectedFrames, setSelectedFrames] = useState<SelectedFrame[]>([]);
   const [duration, setDuration] = useState<number>(0);
   const [thumbnailUrls, setThumbnailUrls] = useState<{ [key: string]: string }>({});
 
@@ -58,23 +65,60 @@ export const VideoTimeline: React.FC<VideoTimelineProps> = ({ videoPath, onTimeS
     <div className="p-4 bg-gray-50 rounded-lg">
       <div className="flex justify-between items-center mb-2">
         <label className="block text-sm font-medium text-gray-700">
-          Select Frame Time
+          Select Scene Frames
         </label>
         <span className="text-sm text-gray-500">
-          Selected: {formatTime(selectedTime)}
+          {selectedFrames.length} frames selected
         </span>
       </div>
 
+      <div className="mb-4">
+        <div className="text-sm text-gray-600 mb-2">
+          Click thumbnails to select frames for scene descriptions. Selected frames will be analyzed in chronological order.
+        </div>
+      </div>
+
       <div className="flex overflow-x-auto space-x-2 pb-2">
         {thumbnails.map((thumb, index) => (
           <div
             key={index}
             className={`relative flex-shrink-0 cursor-pointer ${
-              Math.abs(thumb.time - selectedTime) < 2 ? 'ring-2 ring-blue-500' : ''
+              selectedFrames.some(f => Math.abs(f.time - thumb.time) < 2) 
+                ? 'ring-2 ring-blue-500' 
+                : 'hover:ring-2 hover:ring-blue-300'
             }`}
-            onClick={() => {
-              setSelectedTime(thumb.time);
-              onTimeSelect(thumb.time);
+            onClick={async () => {
+              const isSelected = selectedFrames.some(
+                f => Math.abs(f.time - thumb.time) < 2
+              );
+
+              if (isSelected) {
+                // Remove from selection
+                const newFrames = selectedFrames.filter(
+                  f => Math.abs(f.time - thumb.time) >= 2
+                );
+                setSelectedFrames(newFrames);
+                onFramesSelect(newFrames);
+              } else {
+                // Add to selection
+                try {
+                  const base64 = await invoke<string>('read_thumbnail', { 
+                    path: thumb.path 
+                  });
+                  const newFrame: SelectedFrame = {
+                    path: thumb.path,
+                    time: thumb.time,
+                    base64Image: base64
+                  };
+                  const newFrames = [...selectedFrames, newFrame].sort(
+                    (a, b) => a.time - b.time
+                  );
+                  setSelectedFrames(newFrames);
+                  onFramesSelect(newFrames);
+                } catch (error) {
+                  console.error('Error loading thumbnail:', error);
+                }
+              }
             }}
           >
             {thumbnailUrls[thumb.path] && (
@@ -87,9 +131,29 @@ export const VideoTimeline: React.FC<VideoTimelineProps> = ({ videoPath, onTimeS
             <div className="absolute bottom-0 left-0 right-0 bg-black bg-opacity-50 text-white text-xs text-center py-1">
               {formatTime(thumb.time)}
             </div>
+            {selectedFrames.some(f => Math.abs(f.time - thumb.time) < 2) && (
+              <div className="absolute top-2 right-2 bg-blue-500 text-white rounded-full w-5 h-5 flex items-center justify-center text-xs">
+                {selectedFrames.findIndex(f => Math.abs(f.time - thumb.time) < 2) + 1}
+              </div>
+            )}
           </div>
         ))}
       </div>
+
+      {selectedFrames.length > 0 && (
+        <div className="mt-4">
+          <div className="text-sm font-medium text-gray-700 mb-2">
+            Selected Frames Order:
+          </div>
+          <div className="flex flex-wrap gap-2">
+            {selectedFrames.map((frame, index) => (
+              <div key={index} className="text-sm text-gray-600 bg-gray-100 rounded px-2 py-1">
+                {index + 1}. {formatTime(frame.time)}
+              </div>
+            ))}
+          </div>
+        </div>
+      )}
     </div>
   );
 };