diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index db4245e..97849c8 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -53,6 +53,8 @@ pub struct TranscriptionArgs { pub subtitle_path: String, pub selected_frames: Vec, pub api_key: String, + pub visual_prompt: String, + pub subtitle_prompt: String, } #[derive(Debug, Serialize)] @@ -259,15 +261,9 @@ pub async fn process_transcription_impl(args: TranscriptionArgs) -> Result>() .join("\n"); - let subtitle_prompt = format!(r#"I will provide you with timestamped scene descriptions and subtitle content. Create a chronological narrative that combines these elements with these strict rules: - - Scene Descriptions: - {} - - Subtitle Content with Timestamps: - {} - - Required Format Rules: - 1. Use ONLY the exact dialogue lines from the subtitle content - do not add, modify, or create new dialogue but feel free to turn them into paragraphs provided that content of exact words are not modified. - 2. For the first scene description, MOVE it to the beginning of the narration. For the rest of the scenes, ensure that they are located according to their exact timestamps when they occur in the dialogue flow. - 3. Keep all dialogue in strict chronological order and to be quoted verbatim - 4. For multiple speakers, add speaker identification before their lines until the next speaker's dialogue lines. If there is only one speaker, do not add a speaker identifier. - 5. Include speaking manner (e.g., "warmly", "firmly") only when clearly implied by the subtitle text - 6. Use paragraph breaks to separate different scenes if there is more than one scene - 7. Ensure that there are no timestamps in the output including for the scenes - - Critical Requirements: - - Do not invent or modify any dialogue - - Only use the provided subtitle text verbatim - - Scene descriptions should be inserted exactly where their timestamps occur except for the first scene which is to start at the beginning of the narration - - Maintain precise chronological order of all elements - - Keep your thoughts and ideas to yourself - - Format the output as a flowing narrative that preserves all original dialogue while weaving in the visual context at the appropriate moments as like it's a novel."#, - scene_descriptions, - formatted_subtitles - ); + let subtitle_prompt = format!("{}\n\nScene Descriptions:\n{}\n\nSubtitle Content with Timestamps:\n{}", + args.subtitle_prompt, + scene_descriptions, + formatted_subtitles + ); // Process the subtitle content with scene descriptions let subtitle_narrative = query_gemini(&args.api_key, &subtitle_prompt, None).await?; diff --git a/src/App.tsx b/src/App.tsx index 36fd32a..3b24e68 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -2,7 +2,7 @@ import { useState, useEffect } from 'react'; import { invoke } from '@tauri-apps/api/core'; import { FileSelector } from './components/FileSelector'; import { GeminiSettings } from './components/GeminiSettings'; -import { Settings } from './components/Settings'; +import { Settings, DEFAULT_VISUAL_PROMPT, DEFAULT_SUBTITLE_PROMPT } from './components/Settings'; import { VideoTimeline } from './components/VideoTimeline'; import './styles/index.css'; @@ -32,6 +32,14 @@ function App() { const [resizeTo720p, setResizeTo720p] = useState(true); const [geminiEnabled, setGeminiEnabled] = useState(false); const [apiKey, setApiKey] = useState('AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc'); + const [visualPrompt, setVisualPrompt] = useState(() => { + const saved = localStorage.getItem('visualPrompt'); + return saved || DEFAULT_VISUAL_PROMPT; + }); + const [subtitlePrompt, setSubtitlePrompt] = useState(() => { + const saved = localStorage.getItem('subtitlePrompt'); + return saved || DEFAULT_SUBTITLE_PROMPT; + }); const [selectedFrames, setSelectedFrames] = useState([]); const [transcription, setTranscription] = useState(null); const [thumbnails, setThumbnails] = useState([]); @@ -44,6 +52,15 @@ function App() { } const [isSettingsOpen, setIsSettingsOpen] = useState(false); + // Persist prompts to localStorage when they change + useEffect(() => { + localStorage.setItem('visualPrompt', visualPrompt); + }, [visualPrompt]); + + useEffect(() => { + localStorage.setItem('subtitlePrompt', subtitlePrompt); + }, [subtitlePrompt]); + useEffect(() => { invoke('check_dependencies') .catch(err => { @@ -104,7 +121,7 @@ function App() { addBlackBar, resizeTo720p, geminiEnabled, - selectedFrames: selectedFrames + selectedFrames: selectedFrames }); // Process transcription first if Gemini is enabled @@ -116,11 +133,13 @@ function App() { video_path: videoFile.path, subtitle_path: subtitleFile.path, selected_frames: selectedFrames.map(frame => ({ - path: frame.path, - time: frame.time, - base64_image: frame.base64Image - })), - api_key: apiKey + path: frame.path, + time: frame.time, + base64_image: frame.base64Image + })), + api_key: apiKey, + visual_prompt: visualPrompt, + subtitle_prompt: subtitlePrompt } }); console.log('Transcription completed with full response:', transcriptionResult); @@ -324,6 +343,10 @@ function App() { onClose={() => setIsSettingsOpen(false)} apiKey={apiKey} onApiKeyChange={setApiKey} + visualPrompt={visualPrompt} + subtitlePrompt={subtitlePrompt} + onVisualPromptChange={setVisualPrompt} + onSubtitlePromptChange={setSubtitlePrompt} /> ); diff --git a/src/components/Settings.tsx b/src/components/Settings.tsx index 4f2d123..857bfd3 100644 --- a/src/components/Settings.tsx +++ b/src/components/Settings.tsx @@ -6,11 +6,51 @@ interface SettingsProps { onClose: () => void; apiKey: string; onApiKeyChange: (key: string) => void; + visualPrompt: string; + subtitlePrompt: string; + onVisualPromptChange: (prompt: string) => void; + onSubtitlePromptChange: (prompt: string) => void; } -const DEFAULT_API_KEY = 'AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc'; +export const DEFAULT_API_KEY = 'AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc'; +export const DEFAULT_VISUAL_PROMPT = `Analyze this video frame from timestamp {} and provide a visual description focusing on: +1. The visual setup and environment +2. The people present, their appearance, and positioning +3. Any relevant visual context or background details +4. Do not include details about the individuals' hands -export const Settings: React.FC = ({ isOpen, onClose, apiKey, onApiKeyChange }) => { +Format the description in clear, concise paragraphs that would be helpful for DeafBlind readers to understand the visual context. Focus on spatial relationships and important visual details that contribute to understanding the scene. Use maximum of 200 words and do not include your own thoughts-- just provide the visual description alone.`; + +export const DEFAULT_SUBTITLE_PROMPT = `I will provide you with timestamped scene descriptions and subtitle content. Create a chronological narrative that combines these elements with these strict rules: + +Required Format Rules: +1. Use ONLY the exact dialogue lines from the subtitle content - do not add, modify, or create new dialogue but feel free to turn them into paragraphs provided that content of exact words are not modified. +2. For the first scene description, MOVE it to the beginning of the narration. For the rest of the scenes, ensure that they are located according to their exact timestamps when they occur in the dialogue flow. +3. Keep all dialogue in strict chronological order and to be quoted verbatim +4. For multiple speakers, add speaker identification before their lines until the next speaker's dialogue lines. If there is only one speaker, do not add a speaker identifier. +5. Include speaking manner (e.g., "warmly", "firmly") only when clearly implied by the subtitle text +6. Use paragraph breaks to separate different scenes if there is more than one scene +7. Ensure that there are no timestamps in the output including for the scenes + +Critical Requirements: +- Do not invent or modify any dialogue +- Only use the provided subtitle text verbatim +- Scene descriptions should be inserted exactly where their timestamps occur except for the first scene which is to start at the beginning of the narration +- Maintain precise chronological order of all elements +- Keep your thoughts and ideas to yourself + +Format the output as a flowing narrative that preserves all original dialogue while weaving in the visual context at the appropriate moments as like it's a novel.`; + +export const Settings: React.FC = ({ + isOpen, + onClose, + apiKey, + onApiKeyChange, + visualPrompt, + subtitlePrompt, + onVisualPromptChange, + onSubtitlePromptChange +}) => { const [testing, setTesting] = useState(false); const [testResult, setTestResult] = useState(null); const [isEditing, setIsEditing] = useState(false); @@ -40,10 +80,18 @@ export const Settings: React.FC = ({ isOpen, onClose, apiKey, onA setDisplayValue('*'.repeat(apiKey.length)); }; - const resetToDefault = () => { + const resetApiKeyToDefault = () => { onApiKeyChange(DEFAULT_API_KEY); }; + const resetVisualPromptToDefault = () => { + onVisualPromptChange(DEFAULT_VISUAL_PROMPT); + }; + + const resetSubtitlePromptToDefault = () => { + onSubtitlePromptChange(DEFAULT_SUBTITLE_PROMPT); + }; + const testApiKey = async () => { setTesting(true); setTestResult(null); @@ -104,7 +152,7 @@ export const Settings: React.FC = ({ isOpen, onClose, apiKey, onA {apiKey !== DEFAULT_API_KEY && (