Break into two components

This commit is contained in:
TheMaddax 2025-01-26 16:23:25 -06:00
parent f1610c06a8
commit 2b44625655
6 changed files with 546 additions and 409 deletions

View file

@ -50,7 +50,7 @@ pub struct SelectedFrame {
#[derive(Debug, Deserialize)]
pub struct TranscriptionArgs {
pub video_path: String,
pub subtitle_path: String,
pub subtitle_path: Option<String>,
pub selected_frames: Vec<SelectedFrame>,
pub api_key: String,
pub visual_prompt: String,
@ -61,7 +61,7 @@ pub struct TranscriptionArgs {
#[serde(rename_all = "camelCase")]
pub struct TranscriptionResult {
pub visual_description: String,
pub subtitle_narrative: String,
pub narrative: String,
}
#[derive(Debug, Serialize)]
@ -244,7 +244,9 @@ pub async fn extract_thumbnails_impl(video_path: &str, interval: u32) -> Result<
pub async fn process_transcription_impl(args: TranscriptionArgs) -> Result<TranscriptionResult, Error> {
println!("\n=== Starting Transcription Process ===");
println!("Video Path: {}", args.video_path);
println!("Subtitle Path: {}", args.subtitle_path);
if let Some(subtitle_path) = &args.subtitle_path {
println!("Subtitle Path: {}", subtitle_path);
}
println!("Number of selected frames: {}", args.selected_frames.len());
let temp_dir = TempDir::new().map_err(|e| Error::IoError(e))?;
@ -267,64 +269,6 @@ pub async fn process_transcription_impl(args: TranscriptionArgs) -> Result<Trans
frame_descriptions.push((frame.time, description));
}
println!("\nReading and parsing subtitle content...");
let subtitle_content = fs::read_to_string(&args.subtitle_path)
.map_err(|e| Error::IoError(e))?;
println!("Subtitle content length: {} characters", subtitle_content.len());
// Parse subtitle timestamps and content, handling both SRT and ASS formats
let mut subtitle_lines = Vec::new();
let lines: Vec<&str> = subtitle_content.lines().collect();
// Check if it's an ASS file
if args.subtitle_path.to_lowercase().ends_with(".ass") {
let mut in_events = false;
for line in lines {
if line.starts_with("[Events]") {
in_events = true;
continue;
}
if in_events && line.starts_with("Dialogue:") {
let parts: Vec<&str> = line.split(',').collect();
if parts.len() >= 10 {
let start_time = parts[1].trim();
// Combine all remaining parts as the text, skipping style and positioning info
let text = parts[9..].join(",").trim().to_string();
if !text.is_empty() {
subtitle_lines.push((start_time.to_string(), text));
}
}
}
}
} else {
// SRT format parsing
let mut i = 0;
while i < lines.len() {
if i + 3 <= lines.len() && lines[i + 1].contains("-->") {
let timestamp_parts: Vec<&str> = lines[i + 1].split(" --> ").collect();
if timestamp_parts.len() == 2 {
let start_time = timestamp_parts[0].trim();
let mut text = lines[i + 2].trim().to_string();
// Collect multi-line subtitle text
let mut j = i + 3;
while j < lines.len() && !lines[j].trim().is_empty() && !lines[j].contains("-->") {
text.push_str(" ");
text.push_str(lines[j].trim());
j += 1;
}
subtitle_lines.push((start_time.to_string(), text));
i = j;
}
}
i += 1;
}
}
// Sort subtitles by timestamp
subtitle_lines.sort_by(|a, b| a.0.cmp(&b.0));
// Format scene descriptions with timestamps in MM:SS format
let scene_descriptions = frame_descriptions.iter()
.map(|(time, desc)| {
@ -335,47 +279,112 @@ pub async fn process_transcription_impl(args: TranscriptionArgs) -> Result<Trans
.collect::<Vec<_>>()
.join("\n\n");
// Format subtitle content with timestamps
let formatted_subtitles = subtitle_lines.iter()
.map(|(time, text)| format!("[{}] {}", time, text))
.collect::<Vec<_>>()
.join("\n");
let subtitle_prompt = format!("{}\n\nScene Descriptions:\n{}\n\nSubtitle Content with Timestamps:\n{}",
args.subtitle_prompt,
scene_descriptions,
formatted_subtitles
);
// Process the subtitle content with scene descriptions
let subtitle_narrative = query_gemini(&args.api_key, &subtitle_prompt, None).await?;
// Combine all visual descriptions
let visual_description = frame_descriptions.into_iter()
// Combine all visual descriptions for the raw visual description output
let visual_description = frame_descriptions.iter()
.map(|(time, desc)| format!("At timestamp {}:\n{}", time, desc))
.collect::<Vec<_>>()
.join("\n\n");
// Process subtitle content if provided
let narrative = if let Some(subtitle_path) = &args.subtitle_path {
println!("\nReading and parsing subtitle content...");
let subtitle_content = fs::read_to_string(subtitle_path)
.map_err(|e| Error::IoError(e))?;
println!("Subtitle content length: {} characters", subtitle_content.len());
// Parse subtitle timestamps and content, handling both SRT and ASS formats
let mut subtitle_lines = Vec::new();
let lines: Vec<&str> = subtitle_content.lines().collect();
// Check if it's an ASS file
if subtitle_path.to_lowercase().ends_with(".ass") {
let mut in_events = false;
for line in lines {
if line.starts_with("[Events]") {
in_events = true;
continue;
}
if in_events && line.starts_with("Dialogue:") {
let parts: Vec<&str> = line.split(',').collect();
if parts.len() >= 10 {
let start_time = parts[1].trim();
// Combine all remaining parts as the text, skipping style and positioning info
let text = parts[9..].join(",").trim().to_string();
if !text.is_empty() {
subtitle_lines.push((start_time.to_string(), text));
}
}
}
}
} else {
// SRT format parsing
let mut i = 0;
while i < lines.len() {
if i + 3 <= lines.len() && lines[i + 1].contains("-->") {
let timestamp_parts: Vec<&str> = lines[i + 1].split(" --> ").collect();
if timestamp_parts.len() == 2 {
let start_time = timestamp_parts[0].trim();
let mut text = lines[i + 2].trim().to_string();
// Collect multi-line subtitle text
let mut j = i + 3;
while j < lines.len() && !lines[j].trim().is_empty() && !lines[j].contains("-->") {
text.push_str(" ");
text.push_str(lines[j].trim());
j += 1;
}
subtitle_lines.push((start_time.to_string(), text));
i = j;
}
}
i += 1;
}
}
// Sort subtitles by timestamp
subtitle_lines.sort_by(|a, b| a.0.cmp(&b.0));
// Format subtitle content with timestamps
let formatted_subtitles = subtitle_lines.iter()
.map(|(time, text)| format!("[{}] {}", time, text))
.collect::<Vec<_>>()
.join("\n");
let narrative_prompt = format!("{}\n\nScene Descriptions:\n{}\n\nSubtitle Content with Timestamps:\n{}",
args.subtitle_prompt,
scene_descriptions,
formatted_subtitles
);
// Process the subtitle content with scene descriptions
query_gemini(&args.api_key, &narrative_prompt, None).await?
} else {
// If no subtitle file, just format the scene descriptions as a narrative
let narrative_prompt = format!("Create a flowing narrative from these scene descriptions, maintaining their chronological order and focusing on spatial relationships and visual details that would be helpful for DeafBlind readers:\n\n{}", scene_descriptions);
query_gemini(&args.api_key, &narrative_prompt, None).await?
};
println!("\n=== Final Transcription Result ===");
println!("\nVisual Description:");
println!("{}", visual_description);
println!("\nSubtitle Narrative:");
println!("{}", subtitle_narrative);
println!("\nNarrative:");
println!("{}", narrative);
// Create the result struct
let result = TranscriptionResult {
visual_description: visual_description.clone(),
subtitle_narrative: subtitle_narrative.clone(),
narrative: narrative.clone(),
};
// Log the final result structure
println!("\nReturning transcription result with:");
println!("- Visual description length: {}", result.visual_description.len());
println!("- Subtitle narrative length: {}", result.subtitle_narrative.len());
println!("- Narrative length: {}", result.narrative.len());
println!("\nFinal TranscriptionResult struct:");
println!("{{");
println!(" \"visualDescription\": \"{}\",", result.visual_description);
println!(" \"subtitleNarrative\": \"{}\"", result.subtitle_narrative);
println!(" \"narrative\": \"{}\"", result.narrative);
println!("}}");
Ok(result)

View file

@ -1,36 +1,16 @@
import { useState, useEffect } from 'react';
import { invoke } from '@tauri-apps/api/core';
import { FileSelector } from './components/FileSelector';
import { GeminiSettings } from './components/GeminiSettings';
import { Settings, DEFAULT_VISUAL_PROMPT, DEFAULT_SUBTITLE_PROMPT } from './components/Settings';
import { VideoTimeline } from './components/VideoTimeline';
import { TranscriptionTask } from './components/TranscriptionTask';
import { MergeTask } from './components/MergeTask';
import './styles/index.css';
interface FileState {
path: string;
name: string;
}
// Must match the Rust TranscriptionResult struct exactly
interface TranscriptionState {
visualDescription: string; // from visual_description in Rust
subtitleNarrative: string; // from subtitle_narrative in Rust
}
interface Thumbnail {
path: string;
time: number;
}
type TaskType = 'transcription' | 'merge' | null;
function App() {
const [videoFile, setVideoFile] = useState<FileState | null>(null);
const [subtitleFile, setSubtitleFile] = useState<FileState | null>(null);
const [progress, setProgress] = useState<number>(0);
const [isProcessing, setIsProcessing] = useState(false);
const [selectedTask, setSelectedTask] = useState<TaskType>(null);
const [error, setError] = useState<string | null>(null);
const [addBlackBar, setAddBlackBar] = useState(true);
const [resizeTo720p, setResizeTo720p] = useState(true);
const [geminiEnabled, setGeminiEnabled] = useState(false);
const [isSettingsOpen, setIsSettingsOpen] = useState(false);
const [apiKey, setApiKey] = useState('AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc');
const [visualPrompt, setVisualPrompt] = useState(() => {
const saved = localStorage.getItem('visualPrompt');
@ -40,17 +20,6 @@ function App() {
const saved = localStorage.getItem('subtitlePrompt');
return saved || DEFAULT_SUBTITLE_PROMPT;
});
const [selectedFrames, setSelectedFrames] = useState<SelectedFrame[]>([]);
const [transcription, setTranscription] = useState<TranscriptionState | null>(null);
const [thumbnails, setThumbnails] = useState<Thumbnail[]>([]);
interface SelectedFrame {
path: string;
time: number;
base64Image: string;
description?: string;
}
const [isSettingsOpen, setIsSettingsOpen] = useState(false);
// Persist prompts to localStorage when they change
useEffect(() => {
@ -69,143 +38,6 @@ function App() {
});
}, []);
const handleVideoSelect = (path: string) => {
const name = path.split('/').pop() || '';
setVideoFile({ path, name });
setThumbnails([]); // Clear thumbnails when new video is selected
};
const handleSubtitleSelect = (path: string) => {
const name = path.split('/').pop() || '';
setSubtitleFile({ path, name });
};
const handleThumbnailsGenerated = (newThumbnails: Thumbnail[]) => {
setThumbnails(newThumbnails);
};
useEffect(() => {
let intervalId: number | undefined;
if (isProcessing) {
// Poll for progress every 100ms while processing
intervalId = window.setInterval(async () => {
try {
const currentProgress = await invoke<number>('get_progress');
console.log('Progress update:', currentProgress);
setProgress(currentProgress);
} catch (error) {
console.error('Failed to get progress:', error);
}
}, 100);
}
return () => {
if (intervalId !== undefined) {
window.clearInterval(intervalId);
}
};
}, [isProcessing]);
const handleMerge = async () => {
if (!videoFile || !subtitleFile) return;
setIsProcessing(true);
setProgress(0);
setTranscription(null);
try {
console.log('Starting merge with args:', {
videoPath: videoFile.path,
subtitlePath: subtitleFile.path,
addBlackBar,
resizeTo720p,
geminiEnabled,
selectedFrames: selectedFrames
});
// Process transcription first if Gemini is enabled
if (geminiEnabled) {
try {
console.log('Starting transcription process...');
const transcriptionResult = await invoke<TranscriptionState>('process_transcription', {
args: {
video_path: videoFile.path,
subtitle_path: subtitleFile.path,
selected_frames: selectedFrames.map(frame => ({
path: frame.path,
time: frame.time,
base64_image: frame.base64Image
})),
api_key: apiKey,
visual_prompt: visualPrompt,
subtitle_prompt: subtitlePrompt
}
});
console.log('Transcription completed with full response:', transcriptionResult);
// Log the raw response first
console.log('Raw transcription result:', JSON.stringify(transcriptionResult, null, 2));
// Validate the response structure
if (!transcriptionResult || typeof transcriptionResult !== 'object') {
console.error('Invalid transcription result:', transcriptionResult);
setError('Invalid transcription response format');
return;
}
// Validate and clean the content
const visualDescription = String(transcriptionResult.visualDescription || '').trim();
const subtitleNarrative = String(transcriptionResult.subtitleNarrative || '').trim();
if (!visualDescription || !subtitleNarrative) {
console.error('Missing transcription content:', {
visualDescription: !!visualDescription,
subtitleNarrative: !!subtitleNarrative
});
setError('Transcription response is missing required content');
return;
}
// Create a new state object with the validated content
const newTranscription: TranscriptionState = {
visualDescription,
subtitleNarrative
};
console.log('Setting transcription state:', {
visualDescriptionLength: visualDescription.length,
subtitleNarrativeLength: subtitleNarrative.length
});
setTranscription(newTranscription);
} catch (error) {
console.error('Transcription error:', error);
setError(`Transcription error: ${error}`);
return;
}
}
// Then start video merging process
const mergeResult = await invoke<string>('merge_video_subtitle', {
args: {
video_path: videoFile.path,
subtitle_path: subtitleFile.path,
add_black_bar: addBlackBar,
resize_to_720p: resizeTo720p
}
});
console.log('Merge completed:', mergeResult);
setProgress(100);
} catch (error) {
console.error('Error merging files:', error);
setError(error as string);
} finally {
setIsProcessing(false);
}
};
return (
<div className="min-h-screen bg-gray-100 py-8 px-4">
<div className="max-w-2xl mx-auto bg-white rounded-lg shadow-md p-6">
@ -222,120 +54,68 @@ function App() {
Settings
</button>
</div>
<div className="space-y-6">
<div className="flex items-center justify-between">
<div className="flex-1 mr-4">
<p className="text-sm text-gray-600 mb-2">Video file:</p>
<p className="text-gray-800">{videoFile?.name || 'No file selected'}</p>
</div>
<FileSelector
label="Select Video"
onFileSelect={handleVideoSelect}
accept={['mp4', 'avi', 'mov', 'mkv']}
/>
{error && (
<div className="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative mb-4">
<strong className="font-bold">Error: </strong>
<span className="block sm:inline">{error}</span>
</div>
)}
<div className="flex items-center justify-between">
<div className="flex-1 mr-4">
<p className="text-sm text-gray-600 mb-2">Subtitle file:</p>
<p className="text-gray-800">{subtitleFile?.name || 'No file selected'}</p>
{!selectedTask ? (
<div className="space-y-6">
<h2 className="text-lg font-semibold text-center mb-8">Choose a Task</h2>
<div className="grid grid-cols-2 gap-6">
<button
onClick={() => setSelectedTask('transcription')}
className="flex flex-col items-center p-6 border-2 border-gray-200 rounded-lg hover:border-blue-500 hover:bg-blue-50 transition-colors"
>
<svg className="w-12 h-12 text-blue-500 mb-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
</svg>
<span className="text-lg font-medium">Generate Transcription</span>
<p className="text-sm text-gray-500 text-center mt-2">Create AI-powered descriptions for DeafBlind accessibility</p>
</button>
<button
onClick={() => setSelectedTask('merge')}
className="flex flex-col items-center p-6 border-2 border-gray-200 rounded-lg hover:border-blue-500 hover:bg-blue-50 transition-colors"
>
<svg className="w-12 h-12 text-blue-500 mb-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M7 4v16M17 4v16M3 8h4m10 0h4M3 12h18M3 16h4m10 0h4M4 20h16a1 1 0 001-1V5a1 1 0 00-1-1H4a1 1 0 00-1 1v14a1 1 0 001 1z" />
</svg>
<span className="text-lg font-medium">Merge Subtitles</span>
<p className="text-sm text-gray-500 text-center mt-2">Embed subtitle files into your video</p>
</button>
</div>
<FileSelector
label="Select Subtitle"
onFileSelect={handleSubtitleSelect}
accept={['srt', 'ass']}
/>
</div>
) : (
<div className="space-y-6">
<div className="flex items-center mb-6">
<button
onClick={() => setSelectedTask(null)}
className="inline-flex items-center text-gray-600 hover:text-gray-900"
>
<svg className="w-5 h-5 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M10 19l-7-7m0 0l7-7m-7 7h18" />
</svg>
Back to Tasks
</button>
</div>
<GeminiSettings
enabled={geminiEnabled}
onEnableChange={setGeminiEnabled}
videoFile={videoFile?.path || null}
onThumbnailsGenerated={handleThumbnailsGenerated}
/>
{geminiEnabled && videoFile && (
<VideoTimeline
videoPath={videoFile.path}
onFramesSelect={setSelectedFrames}
thumbnails={thumbnails}
/>
)}
<div className="space-y-3">
<label className="flex items-center space-x-2">
<input
type="checkbox"
checked={addBlackBar}
onChange={(e) => setAddBlackBar(e.target.checked)}
className="input-checkbox"
{selectedTask === 'transcription' && (
<TranscriptionTask
apiKey={apiKey}
visualPrompt={visualPrompt}
subtitlePrompt={subtitlePrompt}
/>
<span>Add black bar for subtitles</span>
</label>
)}
<label className="flex items-center space-x-2">
<input
type="checkbox"
checked={resizeTo720p}
onChange={(e) => setResizeTo720p(e.target.checked)}
className="input-checkbox"
/>
<span>Resize to 720p resolution</span>
</label>
{selectedTask === 'merge' && (
<MergeTask />
)}
</div>
{transcription ? (
<div className="mt-6 space-y-4">
<h3 className="text-lg font-semibold">DeafBlind Accessibility Transcription</h3>
<div className="h-[400px] overflow-y-auto p-4 bg-gray-50 rounded-lg text-sm space-y-6">
<div>
<h4 className="font-semibold text-gray-700 mb-2">Visual Description</h4>
<div className="bg-white p-4 rounded border border-gray-200">
<p className="whitespace-pre-wrap leading-relaxed">{transcription.visualDescription}</p>
</div>
</div>
<div>
<h4 className="font-semibold text-gray-700 mb-2">Subtitle Narrative</h4>
<div className="bg-white p-4 rounded border border-gray-200">
<p className="whitespace-pre-wrap leading-relaxed">{transcription.subtitleNarrative}</p>
</div>
</div>
</div>
</div>
) : null}
<div className="space-y-2">
<div className="h-2.5 bg-gray-200 rounded-full">
<div
className="h-2.5 bg-blue-600 rounded-full transition-all duration-300"
style={{ width: `${progress}%` }}
/>
</div>
<p className="text-center text-sm text-gray-600">
{isProcessing ? `${Math.round(progress)}%` : ''}
</p>
</div>
{error && (
<div className="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative mb-4">
<strong className="font-bold">Error: </strong>
<span className="block sm:inline">{error}</span>
</div>
)}
<button
onClick={handleMerge}
disabled={!videoFile || !subtitleFile || isProcessing}
className={`w-full btn ${
!videoFile || !subtitleFile || isProcessing
? 'bg-gray-400 cursor-not-allowed'
: 'btn-primary'
}`}
>
{isProcessing ? 'Processing...' : 'Merge Video and Subtitle'}
</button>
</div>
)}
</div>
<Settings

View file

@ -2,25 +2,17 @@ import React, { useState, useCallback } from 'react';
import { invoke } from '@tauri-apps/api/core';
interface GeminiSettingsProps {
enabled: boolean;
onEnableChange: (enabled: boolean) => void;
videoFile: string | null;
onThumbnailsGenerated: (thumbnails: { path: string; time: number; }[]) => void;
}
export const GeminiSettings: React.FC<GeminiSettingsProps> = ({
enabled,
onEnableChange,
videoFile,
onThumbnailsGenerated
}) => {
const [generatingThumbnails, setGeneratingThumbnails] = useState(false);
const [thumbnailResult, setThumbnailResult] = useState<string | null>(null);
const handleEnableChange = (e: React.ChangeEvent<HTMLInputElement>) => {
onEnableChange(e.target.checked);
};
const generateThumbnails = useCallback(async () => {
if (!videoFile) {
setThumbnailResult("Please select a video file before generating thumbnails");
@ -42,46 +34,31 @@ export const GeminiSettings: React.FC<GeminiSettingsProps> = ({
return (
<div className="space-y-4 p-4 bg-gray-50 rounded-lg">
<div className="flex items-center space-x-2">
<input
type="checkbox"
id="enableGemini"
checked={enabled}
onChange={handleEnableChange}
className="h-4 w-4 text-blue-500 rounded border-gray-300 focus:ring-blue-500"
/>
<label htmlFor="enableGemini" className="text-sm font-medium text-gray-700">
Enable DeafBlind Accessibility (Gemini Vision)
</label>
</div>
{enabled && (
<div className="space-y-3">
<button
onClick={generateThumbnails}
disabled={generatingThumbnails || !videoFile}
className="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-blue-500 hover:bg-blue-600 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50"
>
{generatingThumbnails ? (
<>
<svg className="animate-spin -ml-1 mr-3 h-5 w-5 text-white" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
Generating...
</>
) : (
'Generate Thumbnails'
)}
</button>
{thumbnailResult && (
<div className={`text-sm ${thumbnailResult.startsWith('Error') || thumbnailResult.startsWith('Please') ? 'text-red-600' : 'text-green-600'}`}>
{thumbnailResult}
</div>
<div className="space-y-3">
<button
onClick={generateThumbnails}
disabled={generatingThumbnails || !videoFile}
className="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-blue-500 hover:bg-blue-600 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50"
>
{generatingThumbnails ? (
<>
<svg className="animate-spin -ml-1 mr-3 h-5 w-5 text-white" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
Generating...
</>
) : (
'Generate Thumbnails'
)}
</div>
)}
</button>
{thumbnailResult && (
<div className={`text-sm ${thumbnailResult.startsWith('Error') || thumbnailResult.startsWith('Please') ? 'text-red-600' : 'text-green-600'}`}>
{thumbnailResult}
</div>
)}
</div>
</div>
);
};

View file

@ -0,0 +1,158 @@
import React, { useState, useEffect } from 'react';
import { invoke } from '@tauri-apps/api/core';
import { FileSelector } from './FileSelector';
interface FileState {
path: string;
name: string;
}
export const MergeTask: React.FC = () => {
const [videoFile, setVideoFile] = useState<FileState | null>(null);
const [subtitleFile, setSubtitleFile] = useState<FileState | null>(null);
const [progress, setProgress] = useState<number>(0);
const [isProcessing, setIsProcessing] = useState(false);
const [error, setError] = useState<string | null>(null);
const [addBlackBar, setAddBlackBar] = useState(true);
const [resizeTo720p, setResizeTo720p] = useState(true);
useEffect(() => {
let intervalId: number | undefined;
if (isProcessing) {
// Poll for progress every 100ms while processing
intervalId = window.setInterval(async () => {
try {
const currentProgress = await invoke<number>('get_progress');
setProgress(currentProgress);
} catch (error) {
console.error('Failed to get progress:', error);
}
}, 100);
}
return () => {
if (intervalId !== undefined) {
window.clearInterval(intervalId);
}
};
}, [isProcessing]);
const handleVideoSelect = (path: string) => {
const name = path.split('/').pop() || '';
setVideoFile({ path, name });
};
const handleSubtitleSelect = (path: string) => {
const name = path.split('/').pop() || '';
setSubtitleFile({ path, name });
};
const handleMerge = async () => {
if (!videoFile || !subtitleFile) return;
setIsProcessing(true);
setProgress(0);
setError(null);
try {
const mergeResult = await invoke<string>('merge_video_subtitle', {
args: {
video_path: videoFile.path,
subtitle_path: subtitleFile.path,
add_black_bar: addBlackBar,
resize_to_720p: resizeTo720p
}
});
console.log('Merge completed:', mergeResult);
setProgress(100);
} catch (error) {
console.error('Error merging files:', error);
setError(error as string);
} finally {
setIsProcessing(false);
}
};
return (
<div className="space-y-6">
<div className="flex items-center justify-between">
<div className="flex-1 mr-4">
<p className="text-sm text-gray-600 mb-2">Video file:</p>
<p className="text-gray-800">{videoFile?.name || 'No file selected'}</p>
</div>
<FileSelector
label="Select Video"
onFileSelect={handleVideoSelect}
accept={['mp4', 'avi', 'mov', 'mkv']}
/>
</div>
<div className="flex items-center justify-between">
<div className="flex-1 mr-4">
<p className="text-sm text-gray-600 mb-2">Subtitle file:</p>
<p className="text-gray-800">{subtitleFile?.name || 'No file selected'}</p>
</div>
<FileSelector
label="Select Subtitle"
onFileSelect={handleSubtitleSelect}
accept={['srt', 'ass']}
/>
</div>
<div className="space-y-3">
<label className="flex items-center space-x-2">
<input
type="checkbox"
checked={addBlackBar}
onChange={(e) => setAddBlackBar(e.target.checked)}
className="h-4 w-4 text-blue-500 rounded border-gray-300 focus:ring-blue-500"
/>
<span>Add black bar for subtitles</span>
</label>
<label className="flex items-center space-x-2">
<input
type="checkbox"
checked={resizeTo720p}
onChange={(e) => setResizeTo720p(e.target.checked)}
className="h-4 w-4 text-blue-500 rounded border-gray-300 focus:ring-blue-500"
/>
<span>Resize to 720p resolution</span>
</label>
</div>
<div className="space-y-2">
<div className="h-2.5 bg-gray-200 rounded-full">
<div
className="h-2.5 bg-blue-600 rounded-full transition-all duration-300"
style={{ width: `${progress}%` }}
/>
</div>
<p className="text-center text-sm text-gray-600">
{isProcessing ? `${Math.round(progress)}%` : ''}
</p>
</div>
{error && (
<div className="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative mb-4">
<strong className="font-bold">Error: </strong>
<span className="block sm:inline">{error}</span>
</div>
)}
<button
onClick={handleMerge}
disabled={!videoFile || !subtitleFile || isProcessing}
className={`w-full px-4 py-2 rounded-md text-white font-medium ${
!videoFile || !subtitleFile || isProcessing
? 'bg-gray-400 cursor-not-allowed'
: 'bg-blue-500 hover:bg-blue-600 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500'
}`}
>
{isProcessing ? 'Merging...' : 'Merge Video and Subtitle'}
</button>
</div>
);
};

View file

@ -12,6 +12,8 @@ interface SettingsProps {
onSubtitlePromptChange: (prompt: string) => void;
}
// Note: We keep subtitle prompt in Settings since it's still used by the merge task
export const DEFAULT_API_KEY = 'AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc';
export const DEFAULT_VISUAL_PROMPT = `Analyze this video frame from timestamp {} and provide a visual description focusing on:
1. The visual setup and environment
@ -21,6 +23,7 @@ export const DEFAULT_VISUAL_PROMPT = `Analyze this video frame from timestamp {}
Format the description in clear, concise paragraphs that would be helpful for DeafBlind readers to understand the visual context. Focus on spatial relationships and important visual details that contribute to understanding the scene. Use maximum of 200 words and do not include your own thoughts-- just provide the visual description alone.`;
// Note: We keep subtitle prompt since it's still used by the merge task
export const DEFAULT_SUBTITLE_PROMPT = `I will provide you with timestamped scene descriptions and subtitle content. Create a chronological narrative that combines these elements with these strict rules:
Required Format Rules:

View file

@ -0,0 +1,210 @@
import React, { useState } from 'react';
import { invoke } from '@tauri-apps/api/core';
import { FileSelector } from './FileSelector';
import { GeminiSettings } from './GeminiSettings';
import { VideoTimeline } from './VideoTimeline';
interface FileState {
path: string;
name: string;
}
interface TranscriptionState {
visualDescription: string;
narrative: string;
}
interface Thumbnail {
path: string;
time: number;
}
interface SelectedFrame {
path: string;
time: number;
base64Image: string;
description?: string;
}
interface TranscriptionTaskProps {
apiKey: string;
visualPrompt: string;
subtitlePrompt: string;
}
export const TranscriptionTask: React.FC<TranscriptionTaskProps> = ({
apiKey,
visualPrompt,
subtitlePrompt,
}) => {
const [videoFile, setVideoFile] = useState<FileState | null>(null);
const [subtitleFile, setSubtitleFile] = useState<FileState | null>(null);
const [progress, setProgress] = useState<number>(0);
const [isProcessing, setIsProcessing] = useState(false);
const [error, setError] = useState<string | null>(null);
const [selectedFrames, setSelectedFrames] = useState<SelectedFrame[]>([]);
const [transcription, setTranscription] = useState<TranscriptionState | null>(null);
const [thumbnails, setThumbnails] = useState<Thumbnail[]>([]);
const handleVideoSelect = (path: string) => {
const name = path.split('/').pop() || '';
setVideoFile({ path, name });
setThumbnails([]); // Clear thumbnails when new video is selected
};
const handleSubtitleSelect = (path: string) => {
const name = path.split('/').pop() || '';
setSubtitleFile({ path, name });
};
const handleThumbnailsGenerated = (newThumbnails: Thumbnail[]) => {
setThumbnails(newThumbnails);
};
const handleGenerateTranscription = async () => {
if (!videoFile || selectedFrames.length === 0) {
setError('Please select a video and at least one frame');
return;
}
setIsProcessing(true);
setProgress(0);
setError(null);
setTranscription(null);
try {
setProgress(10); // Starting
console.log('Starting transcription process...');
setProgress(20); // Processing frames
const transcriptionResult = await invoke<TranscriptionState>('process_transcription', {
args: {
video_path: videoFile.path,
subtitle_path: subtitleFile?.path,
selected_frames: selectedFrames.map(frame => ({
path: frame.path,
time: frame.time,
base64_image: frame.base64Image
})),
api_key: apiKey,
visual_prompt: visualPrompt,
subtitle_prompt: subtitlePrompt
}
});
// Validate and clean the content
const visualDescription = String(transcriptionResult.visualDescription || '').trim();
const narrative = String(transcriptionResult.narrative || '').trim();
if (!visualDescription) {
throw new Error('Visual description is missing from the response');
}
setTranscription({
visualDescription,
narrative
});
setProgress(100); // Complete
} catch (error) {
console.error('Transcription error:', error);
setError(`Transcription error: ${error}`);
} finally {
setIsProcessing(false);
}
};
return (
<div className="space-y-6">
<div className="space-y-4">
<div className="flex items-center justify-between">
<div className="flex-1 mr-4">
<p className="text-sm text-gray-600 mb-2">Video file:</p>
<p className="text-gray-800">{videoFile?.name || 'No file selected'}</p>
</div>
<FileSelector
label="Select Video"
onFileSelect={handleVideoSelect}
accept={['mp4', 'avi', 'mov', 'mkv']}
/>
</div>
<div className="flex items-center justify-between">
<div className="flex-1 mr-4">
<p className="text-sm text-gray-600 mb-2">Subtitle file (optional):</p>
<p className="text-gray-800">{subtitleFile?.name || 'No file selected'}</p>
</div>
<FileSelector
label="Select Subtitle"
onFileSelect={handleSubtitleSelect}
accept={['srt', 'ass']}
/>
</div>
</div>
<GeminiSettings
videoFile={videoFile?.path || null}
onThumbnailsGenerated={handleThumbnailsGenerated}
/>
{videoFile && (
<VideoTimeline
videoPath={videoFile.path}
onFramesSelect={setSelectedFrames}
thumbnails={thumbnails}
/>
)}
{transcription && (
<div className="mt-6 space-y-4">
<h3 className="text-lg font-semibold">DeafBlind Accessibility Transcription</h3>
<div className="h-[400px] overflow-y-auto p-4 bg-gray-50 rounded-lg text-sm space-y-6">
<div>
<h4 className="font-semibold text-gray-700 mb-2">Visual Description</h4>
<div className="bg-white p-4 rounded border border-gray-200">
<p className="whitespace-pre-wrap leading-relaxed">{transcription.visualDescription}</p>
</div>
</div>
<div>
<h4 className="font-semibold text-gray-700 mb-2">Narrative</h4>
<div className="bg-white p-4 rounded border border-gray-200">
<p className="whitespace-pre-wrap leading-relaxed">{transcription.narrative}</p>
</div>
</div>
</div>
</div>
)}
<div className="space-y-2">
<div className="h-2.5 bg-gray-200 rounded-full">
<div
className="h-2.5 bg-blue-600 rounded-full transition-all duration-300"
style={{ width: `${progress}%` }}
/>
</div>
<p className="text-center text-sm text-gray-600">
{isProcessing ? `${Math.round(progress)}%` : ''}
</p>
</div>
{error && (
<div className="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative mb-4">
<strong className="font-bold">Error: </strong>
<span className="block sm:inline">{error}</span>
</div>
)}
<button
onClick={handleGenerateTranscription}
disabled={!videoFile || selectedFrames.length === 0 || isProcessing}
className={`w-full px-4 py-2 rounded-md text-white font-medium ${
!videoFile || selectedFrames.length === 0 || isProcessing
? 'bg-gray-400 cursor-not-allowed'
: 'bg-blue-500 hover:bg-blue-600 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500'
}`}
>
{isProcessing ? 'Generating Transcription...' : 'Generate Transcription'}
</button>
</div>
);
};