From fe323e38b53f1320f0312bb7741d1b94d22021b3 Mon Sep 17 00:00:00 2001 From: TheMaddax Date: Sat, 25 Jan 2025 18:19:15 -0600 Subject: [PATCH] Add gemini capabilities --- package.json | 1 + query_gemini_api.py | 80 +++++ src-tauri/Cargo.lock | 553 ++++++++++++++++++++++++++++-- src-tauri/Cargo.toml | 4 + src-tauri/src/lib.rs | 281 ++++++++++++++- src-tauri/src/main.rs | 52 ++- src/App.tsx | 126 ++++++- src/components/GeminiSettings.tsx | 178 ++++++++++ src/components/VideoTimeline.tsx | 95 +++++ 9 files changed, 1324 insertions(+), 46 deletions(-) create mode 100755 query_gemini_api.py create mode 100644 src/components/GeminiSettings.tsx create mode 100644 src/components/VideoTimeline.tsx diff --git a/package.json b/package.json index 098e26a..d950efa 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dependencies": { "@tauri-apps/api": "^2.1.1", "@tauri-apps/plugin-dialog": "2", + "@tauri-apps/plugin-fs": "2", "@tauri-apps/plugin-opener": "^2", "react": "^18.3.1", "react-dom": "^18.3.1" diff --git a/query_gemini_api.py b/query_gemini_api.py new file mode 100755 index 0000000..c5d9ef6 --- /dev/null +++ b/query_gemini_api.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +import google.generativeai as genai +import sys +import json +import base64 + +def log(msg): + """Print to stderr for logging""" + print(msg, file=sys.stderr, flush=True) + +def query_gemini(api_key, prompt, image_path=None): + log("\n=== Gemini API Request ===") + log(f"API Key: {api_key[:4]}...{api_key[-4:]}") # Show only first/last 4 chars + log(f"Prompt:\n{prompt}") + + # Configure the Gemini API + genai.configure(api_key=api_key) + model = genai.GenerativeModel('gemini-2.0-flash-exp') + + try: + if image_path: + log(f"\nProcessing image: {image_path}") + with open(image_path, 'rb') as f: + image_data = f.read() + log(f"Image size: {len(image_data):,} bytes") + + image_parts = [ + { + "mime_type": "image/jpeg", + "data": base64.b64encode(image_data).decode('utf-8') + } + ] + response = model.generate_content([prompt, image_parts[0]]) + else: + log("\nProcessing text-only request") + response = model.generate_content(prompt) + + log("\n=== Gemini API Response ===") + log(f"Response:\n{response.text}") + + # Create the JSON response + result = { + "success": True, + "text": response.text + } + + # Log the result to stderr + log("\n=== JSON Response ===") + log(json.dumps(result, indent=2)) + + # Return the compact JSON to stdout without any extra newlines + return json.dumps(result, separators=(',', ':')) + except Exception as e: + log("\n=== Gemini API Error ===") + log(f"Error: {str(e)}") + + error_result = { + "success": False, + "error": str(e) + } + + # Log the error to stderr + log("\n=== JSON Error Response ===") + log(json.dumps(error_result, indent=2)) + + # Return the compact JSON to stdout without any extra newlines + return json.dumps(error_result, separators=(',', ':')) + +if __name__ == "__main__": + if len(sys.argv) < 3: + log("Usage: query_gemini_api.py [image_path]") + sys.exit(1) + + api_key = sys.argv[1] + prompt = sys.argv[2] + image_path = sys.argv[3] if len(sys.argv) > 3 else None + + # Get the result and print it to stdout without any extra newlines + result = query_gemini(api_key, prompt, image_path) + print(result, end='', flush=True) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index b7137d0..8b5f00b 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -264,6 +264,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -501,9 +507,9 @@ dependencies = [ "bitflags 2.6.0", "block", "cocoa-foundation", - "core-foundation", + "core-foundation 0.10.0", "core-graphics", - "foreign-types", + "foreign-types 0.5.0", "libc", "objc", ] @@ -516,12 +522,18 @@ checksum = "e14045fb83be07b5acf1c0884b2180461635b433455fa35d1cd6f17f1450679d" dependencies = [ "bitflags 2.6.0", "block", - "core-foundation", + "core-foundation 0.10.0", "core-graphics-types", "libc", "objc", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.7" @@ -557,6 +569,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.0" @@ -580,9 +602,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.10.0", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -593,7 +615,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.10.0", "libc", ] @@ -624,12 +646,37 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + [[package]] name = "crypto-common" version = "0.1.6" @@ -868,6 +915,12 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "embed-resource" version = "2.5.1" @@ -879,7 +932,7 @@ dependencies = [ "rustc_version", "toml 0.8.2", "vswhom", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -947,7 +1000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -971,6 +1024,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1012,6 +1080,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -1019,7 +1096,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -1033,6 +1110,12 @@ dependencies = [ "syn 2.0.91", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -1283,6 +1366,16 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1437,6 +1530,35 @@ dependencies = [ "syn 2.0.91", ] +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.7.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1487,6 +1609,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.14", +] + [[package]] name = "http" version = "1.2.0" @@ -1498,6 +1631,17 @@ dependencies = [ "itoa 1.0.14", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1505,7 +1649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.2.0", ] [[package]] @@ -1516,8 +1660,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1527,6 +1671,36 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa 1.0.14", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.5.2" @@ -1536,8 +1710,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "httparse", "itoa 1.0.14", "pin-project-lite", @@ -1546,6 +1720,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.32", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "hyper-util" version = "0.1.10" @@ -1555,9 +1742,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -1743,6 +1930,24 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "jpeg-decoder", + "num-traits", + "png", + "qoi", + "tiff", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1865,6 +2070,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.76" @@ -1927,6 +2141,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libappindicator" version = "0.9.0" @@ -2061,6 +2281,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.8.2" @@ -2102,6 +2332,23 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndk" version = "0.9.0" @@ -2446,6 +2693,50 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.91", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -2822,6 +3113,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + [[package]] name = "quick-xml" version = "0.32.0" @@ -2936,6 +3236,26 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.8" @@ -2985,6 +3305,47 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.50.0", +] + [[package]] name = "reqwest" version = "0.12.9" @@ -2995,10 +3356,10 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.5.2", "hyper-util", "ipnet", "js-sys", @@ -3010,7 +3371,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tokio-util", "tower-service", @@ -3070,7 +3431,16 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", ] [[package]] @@ -3088,6 +3458,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "schemars" version = "0.8.21" @@ -3127,6 +3506,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.22.0" @@ -3397,7 +3799,7 @@ dependencies = [ "bytemuck", "cfg_aliases", "core-graphics", - "foreign-types", + "foreign-types 0.5.0", "js-sys", "log", "objc2", @@ -3485,6 +3887,9 @@ name = "subtitle-merge" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.21.7", + "image", + "reqwest 0.11.27", "serde", "serde_json", "tauri", @@ -3492,6 +3897,7 @@ dependencies = [ "tauri-plugin-dialog", "tauri-plugin-fs", "tauri-plugin-shell", + "tempfile", "thiserror 1.0.69", "tokio", ] @@ -3529,6 +3935,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -3549,6 +3961,27 @@ dependencies = [ "syn 2.0.91", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-deps" version = "6.2.2" @@ -3570,7 +4003,7 @@ checksum = "6682a07cf5bab0b8a2bd20d0a542917ab928b5edb75ebd4eda6b05cbaab872da" dependencies = [ "bitflags 2.6.0", "cocoa", - "core-foundation", + "core-foundation 0.10.0", "core-graphics", "crossbeam-channel", "dispatch", @@ -3634,7 +4067,7 @@ dependencies = [ "glob", "gtk", "heck 0.5.0", - "http", + "http 1.2.0", "jni", "libc", "log", @@ -3646,7 +4079,7 @@ dependencies = [ "percent-encoding", "plist", "raw-window-handle", - "reqwest", + "reqwest 0.12.9", "serde", "serde_json", "serde_repr", @@ -3815,7 +4248,7 @@ checksum = "cce18d43f80d4aba3aa8a0c953bbe835f3d0f2370aca75e8dbb14bd4bab27958" dependencies = [ "dpi", "gtk", - "http", + "http 1.2.0", "jni", "raw-window-handle", "serde", @@ -3833,7 +4266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f442a38863e10129ffe2cec7bd09c2dcf8a098a3a27801a476a304d5bb991d2" dependencies = [ "gtk", - "http", + "http 1.2.0", "jni", "log", "objc2", @@ -3864,7 +4297,7 @@ dependencies = [ "dunce", "glob", "html5ever", - "http", + "http 1.2.0", "infer", "json-patch", "kuchikiki", @@ -3909,7 +4342,7 @@ dependencies = [ "fastrand", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3969,6 +4402,17 @@ dependencies = [ "syn 2.0.91", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.37" @@ -4040,6 +4484,16 @@ dependencies = [ "syn 2.0.91", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.13" @@ -4240,6 +4694,12 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.14" @@ -4304,6 +4764,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.2.0" @@ -4597,6 +5063,12 @@ dependencies = [ "windows-core 0.58.0", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "winapi" version = "0.3.9" @@ -4958,6 +5430,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "winreg" version = "0.52.0" @@ -4995,7 +5477,7 @@ dependencies = [ "gdkx11", "gtk", "html5ever", - "http", + "http 1.2.0", "javascriptcore-rs", "jni", "kuchikiki", @@ -5201,6 +5683,15 @@ dependencies = [ "syn 2.0.91", ] +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + [[package]] name = "zvariant" version = "4.0.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 3f6c177..1b2d193 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -21,6 +21,10 @@ thiserror = "1.0" tauri-plugin-shell = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v2" } tauri-plugin-dialog = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v2" } tauri-plugin-fs = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v2" } +reqwest = { version = "0.11", features = ["json", "multipart"] } +base64 = "0.21" +image = "0.24" +tempfile = "3.9" [features] custom-protocol = ["tauri/custom-protocol"] diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 85b526a..b3d6e74 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -3,6 +3,8 @@ use std::process::{Command, Stdio}; use std::sync::Mutex; use std::io::BufRead; use tauri::{Runtime, State, Emitter}; +use tempfile::TempDir; +use std::fs; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -10,6 +12,12 @@ pub enum Error { FFmpegError(String), #[error("IO error: {0}")] IoError(#[from] std::io::Error), + #[error("Gemini API error: {0}")] + GeminiError(String), + #[error("Image processing error: {0}")] + ImageError(String), + #[error("HTTP error: {0}")] + HttpError(#[from] reqwest::Error), } impl serde::Serialize for Error { @@ -32,6 +40,74 @@ pub struct MergeArgs { pub resize_to_720p: bool, } +#[derive(Debug, Deserialize)] +pub struct TranscriptionArgs { + pub video_path: String, + pub subtitle_path: String, + pub frame_time: u32, + pub api_key: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TranscriptionResult { + pub visual_description: String, + pub subtitle_narrative: String, +} + +#[derive(Debug, Serialize)] +pub struct Thumbnail { + pub path: String, + pub time: u32, +} + +#[derive(Debug, Serialize, Deserialize)] +struct GeminiRequest { + contents: GeminiContent, +} + +#[derive(Debug, Serialize, Deserialize)] +struct GeminiContent { + parts: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct GeminiPart { + #[serde(skip_serializing_if = "Option::is_none")] + text: Option, + #[serde(skip_serializing_if = "Option::is_none")] + inline_data: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct ImageData { + mime_type: String, + data: String, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum GeminiResponse { + Success { + candidates: Vec, + }, + Error { + error: GeminiError, + }, +} + +#[derive(Debug, Deserialize)] +struct GeminiError { + code: i32, + message: String, + status: String, +} + +#[derive(Debug, Deserialize)] +struct GeminiCandidate { + content: GeminiContent, +} + #[derive(Debug, Serialize, Clone)] pub struct ProgressUpdate { pub progress: f32, @@ -54,7 +130,7 @@ pub async fn check_ffmpeg() -> Result<(), Error> { Ok(()) } -async fn get_video_duration(video_path: &str) -> Result { +pub async fn get_video_duration(video_path: &str) -> Result { let output = Command::new("/opt/homebrew/bin/ffprobe") .args([ "-v", "error", @@ -75,6 +151,200 @@ async fn get_video_duration(video_path: &str) -> Result { .map_err(|e| Error::FFmpegError(format!("Failed to parse duration: {}", e))) } +async fn extract_frame(video_path: &str, time: u32, output_path: &str) -> Result<(), Error> { + let output = Command::new("/opt/homebrew/bin/ffmpeg") + .args([ + "-ss", &time.to_string(), + "-i", video_path, + "-vframes", "1", + "-q:v", "2", + "-y", + output_path + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .map_err(|e| Error::FFmpegError(format!("Failed to execute FFmpeg: {}", e)))?; + + if !output.status.success() { + let error = String::from_utf8_lossy(&output.stderr); + return Err(Error::FFmpegError(format!("Frame extraction failed: {}", error))); + } + + if !std::path::Path::new(output_path).exists() { + return Err(Error::FFmpegError(format!("Output file was not created: {}", output_path))); + } + + Ok(()) +} + +async fn query_gemini(api_key: &str, prompt: &str, image_path: Option<&str>) -> Result { + println!("\n=== Gemini API Request ==="); + println!("Prompt: {}", prompt); + if let Some(path) = image_path { + println!("Image Path: {}", path); + } + + let mut command = Command::new("../.venv/bin/python3"); + command.arg("../query_gemini_api.py") + .arg(api_key) + .arg(prompt); + + if let Some(path) = image_path { + command.arg(path); + } + + println!("\nExecuting command: {:?}", command); + let output = command.output() + .map_err(|e| Error::GeminiError(format!("Failed to execute Python script: {}", e)))?; + + if !output.status.success() { + let error = String::from_utf8_lossy(&output.stderr); + println!("\n=== Gemini API Error ==="); + println!("Error: {}", error); + return Err(Error::GeminiError(format!("Python script error: {}", error))); + } + + let response_text = String::from_utf8_lossy(&output.stdout); + + // Try to parse the response as JSON + match serde_json::from_str::(&response_text) { + Ok(response) => { + if response["success"].as_bool().unwrap_or(false) { + let result = response["text"].as_str().unwrap_or("").to_string(); + println!("\n=== Gemini API Success ==="); + println!("Response length: {} characters", result.len()); + Ok(result) + } else { + let error = response["error"].as_str().unwrap_or("Unknown error").to_string(); + println!("\n=== Gemini API Error ==="); + println!("Error: {}", error); + Err(Error::GeminiError(error)) + } + }, + Err(e) => { + println!("\n=== Gemini API Parse Error ==="); + println!("Raw output: {}", response_text); + println!("Parse error: {}", e); + Err(Error::GeminiError(format!("Failed to parse response: {}", e))) + } + } +} + +pub async fn test_gemini_api_impl(api_key: &str) -> Result<(), Error> { + query_gemini(api_key, "Test connection", None).await?; + Ok(()) +} + +pub async fn extract_thumbnails_impl(video_path: &str, interval: u32) -> Result, Error> { + // Create a directory in the system temp directory + let temp_dir = std::env::temp_dir().join("video_thumbs"); + let thumb_dir = temp_dir.join(format!("session_{}", std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs())); + + std::fs::create_dir_all(&thumb_dir) + .map_err(|e| Error::IoError(e))?; + + println!("Extracting thumbnails to: {:?}", thumb_dir); + + let duration = get_video_duration(video_path).await? as u32; + let mut thumbnails = Vec::new(); + + for time in (0..duration).step_by(interval as usize) { + let output_path = thumb_dir.join(format!("thumb_{}.jpg", time)); + match extract_frame(video_path, time, output_path.to_str().unwrap()).await { + Ok(_) => { + println!("Successfully extracted frame at time {}", time); + thumbnails.push(Thumbnail { + path: output_path.to_str().unwrap().to_string(), + time, + }); + }, + Err(e) => { + println!("Failed to extract frame at time {}: {}", time, e); + continue; + } + } + } + + if thumbnails.is_empty() { + return Err(Error::FFmpegError("Failed to extract any thumbnails".to_string())); + } + + Ok(thumbnails) +} + +pub async fn process_transcription_impl(args: TranscriptionArgs) -> Result { + println!("\n=== Starting Transcription Process ==="); + println!("Video Path: {}", args.video_path); + println!("Subtitle Path: {}", args.subtitle_path); + println!("Frame Time: {}", args.frame_time); + + let temp_dir = TempDir::new().map_err(|e| Error::IoError(e))?; + let frame_path = temp_dir.path().join("frame.jpg"); + println!("\nExtracting frame to: {:?}", frame_path); + extract_frame(&args.video_path, args.frame_time, frame_path.to_str().unwrap()).await?; + + println!("\nReading subtitle content..."); + let subtitle_content = fs::read_to_string(&args.subtitle_path) + .map_err(|e| Error::IoError(e))?; + println!("Subtitle content length: {} characters", subtitle_content.len()); + + let visual_prompt = "Analyze this video frame and provide a visual description focusing on: +1. The visual setup and environment +2. The people present, their appearance, and positioning +3. Any relevant visual context or background details + +Format the description in clear, concise paragraphs that would be helpful for DeafBlind readers to understand the visual context. Focus on spatial relationships and important visual details that contribute to understanding the scene. Use maximum of 200 words."; + + let subtitle_prompt = format!("Convert the following subtitle content into a natural, flowing narrative that includes: +1. Speaker identification when there are multiple speakers +2. Speaking manner and tone where relevant (e.g., \"warmly\", \"enthusiastically\") +3. Clear paragraph breaks between different speakers or topics +4. Natural transition words to connect dialogue +5. Integration of any important non-verbal cues from the original subtitles + +Subtitle content: +{} + +Format the text as a professional transcript, maintaining chronological flow while making it read naturally.", subtitle_content); + + let (visual_description, subtitle_narrative) = tokio::join!( + query_gemini(&args.api_key, visual_prompt, Some(frame_path.to_str().unwrap())), + query_gemini(&args.api_key, &subtitle_prompt, None) + ); + + // Get the results, handling any errors + let visual_description = visual_description?; + let subtitle_narrative = subtitle_narrative?; + + println!("\n=== Final Transcription Result ==="); + println!("\nVisual Description:"); + println!("{}", visual_description); + println!("\nSubtitle Narrative:"); + println!("{}", subtitle_narrative); + + // Create the result struct + let result = TranscriptionResult { + visual_description: visual_description.clone(), + subtitle_narrative: subtitle_narrative.clone(), + }; + + // Log the final result structure + println!("\nReturning transcription result with:"); + println!("- Visual description length: {}", result.visual_description.len()); + println!("- Subtitle narrative length: {}", result.subtitle_narrative.len()); + println!("\nFinal TranscriptionResult struct:"); + println!("{{"); + println!(" \"visualDescription\": \"{}\",", result.visual_description); + println!(" \"subtitleNarrative\": \"{}\"", result.subtitle_narrative); + println!("}}"); + + Ok(result) +} + pub async fn merge_video_subtitle_impl( args: MergeArgs, progress_state: State<'_, ProgressState>, @@ -82,11 +352,9 @@ pub async fn merge_video_subtitle_impl( ) -> Result { let output_path = format!("{}-subbed.mp4", args.video_path.trim_end_matches(".mp4")); - // Get video duration first let total_duration = get_video_duration(&args.video_path).await?; println!("Video duration: {} seconds", total_duration); - // Build filter chain with proper escaping and joining let mut filters = Vec::new(); if args.add_black_bar { filters.push("pad=iw:ih+170:0:0:black".to_string()); @@ -95,7 +363,6 @@ pub async fn merge_video_subtitle_impl( filters.push("scale=-1:720".to_string()); } - // Escape special characters in the subtitle path let subtitle_path = args.subtitle_path.replace("'", "'\\''").replace(",", "\\,").replace(" ", "\\ "); filters.push(format!("subtitles='{}'", subtitle_path)); filters.push("format=nv12|qsv".to_string()); @@ -103,7 +370,6 @@ pub async fn merge_video_subtitle_impl( let filter_chain = filters.join(","); println!("Filter chain: {}", filter_chain); - // Verify FFmpeg is installed check_ffmpeg().await?; println!("Starting FFmpeg process with paths: video={}, subtitle={}", args.video_path, args.subtitle_path); @@ -137,7 +403,6 @@ pub async fn merge_video_subtitle_impl( let mut reader = std::io::BufReader::new(stdout); let mut line = String::new(); - // Read progress and update state using pre-probed duration let mut last_progress = 0.0; while let Ok(bytes) = reader.read_line(&mut line) { if bytes == 0 { @@ -146,7 +411,6 @@ pub async fn merge_video_subtitle_impl( if line.starts_with("out_time=") { if let Some(time_str) = line.split('=').nth(1) { - // Parse time in format HH:MM:SS.ms let parts: Vec<&str> = time_str.trim().split(':').collect(); if parts.len() == 3 { if let (Ok(hours), Ok(minutes), Ok(seconds)) = ( @@ -157,10 +421,8 @@ pub async fn merge_video_subtitle_impl( let current_seconds = hours * 3600.0 + minutes * 60.0 + seconds; let progress = ((current_seconds / total_duration) * 100.0).min(100.0).max(0.0); - // Only update if progress has changed significantly (avoid spam) if progress - last_progress >= 1.0 || progress == 100.0 { *progress_state.0.lock().unwrap() = progress; - // Emit progress event to frontend if let Err(e) = app_handle.emit("progress", ProgressUpdate { progress }) { println!("Failed to emit progress: {}", e); } @@ -182,7 +444,6 @@ pub async fn merge_video_subtitle_impl( } println!("FFmpeg process completed successfully"); - // Open the output file in Finder let _ = Command::new("open") .arg("-R") .arg(&output_path) diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index 0a3b6da..cc0d54d 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -1,7 +1,13 @@ // Prevents additional console window on Windows in release #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] -use subtitle_merge_lib::{ProgressState, MergeArgs, Error, merge_video_subtitle_impl, check_ffmpeg}; +use subtitle_merge_lib::{ + ProgressState, MergeArgs, Error, merge_video_subtitle_impl, check_ffmpeg, + test_gemini_api_impl, extract_thumbnails_impl, process_transcription_impl, + TranscriptionArgs, get_video_duration +}; +use std::fs; +use base64::prelude::*; use tauri::Runtime; #[tauri::command] @@ -23,6 +29,38 @@ async fn merge_video_subtitle( merge_video_subtitle_impl(args, progress_state, app_handle).await } +#[tauri::command] +async fn test_gemini_api(api_key: String) -> Result<(), Error> { + test_gemini_api_impl(&api_key).await +} + +#[tauri::command] +async fn get_video_duration_command(path: String) -> Result { + get_video_duration(&path).await +} + +#[tauri::command] +async fn extract_thumbnails(path: String, interval: u32) -> Result, Error> { + extract_thumbnails_impl(&path, interval).await +} + +#[tauri::command] +async fn process_transcription(args: TranscriptionArgs) -> Result { + process_transcription_impl(args).await +} + +#[tauri::command] +async fn generate_thumbnails(video_path: String) -> Result, Error> { + // Generate thumbnails every 10 seconds + extract_thumbnails_impl(&video_path, 10).await +} + +#[tauri::command] +async fn read_thumbnail(path: String) -> Result { + let data = fs::read(&path).map_err(|e| Error::IoError(e))?; + Ok(BASE64_STANDARD.encode(&data)) +} + fn main() { tauri::Builder::default() .plugin(tauri_plugin_shell::init()) @@ -32,7 +70,17 @@ fn main() { .setup(|_app| { Ok(()) }) - .invoke_handler(tauri::generate_handler![merge_video_subtitle, check_dependencies, get_progress]) + .invoke_handler(tauri::generate_handler![ + merge_video_subtitle, + check_dependencies, + get_progress, + test_gemini_api, + get_video_duration_command, + extract_thumbnails, + process_transcription, + generate_thumbnails, + read_thumbnail + ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } diff --git a/src/App.tsx b/src/App.tsx index c84441a..9335cc0 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,6 +1,8 @@ import { useState, useEffect } from 'react'; import { invoke } from '@tauri-apps/api/core'; import { FileSelector } from './components/FileSelector'; +import { GeminiSettings, GeminiConfig } from './components/GeminiSettings'; +import { VideoTimeline } from './components/VideoTimeline'; import './styles/index.css'; interface FileState { @@ -8,6 +10,17 @@ interface FileState { name: string; } +// Must match the Rust TranscriptionResult struct exactly +interface TranscriptionState { + visualDescription: string; // from visual_description in Rust + subtitleNarrative: string; // from subtitle_narrative in Rust +} + +interface Thumbnail { + path: string; + time: number; +} + function App() { const [videoFile, setVideoFile] = useState(null); const [subtitleFile, setSubtitleFile] = useState(null); @@ -16,6 +29,14 @@ function App() { const [error, setError] = useState(null); const [addBlackBar, setAddBlackBar] = useState(true); const [resizeTo720p, setResizeTo720p] = useState(true); + const [geminiConfig, setGeminiConfig] = useState({ + enabled: false, + apiKey: 'AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc', + isDefault: true + }); + const [selectedTime, setSelectedTime] = useState(10); + const [transcription, setTranscription] = useState(null); + const [thumbnails, setThumbnails] = useState([]); useEffect(() => { invoke('check_dependencies') @@ -28,6 +49,7 @@ function App() { const handleVideoSelect = (path: string) => { const name = path.split('/').pop() || ''; setVideoFile({ path, name }); + setThumbnails([]); // Clear thumbnails when new video is selected }; const handleSubtitleSelect = (path: string) => { @@ -35,6 +57,10 @@ function App() { setSubtitleFile({ path, name }); }; + const handleThumbnailsGenerated = (newThumbnails: Thumbnail[]) => { + setThumbnails(newThumbnails); + }; + useEffect(() => { let intervalId: number | undefined; @@ -63,16 +89,76 @@ function App() { setIsProcessing(true); setProgress(0); + setTranscription(null); try { console.log('Starting merge with args:', { videoPath: videoFile.path, subtitlePath: subtitleFile.path, addBlackBar, - resizeTo720p + resizeTo720p, + geminiEnabled: geminiConfig.enabled, + frameTime: selectedTime }); - const result = await invoke('merge_video_subtitle', { + // Process transcription first if Gemini is enabled + if (geminiConfig.enabled) { + try { + console.log('Starting transcription process...'); + const transcriptionResult = await invoke('process_transcription', { + args: { + video_path: videoFile.path, + subtitle_path: subtitleFile.path, + frame_time: selectedTime, + api_key: geminiConfig.apiKey + } + }); + console.log('Transcription completed with full response:', transcriptionResult); + + // Log the raw response first + console.log('Raw transcription result:', JSON.stringify(transcriptionResult, null, 2)); + + // Validate the response structure + if (!transcriptionResult || typeof transcriptionResult !== 'object') { + console.error('Invalid transcription result:', transcriptionResult); + setError('Invalid transcription response format'); + return; + } + + // Validate and clean the content + const visualDescription = String(transcriptionResult.visualDescription || '').trim(); + const subtitleNarrative = String(transcriptionResult.subtitleNarrative || '').trim(); + + if (!visualDescription || !subtitleNarrative) { + console.error('Missing transcription content:', { + visualDescription: !!visualDescription, + subtitleNarrative: !!subtitleNarrative + }); + setError('Transcription response is missing required content'); + return; + } + + // Create a new state object with the validated content + const newTranscription: TranscriptionState = { + visualDescription, + subtitleNarrative + }; + + console.log('Setting transcription state:', { + visualDescriptionLength: visualDescription.length, + subtitleNarrativeLength: subtitleNarrative.length + }); + + setTranscription(newTranscription); + } catch (error) { + console.error('Transcription error:', error); + setError(`Transcription error: ${error}`); + return; + } + } + + // Then start video merging process + const mergeResult = await invoke('merge_video_subtitle', { args: { video_path: videoFile.path, subtitle_path: subtitleFile.path, @@ -81,7 +167,7 @@ function App() { } }); - console.log('Merge completed:', result); + console.log('Merge completed:', mergeResult); setProgress(100); } catch (error) { console.error('Error merging files:', error); @@ -121,6 +207,20 @@ function App() { /> + + + {geminiConfig.enabled && videoFile && ( + + )} +
+ {transcription ? ( +
+

DeafBlind Accessibility Transcription

+
+
+

Visual Description

+
+

{transcription.visualDescription}

+
+
+
+

Subtitle Narrative

+
+

{transcription.subtitleNarrative}

+
+
+
+
+ ) : null} +
void; + videoFile: string | null; + onThumbnailsGenerated: (thumbnails: { path: string; time: number; }[]) => void; +} + +export interface GeminiConfig { + enabled: boolean; + apiKey: string; + isDefault: boolean; +} + +const DEFAULT_API_KEY = 'AIzaSyAF825tPTh77oL0knsGFEyvsN0iPUO_bXc'; + +export const GeminiSettings: React.FC = ({ onSettingsChange, videoFile, onThumbnailsGenerated }) => { + const [settings, setSettings] = useState({ + enabled: false, + apiKey: DEFAULT_API_KEY, + isDefault: true + }); + const [testing, setTesting] = useState(false); + const [testResult, setTestResult] = useState(null); + const [generatingThumbnails, setGeneratingThumbnails] = useState(false); + const [thumbnailResult, setThumbnailResult] = useState(null); + + const handleEnableChange = (e: React.ChangeEvent) => { + const newSettings = { + ...settings, + enabled: e.target.checked + }; + setSettings(newSettings); + onSettingsChange(newSettings); + }; + + const handleApiKeyChange = (e: React.ChangeEvent) => { + const newSettings = { + ...settings, + apiKey: e.target.value, + isDefault: e.target.value === DEFAULT_API_KEY + }; + setSettings(newSettings); + onSettingsChange(newSettings); + }; + + const resetToDefault = () => { + const newSettings = { + ...settings, + apiKey: DEFAULT_API_KEY, + isDefault: true + }; + setSettings(newSettings); + onSettingsChange(newSettings); + }; + + const testApiKey = useCallback(async () => { + setTesting(true); + setTestResult(null); + try { + // We'll implement this Rust command later + await invoke('test_gemini_api', { apiKey: settings.apiKey }); + setTestResult('API key is valid'); + } catch (error) { + setTestResult(`Error: ${error}`); + } finally { + setTesting(false); + } + }, [settings.apiKey]); + + const generateThumbnails = useCallback(async () => { + if (!videoFile) { + setThumbnailResult("Please select a video file before generating thumbnails"); + return; + } + + setGeneratingThumbnails(true); + setThumbnailResult(null); + try { + const thumbnails = await invoke<{ path: string; time: number; }[]>('generate_thumbnails', { videoPath: videoFile }); + onThumbnailsGenerated(thumbnails); + setThumbnailResult("Thumbnails generated successfully!"); + } catch (error) { + setThumbnailResult(`Error: ${error}`); + } finally { + setGeneratingThumbnails(false); + } + }, [videoFile, onThumbnailsGenerated]); + + return ( +
+
+ + +
+ + {settings.enabled && ( +
+
+ +
+ +
+
+ +
+ + + {!settings.isDefault && ( + + )} +
+ + {testResult && ( +
+ {testResult} +
+ )} + +
+ +
+ + {thumbnailResult && ( +
+ {thumbnailResult} +
+ )} +
+ )} +
+ ); +}; diff --git a/src/components/VideoTimeline.tsx b/src/components/VideoTimeline.tsx new file mode 100644 index 0000000..1f60fe8 --- /dev/null +++ b/src/components/VideoTimeline.tsx @@ -0,0 +1,95 @@ +import React, { useState, useEffect } from 'react'; +import { invoke } from '@tauri-apps/api/core'; + +interface VideoTimelineProps { + videoPath: string | null; + onTimeSelect: (time: number) => void; + thumbnails: Thumbnail[]; +} + +interface Thumbnail { + path: string; + time: number; +} + +export const VideoTimeline: React.FC = ({ videoPath, onTimeSelect, thumbnails }) => { + const [selectedTime, setSelectedTime] = useState(10); // Default 10 seconds + const [duration, setDuration] = useState(0); + const [thumbnailUrls, setThumbnailUrls] = useState<{ [key: string]: string }>({}); + + useEffect(() => { + if (videoPath) { + invoke('get_video_duration', { path: videoPath }) + .then(videoDuration => setDuration(videoDuration)) + .catch(error => console.error('Error getting video duration:', error)); + } else { + setDuration(0); + } + }, [videoPath]); + + useEffect(() => { + const loadThumbnails = async () => { + const urls: { [key: string]: string } = {}; + for (const thumb of thumbnails) { + try { + const base64 = await invoke('read_thumbnail', { path: thumb.path }); + urls[thumb.path] = `data:image/jpeg;base64,${base64}`; + } catch (error) { + console.error('Error loading thumbnail:', error); + } + } + setThumbnailUrls(urls); + }; + + loadThumbnails(); + }, [thumbnails]); + + const formatTime = (seconds: number): string => { + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + return `${minutes}:${remainingSeconds.toString().padStart(2, '0')}`; + }; + + if (!videoPath) { + return null; + } + + return ( +
+
+ + + Selected: {formatTime(selectedTime)} + +
+ +
+ {thumbnails.map((thumb, index) => ( +
{ + setSelectedTime(thumb.time); + onTimeSelect(thumb.time); + }} + > + {thumbnailUrls[thumb.path] && ( + {`Thumbnail + )} +
+ {formatTime(thumb.time)} +
+
+ ))} +
+
+ ); +};