Add Image Description

Dockerfile

@@ -45,7 +45,7 @@ RUN pip install --no-cache-dir ocrmypdf==14.4.0 pdf2image==1.16.3
 
 # Install AI-related dependencies
 RUN pip install --no-cache-dir \
-    google-genai==0.2.0 \
+    google-generativeai==0.3.1 \
     openai==1.3.0 \
     mistralai==0.0.7 \
     beautifulsoup4==4.12.2

backend/app.py

@@ -70,6 +70,40 @@ def run_translation(job_id, file_path, api, model, target_language, prompt_key):
         update_progress(job_id, 100, "🎉 Process completed")
     except Exception as e:
         update_progress(job_id, active_jobs[job_id]["progress"], f"❌ Error: {str(e)}")
+        
+def run_image_description(job_id, file_path, api, model, prompt_key):
+    try:
+        update_progress(job_id, 25, "🖼️ Image uploaded, starting description")
+        
+        # Import function here since we're adding it to utils.py
+        from utils import describe_image
+        
+        print(f"DEBUG APP: Starting describe_image with file_path={file_path}, api={api}, model={model}, prompt_key={prompt_key}")
+        
+        result = describe_image(
+            file_path, api, model, prompt_key,
+            update_progress=lambda prog, stat: update_progress(job_id, prog, stat),
+            is_cancelled=lambda: is_cancelled(job_id)
+        )
+        
+        print(f"DEBUG APP: describe_image returned result of length: {len(result) if result else 0}")
+        
+        # Save the description result to a text file
+        base_name = os.path.splitext(os.path.basename(file_path))[0]
+        desc_file = os.path.join(OUTPUT_FOLDER, base_name + "_description.txt")
+        with open(desc_file, "w", encoding="utf-8") as f:
+            f.write(result)
+            
+        active_jobs[job_id]["result"] = desc_file
+        update_progress(job_id, 100, "🎉 Description completed")
+    except Exception as e:
+        import traceback
+        error_detail = f"Error: {str(e)}\n"
+        error_detail += f"Exception type: {type(e).__name__}\n"
+        error_detail += f"Traceback: {traceback.format_exc()}"
+        print(f"DEBUG APP ERROR in run_image_description: {error_detail}")
+        
+        update_progress(job_id, active_jobs[job_id]["progress"], f"❌ Error: {str(e)}")
 
 # Login endpoint
 @app.route('/api/login', methods=['POST'])
@@ -151,6 +185,48 @@ def upload_file():
 
     return jsonify({"message": "File uploaded, processing started", "job_id": job_id})
 
+@app.route('/api/image-description', methods=['POST'])
+@login_required
+def describe_image():
+    if 'file' not in request.files:
+        return jsonify({"error": "No file found"}), 400
+
+    file = request.files['file']
+    api = request.form.get('api')
+    model = request.form.get('model')
+    # Use a default image description prompt if prompt_key is not provided or if it's 'image_desc' which was removed
+    prompt_key = request.form.get('prompt_key')
+    if not prompt_key or prompt_key == 'image_desc':
+        # Use a simple default description prompt
+        prompt_text = "Describe this image in detail."
+        # Store this temporarily for this request
+        from models import update_prompt
+        update_prompt("temp_image_desc", prompt_text)
+        prompt_key = "temp_image_desc"
+
+    if file.filename == '':
+        return jsonify({"error": "Empty filename"}), 400
+    
+    # Check if the file is an image
+    allowed_extensions = {'png', 'jpg', 'jpeg', 'gif', 'bmp'}
+    if not '.' in file.filename or file.filename.rsplit('.', 1)[1].lower() not in allowed_extensions:
+        return jsonify({"error": "File must be an image (PNG, JPG, JPEG, GIF, BMP)"}), 400
+
+    filename = secure_filename(file.filename)
+    if os.path.exists(os.path.join(UPLOAD_FOLDER, filename)):
+        base, ext = os.path.splitext(filename)
+        filename = f"{base}_{uuid.uuid4().hex}{ext}"
+    file_path = os.path.join(UPLOAD_FOLDER, filename)
+    file.save(file_path)
+
+    job_id = str(uuid.uuid4())
+    active_jobs[job_id] = {"progress": 0, "status": "📤 Image uploaded", "cancelled": False, "result": None}
+
+    thread = threading.Thread(target=run_image_description, args=(job_id, file_path, api, model, prompt_key))
+    thread.start()
+
+    return jsonify({"message": "Image uploaded, processing started", "job_id": job_id})
+
 @app.route('/api/progress/<job_id>', methods=['GET'])
 @login_required
 def get_progress(job_id):

backend/models.py

@@ -1,12 +1,16 @@
 # backend/models.py
 default_prompts = {
-    "ocr_correction": "Correct the following text. Begin your answer directly with the corrected text. Format the result as HTML, indicating where each page begins, each paragraph begins, and marking titles with the appropriate HTML tags (e.g., <h1>, <h2>, etc.).",
-    "ocr": "Perform OCR on the following document. Begin your answer directly with the OCR. Format the result as HTML, indicating where each page begins, each paragraph begins, and marking titles with the appropriate HTML tags (e.g., <h1>, <h2>, etc.).",
-    "translation": "Translate the following text to Spanish. Do not add any commentary; only output the translated text. Begin your answer directly with the translation. Format the result as HTML, indicating where each page begins, each paragraph begins, and marking titles with the appropriate HTML tags (e.g., <h1>, <h2>, etc.)."
+    "ocr_correction": "Correct the following text. Begin your answer directly with the corrected text. Format the result as MARKUP, indicating where each page begins, each paragraph begins, and marking titles with the appropriate MARKUP tags (e.g., #, ##, **bold for any text**, etc.).",
+    "ocr": "Perform OCR on the following document. Begin your answer directly with the OCR. Format the result as MARKUP, indicating where each page begins, each paragraph begins, and marking titles with the appropriate MARKUP tags (e.g., #, ##, **bold for any text** etc.).",
+    "translation": "Translate the following text to Spanish. Do not add any commentary; only output the translated text. Begin your answer directly with the translation. Format the result as MARKUP, indicating where each page begins, each paragraph begins, and marking titles with the appropriate MARKUP tags (e.g., #, ##, **bold for any text**, etc.)."
 }
 
 custom_prompts = {}
 
+# Remove image_desc from custom_prompts if it exists
+if "image_desc" in custom_prompts:
+    del custom_prompts["image_desc"]
+
 def get_prompt(key):
     return custom_prompts.get(key, default_prompts.get(key, ""))
 

backend/requirements.txt

@@ -3,10 +3,11 @@ Flask-Login==0.6.3
 flask-cors==4.0.0
 bcrypt==4.1.2
 pytesseract==0.3.10
-ocrmypdf==14.4.0
+ocrmypdf==16.10.0
 pdf2image==1.16.3
 Pillow==10.0.0
-google-genai==0.2.0
+google-generativeai==0.3.1
 openai==1.3.0
 mistralai==0.0.7
 beautifulsoup4==4.12.2
+pikepdf==9.5.2

backend/utils.py

@@ -50,48 +50,432 @@ def run_tesseract(file_path):
 def call_api_correction(api, model, text, prompt_key="ocr_correction"):
     prompt = get_prompt(prompt_key) + text
     try:
-        from google import genai
+        import google.generativeai as genai
     except ImportError:
-        raise ImportError("Please install 'google-genai' to use Gemini.")
+        raise ImportError("Please install 'google-generativeai' to use Gemini.")
     try:
         loop = asyncio.get_event_loop()
     except RuntimeError:
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-    client = genai.Client(api_key=os.environ.get("GEMINI_API_KEY"))
-    response = client.models.generate_content(model=model, contents=[prompt])
+    
+    # Get the API key
+    api_key = os.environ.get("GEMINI_API_KEY")
+    if not api_key:
+        return "❌ GEMINI_API_KEY not found in environment variables"
+    
+    # Configure API
+    genai.configure(api_key=api_key)
+    
+    # Create model instance
+    text_model = genai.GenerativeModel(model)
+    
+    # Generate content
+    response = text_model.generate_content(prompt)
+    
+    # Get text response
     return response.text
 
 def call_api_ocr(api, model, file_path, prompt_key="ocr"):
+    """
+    OCR processing using Gemini's vision capabilities.
+    Implements image optimization and retries with exponential backoff for handling timeouts.
+    """
     try:
-        from google import genai
+        import sys
+        import traceback
+        import mimetypes
+        import base64
+        import time
+        from PIL import Image
+        import google.generativeai as genai
     except ImportError:
-        raise ImportError("Please install 'google-genai' to use Gemini.")
+        raise ImportError("Please install 'google-generativeai' to use Gemini.")
+    
     try:
         loop = asyncio.get_event_loop()
     except RuntimeError:
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-    client = genai.Client(api_key=os.environ.get("GEMINI_API_KEY"))
-    file_ref = client.files.upload(file=file_path)
-    prompt = get_prompt(prompt_key)
-    response = client.models.generate_content(model=model, contents=[file_ref, prompt])
-    return response.text
+        
+    MAX_RETRIES = 3
+    RETRY_DELAY = 2  # Initial delay in seconds, will increase with each retry
+    
+    try:
+        # Debug information
+        print(f"DEBUG OCR: File path: {file_path}")
+        print(f"DEBUG OCR: File exists: {os.path.exists(file_path)}")
+        print(f"DEBUG OCR: File size: {os.path.getsize(file_path) if os.path.exists(file_path) else 'N/A'}")
+        
+        # Get the API key
+        api_key = os.environ.get("GEMINI_API_KEY")
+        if not api_key:
+            error_msg = "GEMINI_API_KEY not found in environment variables"
+            print(f"DEBUG OCR ERROR: {error_msg}")
+            return f"❌ {error_msg}"
+            
+        # Configure API
+        print(f"DEBUG OCR: Configuring API with key: {api_key[:4]}...{api_key[-4:] if len(api_key) > 8 else ''}")
+        genai.configure(api_key=api_key)
+        
+        # Open and optimize the image
+        try:
+            # Open the image
+            img = Image.open(file_path)
+            
+            # Check image size - resize if too large (max 1600px on any side)
+            max_size = 1600
+            original_size = img.size
+            if img.width > max_size or img.height > max_size:
+                # Calculate new size preserving aspect ratio
+                if img.width > img.height:
+                    new_width = max_size
+                    new_height = int(img.height * (max_size / img.width))
+                else:
+                    new_height = max_size
+                    new_width = int(img.width * (max_size / img.height))
+                
+                img = img.resize((new_width, new_height), Image.LANCZOS)
+                print(f"DEBUG OCR: Resized image from {original_size} to {img.size}")
+            
+            # Convert to RGB if needed (e.g., for PNG with transparency)
+            if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info):
+                background = Image.new('RGB', img.size, (255, 255, 255))
+                if img.mode == 'P':
+                    img = img.convert('RGBA')
+                background.paste(img, (0, 0), img)
+                img = background
+                print("DEBUG OCR: Converted image to RGB")
+            elif img.mode != 'RGB':
+                img = img.convert('RGB')
+                print("DEBUG OCR: Converted image to RGB")
+            
+            # Save to BytesIO and encode
+            import io
+            buffer = io.BytesIO()
+            img.save(buffer, format="JPEG", quality=85, optimize=True)
+            buffer.seek(0)
+            image_bytes = buffer.read()
+            file_base64 = base64.b64encode(image_bytes).decode('utf-8')
+            print(f"DEBUG OCR: Successfully processed and encoded image ({len(file_base64)} chars)")
+        except Exception as img_error:
+            error_msg = f"Error processing image: {str(img_error)}"
+            print(f"DEBUG OCR ERROR: {error_msg}")
+            print(traceback.format_exc())
+            return f"❌ {error_msg}"
+            
+        # Get prompt
+        prompt = get_prompt(prompt_key)
+        print(f"DEBUG OCR: Using prompt: {prompt[:50]}...")
+        
+        # Create model instance
+        vision_model = genai.GenerativeModel(model)
+        print(f"DEBUG OCR: Created model instance for model: {model}")
+        
+        # Create image data in the format required by the API
+        image_data = {
+            "mime_type": "image/jpeg",  # Always using JPEG for consistency
+            "data": file_base64
+        }
+        
+        print(f"DEBUG OCR: Image prepared for API. Generating content with retries...")
+        
+        # Initialize variables for retry logic
+        retry_count = 0
+        current_delay = RETRY_DELAY
+        last_error = None
+        
+        # Retry loop
+        while retry_count < MAX_RETRIES:
+            try:
+                if retry_count > 0:
+                    print(f"DEBUG OCR: Retry {retry_count}/{MAX_RETRIES} after {current_delay}s delay")
+                
+                # Generate content with the image and the prompt
+                response = vision_model.generate_content([
+                    {"inline_data": image_data},
+                    prompt
+                ], generation_config={
+                    "temperature": 0.2,
+                    "top_p": 0.95,
+                    "top_k": 40,
+                    "max_output_tokens": 1024,  # Limit output size to avoid timeouts
+                })
+                
+                print(f"DEBUG OCR: Response received successfully")
+                
+                # Return the text response
+                return response.text
+                
+            except Exception as api_error:
+                last_error = api_error
+                error_str = str(api_error)
+                print(f"DEBUG OCR ERROR on attempt {retry_count+1}: {error_str}")
+                
+                # Check if this is a timeout or rate limit error (worth retrying)
+                if "Deadline Exceeded" in error_str or "429" in error_str or "quota" in error_str.lower() or "timeout" in error_str.lower():
+                    retry_count += 1
+                    if retry_count < MAX_RETRIES:
+                        print(f"DEBUG OCR: Timeout or rate limit error, will retry in {current_delay}s")
+                        time.sleep(current_delay)
+                        current_delay *= 2  # Exponential backoff
+                    else:
+                        print("DEBUG OCR: Max retries reached")
+                else:
+                    # Non-retryable error
+                    print("DEBUG OCR: Non-retryable error, giving up")
+                    break
+        
+        # If we get here, all retries failed
+        error_detail = f"Error after {MAX_RETRIES} attempts: {str(last_error)}"
+        print(f"DEBUG OCR ERROR: {error_detail}")
+        return f"❌ Error in OCR: {error_detail}"
+        
+    except Exception as e:
+        error_detail = f"Error: {str(e)}\n"
+        error_detail += f"Exception type: {type(e).__name__}\n"
+        error_detail += f"Traceback: {traceback.format_exc()}"
+        print(f"DEBUG OCR ERROR: {error_detail}")
+        
+        return f"❌ Error in OCR: {str(e)}"
+
+def describe_image(file_path, api, model, prompt_key="image_desc", update_progress=None, is_cancelled=None):
+    """
+    Process an image file and generate a description using the specified AI model.
+    Uses direct base64 encoding for Gemini's vision capabilities.
+    Implements retries with exponential backoff for handling timeouts.
+    """
+    import sys
+    import traceback
+    import mimetypes
+    import base64
+    import time
+    from PIL import Image
+    
+    print(f"DEBUG IMAGE_DESC: Starting image description function for file: {file_path}")
+    print(f"DEBUG IMAGE_DESC: Using API: {api}, Model: {model}, Prompt key: {prompt_key}")
+    
+    MAX_RETRIES = 3
+    RETRY_DELAY = 2  # Initial delay in seconds, will increase with each retry
+    
+    try:
+        if update_progress:
+            update_progress(20, "⏳ Loading image file...")
+        
+        if is_cancelled and is_cancelled():
+            return "Process cancelled."
+        
+        # Check if file exists
+        if not os.path.exists(file_path):
+            error_msg = f"File not found: {file_path}"
+            print(f"DEBUG IMAGE_DESC: {error_msg}")
+            if update_progress:
+                update_progress(0, f"❌ {error_msg}")
+            return f"❌ {error_msg}"
+            
+        # Get MIME type of image
+        mime_type = mimetypes.guess_type(file_path)[0] or 'image/png'
+        print(f"DEBUG IMAGE_DESC: File path: {file_path}")
+        print(f"DEBUG IMAGE_DESC: MIME type: {mime_type}")
+        
+        # Read the image, resize if needed to reduce size, and encode it to base64
+        if update_progress:
+            update_progress(40, "🔍 Processing image...")
+            
+        try:
+            # Open and optimize the image
+            img = Image.open(file_path)
+            
+            # Check image size - resize if too large (max 1600px on any side)
+            max_size = 1600
+            original_size = img.size
+            if img.width > max_size or img.height > max_size:
+                # Calculate new size preserving aspect ratio
+                if img.width > img.height:
+                    new_width = max_size
+                    new_height = int(img.height * (max_size / img.width))
+                else:
+                    new_height = max_size
+                    new_width = int(img.width * (max_size / img.height))
+                
+                img = img.resize((new_width, new_height), Image.LANCZOS)
+                print(f"DEBUG IMAGE_DESC: Resized image from {original_size} to {img.size}")
+            
+            # Convert to RGB if needed (e.g., for PNG with transparency)
+            if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info):
+                # Create a white background image
+                background = Image.new('RGB', img.size, (255, 255, 255))
+                # Paste the image on the background (handles transparency)
+                if img.mode == 'P':
+                    img = img.convert('RGBA')
+                background.paste(img, (0, 0), img)
+                img = background
+                print("DEBUG IMAGE_DESC: Converted image to RGB")
+            elif img.mode != 'RGB':
+                img = img.convert('RGB')
+                print("DEBUG IMAGE_DESC: Converted image to RGB")
+            
+            # Save to BytesIO and encode
+            import io
+            buffer = io.BytesIO()
+            img.save(buffer, format="JPEG", quality=85, optimize=True)
+            buffer.seek(0)
+            image_bytes = buffer.read()
+            image_base64 = base64.b64encode(image_bytes).decode('utf-8')
+            print(f"DEBUG IMAGE_DESC: Successfully processed and encoded image ({len(image_base64)} chars)")
+        except Exception as e:
+            error_msg = f"Error processing image file: {str(e)}"
+            print(f"DEBUG IMAGE_DESC ERROR: {error_msg}")
+            print(traceback.format_exc())
+            if update_progress:
+                update_progress(0, f"❌ {error_msg}")
+            return f"❌ {error_msg}"
+        
+        # Import Gemini libraries - using the correct package name
+        try:
+            print("DEBUG IMAGE_DESC: Attempting to import google-generativeai library")
+            import google.generativeai as genai
+            print("DEBUG IMAGE_DESC: Successfully imported google.generativeai library")
+        except ImportError as ie:
+            error_msg = f"Failed to import Google's generativeai library: {str(ie)}. Please install it with: pip install google-generativeai"
+            print(f"DEBUG IMAGE_DESC ERROR: {error_msg}")
+            if update_progress:
+                update_progress(0, f"❌ {error_msg}")
+            return f"❌ {error_msg}"
+        
+        # Configure API 
+        api_key = os.environ.get("GEMINI_API_KEY")
+        if not api_key:
+            error_msg = "GEMINI_API_KEY not found in environment variables"
+            print(f"DEBUG IMAGE_DESC ERROR: {error_msg}")
+            if update_progress:
+                update_progress(0, f"❌ {error_msg}")
+            return f"❌ {error_msg}"
+        
+        print(f"DEBUG IMAGE_DESC: Configuring API with key: {api_key[:4]}...{api_key[-4:] if len(api_key) > 8 else ''}")
+        genai.configure(api_key=api_key)
+        
+        # Get prompt from configuration
+        prompt = get_prompt(prompt_key)
+        print(f"DEBUG IMAGE_DESC: Using prompt: {prompt[:50]}...")
+        
+        if update_progress:
+            update_progress(60, "🤖 Generating description...")
+        
+        # Generation parameters - reduce complexity for better performance
+        generation_config = {
+            "temperature": 0.2,  # Lower temperature for more deterministic output
+            "top_p": 0.95,
+            "top_k": 40,
+            "max_output_tokens": 800,  # Limit output size
+        }
+        
+        print(f"DEBUG IMAGE_DESC: Creating model instance for model: {model}")
+        # Create model instance
+        vision_model = genai.GenerativeModel(model)
+        
+        # Format image for the API
+        image_data = {
+            "mime_type": "image/jpeg",  # Always using JPEG for consistency
+            "data": image_base64
+        }
+        
+        print(f"DEBUG IMAGE_DESC: Image prepared for API. Generating content with retries...")
+        
+        # Initialize variables for retry logic
+        retry_count = 0
+        current_delay = RETRY_DELAY
+        last_error = None
+        
+        # Retry loop
+        while retry_count < MAX_RETRIES:
+            try:
+                if retry_count > 0:
+                    print(f"DEBUG IMAGE_DESC: Retry {retry_count}/{MAX_RETRIES} after {current_delay}s delay")
+                    if update_progress:
+                        update_progress(60, f"🔄 Retry {retry_count}/{MAX_RETRIES}...")
+                
+                # Generate content using the correct format for v0.3.1
+                response = vision_model.generate_content([
+                    {"inline_data": image_data},
+                    prompt
+                ], generation_config=generation_config)
+                
+                print(f"DEBUG IMAGE_DESC: Successfully received response from Gemini API")
+                
+                if update_progress:
+                    update_progress(100, "✅ Description generated")
+                
+                # Get text response 
+                print(f"DEBUG IMAGE_DESC: Returning text response of length: {len(response.text) if response.text else 0}")
+                return response.text
+                
+            except Exception as api_error:
+                last_error = api_error
+                error_str = str(api_error)
+                print(f"DEBUG IMAGE_DESC ERROR on attempt {retry_count+1}: {error_str}")
+                
+                # Check if this is a timeout or rate limit error (worth retrying)
+                if "Deadline Exceeded" in error_str or "429" in error_str or "quota" in error_str.lower() or "timeout" in error_str.lower():
+                    retry_count += 1
+                    if retry_count < MAX_RETRIES:
+                        print(f"DEBUG IMAGE_DESC: Timeout or rate limit error, will retry in {current_delay}s")
+                        time.sleep(current_delay)
+                        current_delay *= 2  # Exponential backoff
+                    else:
+                        print("DEBUG IMAGE_DESC: Max retries reached")
+                else:
+                    # Non-retryable error
+                    print("DEBUG IMAGE_DESC: Non-retryable error, giving up")
+                    break
+        
+        # If we get here, all retries failed
+        error_msg = f"Error generating content after {MAX_RETRIES} attempts: {str(last_error)}"
+        print(f"DEBUG IMAGE_DESC ERROR: {error_msg}")
+        if update_progress:
+            update_progress(0, f"❌ {error_msg}")
+        return f"❌ {error_msg}"
+            
+    except Exception as e:
+        error_msg = f"Unexpected error: {str(e)}"
+        print(f"DEBUG IMAGE_DESC ERROR: {error_msg}")
+        print(traceback.format_exc())
+        if update_progress:
+            update_progress(0, f"❌ {error_msg}")
+        return f"❌ {error_msg}"
 
 def call_api_translation(api, model, text, target_language, prompt_key="translation"):
     prompt_template = get_prompt(prompt_key)
     prompt = prompt_template.format(target_language=target_language) + text
     try:
-        from google import genai
+        import google.generativeai as genai
     except ImportError:
-        raise ImportError("Please install 'google-genai' to use Gemini.")
+        raise ImportError("Please install 'google-generativeai' to use Gemini.")
     try:
         loop = asyncio.get_event_loop()
     except RuntimeError:
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-    client = genai.Client(api_key=os.environ.get("GEMINI_API_KEY"))
-    response = client.models.generate_content(model=model, contents=[prompt])
+        
+    # Get the API key
+    api_key = os.environ.get("GEMINI_API_KEY")
+    if not api_key:
+        return "❌ GEMINI_API_KEY not found in environment variables"
+    
+    # Configure API
+    genai.configure(api_key=api_key)
+    
+    # Create model instance
+    text_model = genai.GenerativeModel(model)
+    
+    # Generate content - simpler format for v0.3.1
+    response = text_model.generate_content(prompt, generation_config={
+        "temperature": 0.2,
+        "top_p": 0.95,
+        "top_k": 40
+    })
+    
+    # Get text response
     return response.text
 
 def embed_ocr_in_pdf(input_pdf, output_pdf):

cline_docs/activeContext.md

@@ -4,6 +4,7 @@
 - Implementing authentication system for DocuLens
 - Securing all API endpoints
 - Adding user management functionality
+- Finalizing image description functionality
 
 ## Recent Changes
 - Added Flask-Login for backend authentication
@@ -12,12 +13,42 @@
 - Protected all API routes
 - Added session management
 - Implemented user authentication flow
+- Added image description functionality:
+  - Added dedicated /api/image-description endpoint
+  - Implemented image description processing pipeline
+  - Created ImageDescription React component
+  - Added new navigation tab for Image Description
+  - Fixed image handling to use proper Gemini Vision API format
+  - Implemented base64 encoding with inlineData format for images
+  - Added comprehensive error handling and debugging
+- Fixed Google Generative AI integration:
+  - Updated package name from 'google-genai' to 'google-generativeai'
+  - Upgraded to latest version 0.3.1
+  - Completely rewrote all API call functions to use the simpler format for v0.3.1
+  - Removed use of genai.types.Content and genai.types.Part classes that don't exist in v0.3.1
+  - Fixed image description functionality to properly use inline_data format
+  - Fixed the error with 'upload_blob' method by using direct inline data approach
+  - Updated Dockerfile to install the correct package
+  - Added advanced image optimization to improve API reliability:
+    - Resizing large images (max 1600px on any side)
+    - Converting to RGB format (handling transparency)
+    - Optimizing quality and compression
+  - Implemented timeout handling with retry logic:
+    - Added exponential backoff retries (3 attempts)
+    - Added specific handling for 504 Deadline Exceeded errors
+    - Optimized output token limits to avoid timeouts
+- Dependency and prompt updates:
+  - Upgraded OCRmyPDF to version 16.10.0
+  - Added pikepdf 9.5.2 as a dependency to fix OCRmyPDF compatibility issue
+  - Removed the image_desc prompt while keeping ocr, ocr_correction, and translation prompts
+  - Added fallback logic for image description functionality to use a simple default prompt
 
 ## Next Steps
 1. Test authentication system thoroughly
-2. Add password change functionality
-3. Consider adding:
+2. Test image description functionality
+3. Add password change functionality
+4. Consider adding:
    - Password reset capability
    - Account lockout after failed attempts
    - Session timeout settings
-4. Update documentation with authentication details
+5. Update documentation with authentication and image description details

cline_docs/productContext.md

@@ -1,7 +1,7 @@
 # Product Context
 
 ## Purpose
-DocuLens is a unified web application that combines Optical Character Recognition (OCR) and Artificial Intelligence (AI) to process and translate documents. It aims to provide an intuitive interface with real-time feedback for document processing tasks.
+DocuLens is a unified web application that combines Optical Character Recognition (OCR), Artificial Intelligence (AI), and computer vision to process, translate, and analyze documents and images. It aims to provide an intuitive interface with real-time feedback for document and image processing tasks.
 
 ## Problems Solved
 1. Complex document processing made simple through a unified interface
@@ -10,18 +10,22 @@ DocuLens is a unified web application that combines Optical Character Recognitio
    - Enhanced OCR (Tesseract + AI correction)
    - Full AI OCR processing
 3. Document translation with support for multiple languages
-4. Real-time progress tracking with visual feedback
+4. Image description and analysis using AI vision models
+5. Real-time progress tracking with visual feedback
+6. Secure access with user authentication
 
 ## How It Works
 1. File Management:
    - Supports PDF and image uploads
    - Automatic file naming to prevent overwrites
    - Bulk file management capabilities
+   - Secure access with user authentication
 
 2. Processing Modes:
    - OCR Mode: Uses Tesseract for text extraction
    - OCR + AI Mode: Combines Tesseract with AI correction
    - AI Mode: Full AI-powered OCR processing
+   - Image Description: Uses Gemini's vision capabilities to analyze and describe images
 
 3. Translation Features:
    - Page-by-page translation
@@ -29,8 +33,15 @@ DocuLens is a unified web application that combines Optical Character Recognitio
    - Progress tracking
    - Outputs translated text with page markers
 
-4. Configuration System:
+4. Image Description:
+   - Upload images for AI-powered analysis
+   - Detailed descriptions generated using Gemini Vision API
+   - Support for various image formats (PNG, JPG, JPEG, GIF, BMP)
+   - Real-time progress tracking
+
+5. Configuration System:
    - AI model management
    - Custom prompt configuration
    - Language settings
    - Import/export of configurations
+   - User account management

cline_docs/progress.md

@@ -36,6 +36,16 @@
    - ✅ Protected API routes
    - ✅ Session management
    - ✅ Remember me functionality
+   
+7. Image Description
+   - ✅ Image upload and processing
+   - ✅ AI vision analysis
+   - ✅ Base64 image encoding
+   - ✅ Progress tracking
+   - ✅ Description text generation
+   - ✅ Fixed Gemini Vision API integration
+   - ✅ Advanced image optimization (resizing, format conversion)
+   - ✅ Error handling with retry mechanisms
 
 ## Current Status
 - Application is fully functional
@@ -77,7 +87,9 @@
 - Overall Progress: ~85% complete
 
 ## Known Issues
-- None reported at this time
+- ✅ Fixed: 'Files' object has no attribute 'upload_blob' error in image description functionality
+- ✅ Fixed: '504 Deadline Exceeded' error in Gemini API calls with large images
+- ✅ Fixed: 'cannot import name PdfMatrix' error in OCRmyPDF by adding pikepdf dependency
 
 ## Next Milestone Goals
 1. Implement batch processing

cline_docs/systemPatterns.md

@@ -23,6 +23,8 @@ The application follows a client-server architecture with containerized deployme
   - ProgressBar: Real-time processing feedback
   - Notifications: User feedback system
   - TxtToPdf: Document conversion utility
+  - ImageDescription: Image analysis and description
+  - ChangePassword: User password management
 
 ### Backend (Flask)
 - RESTful API architecture
@@ -50,6 +52,10 @@ The application follows a client-server architecture with containerized deployme
   - Gemini
   - Mistral
 - Configurable prompts system
+- Vision AI capabilities:
+  - Base64 image encoding
+  - MIME type detection
+  - Proper Gemini Vision API formatting
 
 ### OCR Processing
 - Multiple processing modes:

cline_docs/techContext.md

@@ -21,6 +21,8 @@
 - **Supported AI Providers**:
   - OpenAI
   - Google Gemini
+    - Text generation API
+    - Vision API for image analysis
   - Mistral
 
 ### Containerization

frontend/src/App.js

@@ -8,6 +8,7 @@ import Configurations from './components/Configurations';
 import TxtToPdf from './components/TxtToPdf';
 import Notifications from './components/Notifications';
 import ChangePassword from './components/ChangePassword';
+import ImageDescription from './components/ImageDescription';
 
 function App() {
   const [activeTab, setActiveTab] = useState('DocuLens');
@@ -89,6 +90,12 @@ function App() {
         >
           💡 DocuLens
         </button>
+        <button
+          onClick={() => setActiveTab('image-description')}
+          className={activeTab === 'image-description' ? 'active tab-processing' : 'tab-processing'}
+        >
+          🖼️ Image Description
+        </button>
         <button
           onClick={() => setActiveTab('files')}
           className={activeTab === 'files' ? 'active tab-default' : 'tab-default'}
@@ -110,6 +117,7 @@ function App() {
       </nav>
       <main>
         {activeTab === 'DocuLens' && <FileUpload onJobCompleted={handleJobCompleted} />}
+        {activeTab === 'image-description' && <ImageDescription onJobCompleted={handleJobCompleted} />}
         {activeTab === 'files' && <FileList />}
         {activeTab === 'configurations' && <Configurations />}
         {activeTab === 'txttopdf' && <TxtToPdf />}

frontend/src/components/ImageDescription.js

@@ -0,0 +1,175 @@
+// frontend/src/components/ImageDescription.js
+import React, { useState, useEffect } from 'react';
+import axios from 'axios';
+import ProgressBar from './ProgressBar';
+import ModelSelector from './ModelSelector';
+
+const API_URL = '/api';
+
+function ImageDescription({ onJobCompleted }) {
+  const [file, setFile] = useState(null);
+  const [api, setApi] = useState('Gemini');
+  const [model, setModel] = useState('');
+  const [models, setModels] = useState([]);
+  const [promptKey, setPromptKey] = useState('image_desc');
+  const [uploadProgress, setUploadProgress] = useState(0);
+  const [message, setMessage] = useState('');
+  const [jobId, setJobId] = useState(null);
+
+  useEffect(() => {
+    axios.get(`${API_URL}/models`, { params: { api } })
+      .then(response => {
+        setModels(response.data.models);
+        if (response.data.models.length > 0) {
+          setModel(response.data.models[0]);
+        }
+      })
+      .catch(err => console.error(err));
+  }, [api]);
+
+  useEffect(() => {
+    if (jobId) {
+      const interval = setInterval(() => {
+        axios.get(`${API_URL}/progress/${jobId}`)
+          .then(response => {
+            const data = response.data;
+            setUploadProgress(data.progress);
+            setMessage(data.status);
+            if (data.progress === 100 || data.status.includes("Cancelled") || data.status.includes("Error")) {
+              clearInterval(interval);
+              onJobCompleted && onJobCompleted("Processing job completed");
+              setJobId(null);
+            }
+          })
+          .catch(err => console.error(err));
+      }, 2000);
+      return () => clearInterval(interval);
+    }
+  }, [jobId, onJobCompleted]);
+
+  const handleFileChange = (e) => {
+    const selectedFile = e.target.files[0];
+    if (selectedFile) {
+      // Check if file is an image
+      if (!selectedFile.type.match('image/*')) {
+        setMessage("⚠️ Please select an image file (JPEG, PNG, etc.)");
+        return;
+      }
+      setFile(selectedFile);
+    }
+  };
+
+  const handleDrop = (e) => {
+    e.preventDefault();
+    if (e.dataTransfer.files && e.dataTransfer.files.length > 0) {
+      const droppedFile = e.dataTransfer.files[0];
+      // Check if file is an image
+      if (!droppedFile.type.match('image/*')) {
+        setMessage("⚠️ Please select an image file (JPEG, PNG, etc.)");
+        return;
+      }
+      setFile(droppedFile);
+      e.dataTransfer.clearData();
+    }
+  };
+
+  const handleDragOver = (e) => e.preventDefault();
+
+  const handleStop = () => {
+    if (jobId) {
+      axios.post(`${API_URL}/stop/${jobId}`)
+        .then(response => {
+          setMessage("⏹️ Process stopped by user");
+          setJobId(null);
+        })
+        .catch(err => console.error(err));
+    }
+  };
+
+  const handleSubmit = (e) => {
+    e.preventDefault();
+    if (!file) {
+      setMessage("⚠️ Please select an image file.");
+      return;
+    }
+
+    setMessage("🚀 Uploading image...");
+    const formData = new FormData();
+    formData.append("file", file);
+    formData.append("api", api);
+    formData.append("model", model);
+    formData.append("prompt_key", promptKey);
+
+    axios.post(`${API_URL}/image-description`, formData, { 
+      headers: { "Content-Type": "multipart/form-data" }
+    })
+      .then(response => {
+        setMessage("📤 Image uploaded. Processing started.");
+        setJobId(response.data.job_id);
+      })
+      .catch(err => {
+        console.error("Error details:", err.response?.data || err.message);
+        setMessage(`❌ Error: ${err.response?.data?.error || "Failed to upload image"}`);
+      });
+  };
+
+  return (
+    <div>
+      <h2>Image Description</h2>
+      <p>Upload an image to generate an AI description</p>
+      <form onSubmit={handleSubmit} className="upload-form">
+        <div
+          className="drop-zone"
+          onDrop={handleDrop}
+          onDragOver={handleDragOver}
+          style={{
+            border: '2px dashed #ccc',
+            padding: '20px',
+            borderRadius: '5px',
+            marginBottom: '10px'
+          }}
+        >
+          {file ? 
+            <div>
+              <p>📄 {file.name}</p>
+              <img 
+                src={URL.createObjectURL(file)} 
+                alt="Preview" 
+                style={{ maxWidth: '200px', maxHeight: '200px', marginTop: '10px' }} 
+              />
+            </div> : 
+            <p>📂 Drag and drop an image here or click to select</p>
+          }
+          <input 
+            type="file" 
+            onChange={handleFileChange} 
+            style={{ display: 'none' }} 
+            id="imageInput" 
+            accept="image/*" 
+          />
+          <label htmlFor="imageInput" style={{ cursor: 'pointer', color: 'blue' }}>
+            Select image
+          </label>
+        </div>
+        <div className="selectors" style={{ marginBottom: '10px' }}>
+          <ModelSelector models={models} selectedModel={model} setSelectedModel={setModel} />
+        </div>
+        <button type="submit">Upload and describe</button>
+        {jobId && (
+          <button type="button" onClick={handleStop} style={{ marginLeft: '10px' }}>
+            Stop Process
+          </button>
+        )}
+      </form>
+      {jobId && (
+        <div>
+          <ProgressBar progress={uploadProgress} />
+          <p>{message}</p>
+        </div>
+      )}
+      {!jobId && <p>{message}</p>}
+    </div>
+  );
+}
+
+export default ImageDescription;