From 4b8935afd1fd85ba9e653bb570d174daae03e43e Mon Sep 17 00:00:00 2001 From: TheMaddax Date: Tue, 7 Jan 2025 09:44:37 -0600 Subject: [PATCH] feat: Implement auto-capture and cursor-free snapshots - Add automatic capture start/stop based on board detection - Implement cursor-free snapshot system: * Add SCStreamConfiguration cursor control * Add temporary capture session management * Ensure clean snapshots without cursor artifacts - Add visual feedback: * Status indicator (green/yellow/gray) * Snapshot preview below board * Clear capture state indication - Update documentation: * Add snapshot system patterns * Document cursor control implementation * Update technical constraints --- ChessPrism/ChessPrism/ContentView.swift | 41 ++++- ChessPrism/ChessPrism/ScreenCapture.swift | 21 ++- .../ChessPrism/ScreenCaptureViewModel.swift | 153 ++++++++++++------ cline_docs/activeContext.md | 68 ++++---- cline_docs/productContext.md | 24 +++ cline_docs/systemPatterns.md | 24 +++ cline_docs/techContext.md | 9 ++ 7 files changed, 254 insertions(+), 86 deletions(-) diff --git a/ChessPrism/ChessPrism/ContentView.swift b/ChessPrism/ChessPrism/ContentView.swift index 5c79d1c..d7c36d1 100644 --- a/ChessPrism/ChessPrism/ContentView.swift +++ b/ChessPrism/ChessPrism/ContentView.swift @@ -89,11 +89,26 @@ struct ContentView: View { var body: some View { VStack { - // Capture status indicator - CaptureStatusButton( - isCapturing: viewModel.isCapturing, - isBoardDetected: viewModel.isBoardDetected - ) + // Status and Scan controls + HStack { + CaptureStatusButton( + isCapturing: viewModel.isCapturing, + isBoardDetected: viewModel.isBoardDetected + ) + + Button(action: { + Task { + await viewModel.takeSnapshot() + } + }) { + Text("Scan") + .foregroundColor(.white) + .padding(.horizontal, 20) + .padding(.vertical, 10) + } + .buttonStyle(.borderedProminent) + .disabled(!viewModel.isCapturing || !viewModel.isBoardDetected) + } .onAppear { // Start monitoring for chess boards when view appears viewModel.startMonitoring() @@ -142,6 +157,22 @@ struct ContentView: View { Text("No board detected") .foregroundColor(.gray) } + + if viewModel.snapshotTaken { + Divider() + .padding(.vertical) + + Text("Latest Snapshot") + if let snapshot = viewModel.latestSnapshot { + Image(nsImage: snapshot) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxWidth: 400) + .padding() + .background(Color.black.opacity(0.1)) + .cornerRadius(8) + } + } } } .padding() diff --git a/ChessPrism/ChessPrism/ScreenCapture.swift b/ChessPrism/ChessPrism/ScreenCapture.swift index 95aa85d..a268de2 100644 --- a/ChessPrism/ChessPrism/ScreenCapture.swift +++ b/ChessPrism/ChessPrism/ScreenCapture.swift @@ -23,8 +23,16 @@ class ScreenCapture: NSObject, SCStreamOutput { private var activeStream: SCStream? private var lastCapturedImage: NSImage? + private var excludeCursor: Bool = false - func startCapture() async throws { + func startCapture(excludeCursor: Bool = false) async throws { + // Always stop any existing capture before starting a new one + if activeStream != nil { + try await stopCapture() + } + + self.excludeCursor = excludeCursor + guard #available(macOS 12.3, *) else { return } let content = try await SCShareableContent.current @@ -43,6 +51,9 @@ class ScreenCapture: NSObject, SCStreamOutput { let config = SCStreamConfiguration() config.width = Int(window.frame.width) config.height = Int(window.frame.height) + config.minimumFrameInterval = CMTime(value: 1, timescale: 30) // 30 FPS + config.queueDepth = 5 // Buffer up to 5 frames + config.showsCursor = !excludeCursor // Set cursor visibility let stream = SCStream(filter: filter, configuration: config, delegate: nil) try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: DispatchQueue.global(qos: .userInitiated)) @@ -53,9 +64,11 @@ class ScreenCapture: NSObject, SCStreamOutput { } func stopCapture() async throws { - guard let stream = activeStream else { return } - try await stream.stopCapture() - activeStream = nil + if let stream = activeStream { + try await stream.stopCapture() + activeStream = nil + lastCapturedImage = nil + } } func getCurrentImage() -> NSImage? { diff --git a/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift b/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift index dac2f36..bf943ef 100644 --- a/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift +++ b/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift @@ -11,8 +11,11 @@ class ScreenCaptureViewModel: ObservableObject { @Published var isCapturing = false @Published var isBoardDetected = false @Published var isAutoCapturing = true // Default to auto-capture mode + @Published var snapshotTaken = false // Track if snapshot was taken + @Published var latestSnapshot: NSImage? // Make snapshot accessible to view - private let screenCapture = ScreenCapture() + private let captureManager = ScreenCapture() // For actual capture + private let monitorManager = ScreenCapture() // For monitoring private let boardDetector = BoardDetector() private var captureTask: Task? private var monitorTask: Task? @@ -21,11 +24,13 @@ class ScreenCaptureViewModel: ObservableObject { private var context: CIContext { ScreenCaptureViewModel.shared } private var detectedBoardRect: CGRect? + private var isMonitoring = false enum CaptureError: LocalizedError { case chessWindowNotFound case boardDetectionFailed case noBoardDetected + case snapshotFailed var errorDescription: String? { switch self { @@ -35,92 +40,135 @@ class ScreenCaptureViewModel: ObservableObject { return "Failed to detect chess board" case .noBoardDetected: return "No chess board detected" + case .snapshotFailed: + return "Failed to take snapshot" } } } + func takeSnapshot() async { + // Stop current capture + try? await captureManager.stopCapture() + + // Start new capture without cursor + do { + try await captureManager.startCapture(excludeCursor: true) + // Wait a brief moment for the capture to stabilize + try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds + + if let currentImage = captureManager.getCurrentImage() { + // Process the image to get the cropped board + try await processImage(currentImage) + + // Store the cropped board as snapshot + if let boardImage = croppedBoardImage { + latestSnapshot = boardImage + snapshotTaken = true + } else { + captureError = .snapshotFailed + } + } else { + captureError = .snapshotFailed + } + + // Restart normal capture + try await captureManager.startCapture(excludeCursor: false) + } catch { + captureError = .snapshotFailed + // Ensure we restart normal capture even if snapshot fails + try? await captureManager.startCapture(excludeCursor: false) + } + } + func startMonitoring() { - guard monitorTask == nil else { return } + guard !isMonitoring else { return } + isMonitoring = true monitorTask = Task { - do { - // Start continuous monitoring - try await screenCapture.startCapture() - - monitorLoop: while !Task.isCancelled { - do { - if let image = screenCapture.getCurrentImage(), + monitorLoop: while !Task.isCancelled { + do { + // Start monitoring capture + try await monitorManager.startCapture() + + while !Task.isCancelled { + if let image = monitorManager.getCurrentImage(), let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) { // Check for board let ciImage = CIImage(cgImage: cgImage) if boardDetector.detectBoard(in: ciImage) != nil { // Board detected, start capture if not already capturing if !isCapturing { - startCapture() + await startCapture() } } } try await Task.sleep(nanoseconds: 500_000_000) // 0.5 seconds - } catch is CancellationError { - break monitorLoop - } catch { - // Just log the error and continue monitoring - print("Monitor error: \(error)") + } + } catch { + print("Monitor error: \(error)") + // If there's an error, wait briefly and try again + if !Task.isCancelled { + try? await Task.sleep(nanoseconds: 1_000_000_000) // 1 second + continue monitorLoop } } - - try? await screenCapture.stopCapture() - } catch { - print("Monitor setup error: \(error)") } + + isMonitoring = false + try? await monitorManager.stopCapture() } } func stopMonitoring() { monitorTask?.cancel() monitorTask = nil + isMonitoring = false stopCapture() + + Task { + try? await monitorManager.stopCapture() + } } - func startCapture() { + func startCapture() async { guard !isCapturing else { return } isCapturing = true captureError = nil + snapshotTaken = false + latestSnapshot = nil + + do { + try await captureManager.startCapture() + } catch { + handleCaptureError(error) + isCapturing = false + return + } captureTask = Task { - do { - // Start continuous capture - try await screenCapture.startCapture() - - captureLoop: while !Task.isCancelled { - do { - if let image = screenCapture.getCurrentImage() { - try await processImage(image) - } - try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds - } catch is CancellationError { - break captureLoop - } catch { - // Just update the error state but continue capturing - if !Task.isCancelled { - handleCaptureError(error) - if isAutoCapturing && error as? CaptureError == .noBoardDetected { - // In auto-capture mode, stop capture but keep monitoring - stopCapture() - break captureLoop - } + captureLoop: while !Task.isCancelled { + do { + if let image = captureManager.getCurrentImage() { + try await processImage(image) + } + try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds + } catch is CancellationError { + break captureLoop + } catch { + // Just update the error state but continue capturing + if !Task.isCancelled { + handleCaptureError(error) + if isAutoCapturing && error as? CaptureError == .noBoardDetected { + // In auto-capture mode, stop capture but keep monitoring + stopCapture() + break captureLoop } } } - - // Clean up - try? await screenCapture.stopCapture() - if isCapturing { - isCapturing = false - } - } catch { - handleCaptureError(error) + } + + if isCapturing { isCapturing = false } } @@ -132,6 +180,13 @@ class ScreenCaptureViewModel: ObservableObject { isCapturing = false captureError = nil isBoardDetected = false + snapshotTaken = false + latestSnapshot = nil + + // Clean up capture session + Task { + try? await captureManager.stopCapture() + } } private func processImage(_ image: NSImage) async throws { diff --git a/cline_docs/activeContext.md b/cline_docs/activeContext.md index 5e579cf..5809550 100644 --- a/cline_docs/activeContext.md +++ b/cline_docs/activeContext.md @@ -10,23 +10,43 @@ - Automatic board detection and capture implemented - Visual capture status indicator added - Continuous board monitoring system implemented +- Auto-capture on game start/stop working successfully +- Manual snapshot system implemented with Scan button +- Snapshot preview display added below Chessboard Preview +- Cursor-free snapshot capture implemented ## Recent Changes -1. Implemented Continuous Board Monitoring: +1. Enhanced Manual Snapshot System: + - New cursor-free capture: + * Temporarily disables cursor during snapshot + * Ensures clean board capture without mouse pointer + * Automatically restores cursor after snapshot + - Improved snapshot process: + * Stops current capture + * Takes cursor-free snapshot + * Processes board detection + * Restores normal capture + * Handles errors gracefully + - Snapshot visualization: + * Added preview area below Chessboard Preview + * Shows latest snapshot with visual feedback + * Clear indication when snapshot is taken + +2. Implemented Continuous Board Monitoring: - Added separate monitoring and capture tasks: * Monitor constantly checks for chess boards (0.5s interval) * Capture processes frames when active (0.1s interval) - - Auto-capture behavior: + - Auto-capture behavior working as expected: * Starts monitoring when app launches * Automatically starts capture when board appears - * Stops capture (but keeps monitoring) when board disappears - * Resumes capture when new board is detected - - Fixed image conversion pipeline: - * Proper NSImage → CGImage → CIImage conversion - * Efficient resource management - * Clean error handling + * Stops capture but continues monitoring when board disappears + * Successfully resumes capture when new game starts + - Performance characteristics: + * ~40% CPU usage during operation + * Stable memory management + * Responsive to game state changes -2. Enhanced Status Indication: +3. Enhanced Status Indication: - Visual status indicator shows capture state: * Green: Actively capturing board * Yellow: Waiting for board @@ -34,7 +54,7 @@ - Clear error messages for different states - Automatic status updates based on board detection -3. Resource Management Optimization: +4. Resource Management: - Implemented shared CIContext pattern: * Prevents command queue exhaustion * Reduces Metal resource usage @@ -42,22 +62,13 @@ - Proper cleanup on task completion - Efficient resource utilization -4. Improved Window Detection: +5. Improved Window Detection: - Using SCShareableContent for window access - Precise window identification: * Exact bundle ID matching (com.chess.iphone) * Window visibility verification (isOnScreen) * Size validation (width > 100 && height > 100) -5. Enhanced Capture System: - - Continuous capture implementation: - * Single persistent capture stream - * Smooth frame processing (no flickering) - * Efficient resource usage - - Proper frame dimensions from window - - Clean task cancellation handling - - Main thread safety for UI updates - 6. Error Handling: - Improved error resilience: * Continues monitoring even if capture stops @@ -69,16 +80,17 @@ - Thread-safe state updates ## Current Focus -1. Board Detection: - - Implement chess board recognition - - Handle different board themes - - Process captured frames efficiently +1. Board Recognition: + - Process snapshot images + - Implement piece detection + - Extract board state ## Next Steps -1. Implement board detection: - - Pattern recognition for chess pieces - - Board coordinate mapping - - Position validation +1. Implement board recognition: + - Process snapshot images + - Detect chess pieces + - Map board coordinates + - Validate positions 2. Add position analysis 3. Create move detection system 4. Implement visual overlay diff --git a/cline_docs/productContext.md b/cline_docs/productContext.md index f3e81af..0ae4777 100644 --- a/cline_docs/productContext.md +++ b/cline_docs/productContext.md @@ -15,11 +15,13 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Recognizes board coordinates and boundaries - Handles various board themes and orientations - Maintains accuracy during game play +- Provides clean board snapshots for analysis ### Real-time Processing - Captures and processes screen content in real-time - Provides immediate feedback and analysis - Maintains performance during long sessions +- Supports manual snapshot capture for detailed analysis ## User Experience Goals @@ -28,11 +30,14 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Works with Chess.com desktop app - Minimal setup requirements - Automatic board detection and tracking + - Clean snapshot capture without cursor interference 2. Intuitive Interface - Clear visualization of analysis - Easy-to-understand suggestions - Minimal user intervention required + - Visual feedback for capture states + - Manual snapshot control ### Reliable Detection 1. Board Recognition @@ -41,6 +46,7 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp * Coordinate-based fallback for reliability - Proper coordinate system handling - Consistent board capture across sessions + - High-quality snapshots for analysis 2. Position Analysis - Accurate piece recognition (planned) @@ -59,11 +65,13 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Understanding position evaluation - Learning from mistakes - Exploring alternative moves + - Analyzing specific positions via snapshots 2. Analysis - Real-time position assessment - Move validation - Strategic planning + - Detailed position study ## Product Requirements @@ -73,6 +81,7 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Full board capture - Support for Chess.com desktop app - Reliable coordinate transformations + - Clean snapshot capability 2. Position Analysis (Planned) - Real-time evaluation @@ -83,22 +92,27 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Analysis overlay - Control panel - Settings management + - Snapshot controls + - Visual status indicators ### Quality Standards 1. Accuracy - Reliable board detection - Complete board capture - Precise coordinate handling + - Clean snapshots without artifacts 2. Performance - Real-time processing - Minimal resource usage - Stable operation + - Efficient snapshot handling 3. Usability - Intuitive controls - Clear feedback - Minimal setup + - Simple snapshot workflow ## Success Metrics @@ -107,12 +121,14 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Full board capture success rate - Processing speed per frame - Error recovery rate +- Snapshot quality assessment ### User Metrics - Setup success rate - Analysis accuracy - User engagement time - Feature utilization +- Snapshot usage patterns ## Current Challenges @@ -127,12 +143,14 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Full board capture - Consistent positioning - Reliable boundaries + - Clean snapshots ### Next Steps 1. Refine board detection - Improve coordinate handling - Ensure full board capture - Validate transformations + - Optimize snapshot quality 2. Move to position analysis - Piece recognition @@ -146,16 +164,19 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Deep position evaluation - Opening recognition - Endgame tablebases + - Position comparison from snapshots 2. Learning Tools - Mistake analysis - Improvement suggestions - Progress tracking + - Position database from snapshots 3. Customization - Analysis depth control - Visual preference settings - Platform-specific optimizations + - Snapshot management options ## Product Roadmap @@ -163,13 +184,16 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp - Core board detection system - Coordinate system handling - Basic user interface +- Manual snapshot system ### Next Phase - Position analysis - Move suggestion system - Visual overlay implementation +- Enhanced snapshot analysis ### Future Phase - Advanced analysis features - Learning tools integration - Customization options +- Snapshot database and comparison tools diff --git a/cline_docs/systemPatterns.md b/cline_docs/systemPatterns.md index 87bba30..10166d7 100644 --- a/cline_docs/systemPatterns.md +++ b/cline_docs/systemPatterns.md @@ -24,12 +24,35 @@ - Window-specific capture setup - Frame dimension matching - Proper delegate handling + - Cursor visibility control: + * Configurable cursor display + * Clean snapshot support + * State preservation 2. Frame Processing - Main thread safety for UI updates - Efficient image conversion pipeline - Resource cleanup +### Snapshot System Pattern +1. Cursor-Free Capture + - Temporary capture session: + * Disables cursor visibility + * Takes clean snapshot + * Restores normal capture + - Error handling: + * Session cleanup + * State recovery + * Capture restoration + +2. Process Flow + - Stop current capture + - Start cursor-free capture + - Wait for stabilization + - Take snapshot + - Process image + - Restore normal capture + ### Error Handling Pattern 1. Task Management - Proper cancellation points @@ -89,6 +112,7 @@ - Handles capture session lifecycle and cleanup - Manages permissions and error handling - Optimized resource usage +- Configurable cursor visibility ### Board Detection System Two implemented approaches: diff --git a/cline_docs/techContext.md b/cline_docs/techContext.md index 37881b4..1f5544a 100644 --- a/cline_docs/techContext.md +++ b/cline_docs/techContext.md @@ -25,11 +25,13 @@ * iOS app window capture support * Real-time frame capture * Proper error handling + * Configurable cursor visibility - Key components: * SCShareableContent: Window and display access * SCContentFilter: Window-specific capture * SCStream: Frame capture management * SCStreamOutput: Frame processing + * SCStreamConfiguration: Capture settings including cursor control ### Vision Framework (Planned) - Will be used for board and coordinate detection @@ -89,6 +91,10 @@ - Frame dimension matching - Proper delegate handling - Resource cleanup + - Cursor visibility control: + * Configurable via SCStreamConfiguration + * State preservation between captures + * Clean snapshot support 3. Performance - Main thread safety for UI updates @@ -155,18 +161,21 @@ - Error handling - State management - Resource management +- Cursor control functionality ### Integration Tests - End-to-end workflows - Component interaction - Event propagation - Resource sharing +- Snapshot system ### UI Tests - User interaction flows - Error state handling - Visual feedback - Performance monitoring +- Snapshot visualization ## Documentation Requirements