diff --git a/ChessPrism/ChessPrism/BoardDetector.swift b/ChessPrism/ChessPrism/BoardDetector.swift index 8344afe..d5d99e6 100644 --- a/ChessPrism/ChessPrism/BoardDetector.swift +++ b/ChessPrism/ChessPrism/BoardDetector.swift @@ -1,194 +1,76 @@ import Foundation -import Vision import CoreImage +import Vision class BoardDetector { - private let context = CIContext() - - // Known chess interface patterns - private struct ChessPattern { - let interfaceType: ChessInterface - let boardColor: CGColor - let squareSize: CGFloat - let cornerPattern: [CGPoint] - } - - enum ChessInterface { - case chesscom - case lichess - } - - // Chess.com and Lichess patterns - private let patterns: [ChessPattern] = [ - // Chess.com light theme pattern - ChessPattern( - interfaceType: .chesscom, - boardColor: CGColor(red: 0.82, green: 0.82, blue: 0.82, alpha: 1.0), // Light squares - squareSize: 45.0, // Default square size - cornerPattern: [ - CGPoint(x: 0, y: 0), - CGPoint(x: 45, y: 0), - CGPoint(x: 0, y: 45), - CGPoint(x: 45, y: 45) - ] - ), - // Lichess light theme pattern - ChessPattern( - interfaceType: .lichess, - boardColor: CGColor(red: 0.93, green: 0.93, blue: 0.83, alpha: 1.0), // Light squares - squareSize: 40.0, // Default square size - cornerPattern: [ - CGPoint(x: 0, y: 0), - CGPoint(x: 40, y: 0), - CGPoint(x: 0, y: 40), - CGPoint(x: 40, y: 40) - ] - ) - ] - - // MARK: - Pattern Detection + // Share CIContext to avoid creating too many Metal command queues + private static let shared = CIContext() + private var context: CIContext { BoardDetector.shared } func detectBoard(in image: CIImage) -> CGRect? { - // 1. Color-based detection - let colorMatches = detectColorPatterns(in: image) + // Configure rectangle detection request + let request = VNDetectRectanglesRequest() + request.minimumAspectRatio = 0.8 // Adjusted for standard chess board + request.maximumAspectRatio = 1.2 + request.minimumSize = 0.4 + request.maximumObservations = 1 + request.quadratureTolerance = 30 + request.minimumConfidence = 0.9 - // 2. Grid pattern detection - let gridMatches = detectGridPattern(in: image, colorMatches: colorMatches) - - // 3. Corner validation - if let bestMatch = validateCorners(in: image, candidates: gridMatches) { - // Extract square board from detected area - let width = bestMatch.width - let height = bestMatch.height - - // Calculate the board position - // Vision coordinates are bottom-left origin, so we need to: - // 1. Keep the same width - // 2. Use the top portion of the detected area - // 3. Maintain square aspect ratio - return CGRect( - x: bestMatch.minX, - y: bestMatch.maxY - width, // Position at top of detected area - width: width, - height: width - ) + // Perform the request + let requestHandler = VNImageRequestHandler(ciImage: image, options: [:]) + do { + try requestHandler.perform([request]) + } catch { + print("Failed to perform rectangle detection: \(error)") + return nil } - return nil - } - - private func detectColorPatterns(in image: CIImage) -> [CGRect] { - var matches: [CGRect] = [] - - for pattern in patterns { - // Apply color adjustments to enhance pattern detection - let colorControls = CIFilter(name: "CIColorControls") - colorControls?.setValue(image, forKey: kCIInputImageKey) - colorControls?.setValue(1.2, forKey: kCIInputSaturationKey) // Increase saturation - colorControls?.setValue(0.2, forKey: kCIInputBrightnessKey) // Adjust brightness - colorControls?.setValue(1.1, forKey: kCIInputContrastKey) // Increase contrast - - guard let adjustedImage = colorControls?.outputImage else { continue } - - // Detect rectangles - var rectangles: [VNRectangleObservation] = [] - let request = VNDetectRectanglesRequest() - request.minimumAspectRatio = 0.3 // Allow taller rectangles - request.maximumAspectRatio = 0.5 - request.minimumSize = 0.4 // Look for larger areas - request.maximumObservations = 1 - - let handler = VNImageRequestHandler(ciImage: adjustedImage) - try? handler.perform([request]) - - if let results = request.results as? [VNRectangleObservation] { - matches.append(contentsOf: results.map { $0.boundingBox }) - } + // Process results + guard let observations = request.results, + !observations.isEmpty else { + return nil } - return matches - } - - private func detectGridPattern(in image: CIImage, colorMatches: [CGRect]) -> [CGRect] { - var gridMatches: [CGRect] = [] + let bestObservation = observations[0] - for rect in colorMatches { - // Create edge detection filter - let edgeFilter = CIFilter(name: "CIEdgeWork") - edgeFilter?.setValue(image.cropped(to: rect), forKey: kCIInputImageKey) - edgeFilter?.setValue(1.0, forKey: kCIInputRadiusKey) - - guard let edgeImage = edgeFilter?.outputImage else { continue } - - // Detect rectangles in edge image - var rectangles: [VNRectangleObservation] = [] - let request = VNDetectRectanglesRequest() - request.minimumAspectRatio = 0.3 - request.maximumAspectRatio = 0.5 - request.minimumSize = 0.4 - request.maximumObservations = 1 - - let handler = VNImageRequestHandler(ciImage: edgeImage) - try? handler.perform([request]) - - if let results = request.results as? [VNRectangleObservation] { - for observation in results { - if validateGridDimensions(observation.boundingBox, squareSize: rect.width / 8) { - gridMatches.append(observation.boundingBox) - } - } - } + // Convert Vision coordinates to CoreImage coordinates + let imageSize = image.extent.size + let transform = CGAffineTransform(scaleX: imageSize.width, y: imageSize.height) + + // Create normalized rect in CoreImage coordinate space + let detectedRect = bestObservation.boundingBox.applying(transform) + + // Validate the detected rectangle + guard validateDetectedRect(detectedRect, in: image) else { + return nil } - return gridMatches + return detectedRect } - private func validateGridDimensions(_ rect: CGRect, squareSize: CGFloat) -> Bool { - // Check if dimensions match 8x8 grid with some tolerance - let expectedSize = squareSize * 8 - let tolerance: CGFloat = 0.3 // More flexible tolerance + private func validateDetectedRect(_ rect: CGRect, in image: CIImage) -> Bool { + let imageSize = image.extent.size - // Only validate width since height detection is unreliable - let widthMatch = abs(rect.width - expectedSize) <= (expectedSize * tolerance) - - return widthMatch - } - - private func validateCorners(in image: CIImage, candidates: [CGRect]) -> CGRect? { - var bestMatch: (rect: CGRect, score: Float)? - - for rect in candidates { - var totalScore: Float = 0 - - // Extract corners - let corners = [ - CGPoint(x: rect.minX, y: rect.minY), - CGPoint(x: rect.maxX, y: rect.minY), - CGPoint(x: rect.minX, y: rect.maxY), - CGPoint(x: rect.maxX, y: rect.maxY) - ] - - // Compare with pattern corners - for pattern in patterns { - var patternScore: Float = 0 - - for (corner, patternCorner) in zip(corners, pattern.cornerPattern) { - let distance = hypot( - corner.x - patternCorner.x, - corner.y - patternCorner.y - ) - patternScore += Float(1.0 / (1.0 + distance)) - } - - totalScore = max(totalScore, patternScore / Float(corners.count)) - } - - // Update best match if score is higher - if totalScore > (bestMatch?.score ?? 0) { - bestMatch = (rect, totalScore) - } + // Check if rectangle is within image bounds + guard image.extent.contains(rect) else { + return false } - return bestMatch?.rect + // Validate aspect ratio (standard chess board is square) + let aspectRatio = rect.width / rect.height + guard aspectRatio >= 0.9 && aspectRatio <= 1.1 else { + return false + } + + // Validate size relative to image + let minDimension = min(imageSize.width, imageSize.height) + let boardSize = max(rect.width, rect.height) + guard boardSize >= minDimension * 0.4 else { + return false + } + + return true } + } diff --git a/ChessPrism/ChessPrism/ContentView.swift b/ChessPrism/ChessPrism/ContentView.swift index af4ebd4..291a64c 100644 --- a/ChessPrism/ChessPrism/ContentView.swift +++ b/ChessPrism/ChessPrism/ContentView.swift @@ -2,21 +2,16 @@ import SwiftUI struct ContentView: View { @StateObject private var viewModel = ScreenCaptureViewModel() + @State private var showDebugOverlay = false var body: some View { - VStack(spacing: 20) { - // Screen Capture Controls + VStack { + // Capture controls HStack { Button(action: { - Task { - await viewModel.startCapture() - } + viewModel.startCapture() }) { Text("Start Capture") - .padding() - .background(viewModel.isCapturing ? Color.gray : Color.blue) - .foregroundColor(.white) - .cornerRadius(8) } .disabled(viewModel.isCapturing) @@ -24,112 +19,59 @@ struct ContentView: View { viewModel.stopCapture() }) { Text("Stop Capture") - .padding() - .background(!viewModel.isCapturing ? Color.gray : Color.red) - .foregroundColor(.white) - .cornerRadius(8) } .disabled(!viewModel.isCapturing) } + .padding() - // Board Detection Visualization - if let rect = viewModel.detectedBoardRect { - GeometryReader { geometry in - Rectangle() - .stroke(Color.green, lineWidth: 2) - .frame( - width: rect.width * geometry.size.width, - height: rect.height * geometry.size.height - ) - .position( - x: rect.midX * geometry.size.width, - y: rect.midY * geometry.size.height - ) - } - .background(Color.black.opacity(0.1)) - } - - // Error Display + // Error display if let error = viewModel.captureError { - VStack(spacing: 10) { - Text(error.localizedDescription) - .font(.headline) - .foregroundColor(.red) - .multilineTextAlignment(.center) - - if case .chessWindowNotFound = error { - VStack(alignment: .leading, spacing: 5) { - Text("Requirements:") - .font(.subheadline) - .bold() - Text("• Chess.com app or Firefox must be running") - Text("• A chess game must be open") - Text("• The game window must be visible") - } - .padding() - .background(Color.red.opacity(0.1)) - .cornerRadius(8) - } - } - .padding() - .frame(maxWidth: .infinity) + Text(error.localizedDescription) + .foregroundColor(.red) + .padding() } - // Image Display Section - HStack(spacing: 20) { - // Full Capture Display - if let image = viewModel.capturedImage { - VStack { - Text("Full Capture") - .font(.caption) + // Image display + HStack { + VStack { + Text("Full Capture") + if let image = viewModel.capturedImage { Image(nsImage: image) .resizable() .aspectRatio(contentMode: .fit) - .frame(height: 400) // Fixed height to show full board + .frame(maxWidth: 400) + } else { + Text("No capture available") + .foregroundColor(.gray) } - .padding() - .background(Color.black.opacity(0.05)) - .cornerRadius(8) } - // Cropped Board Display - if let cropped = viewModel.croppedBoardImage { - VStack { - Text("Chessboard") - .font(.caption) - Image(nsImage: cropped) + Divider() + + VStack { + Text("Chessboard Preview") + if let image = viewModel.croppedBoardImage { + Image(nsImage: image) .resizable() .aspectRatio(contentMode: .fit) - .frame(height: 400) // Match the board height + .frame(maxWidth: 400) + .aspectRatio(contentMode: .fit) + } else { + Text("No board detected") + .foregroundColor(.gray) } - .padding() - .background(Color.black.opacity(0.05)) - .cornerRadius(8) } } .padding() - // Vision Model Output Placeholder - if viewModel.croppedBoardImage != nil { - VStack(alignment: .leading) { - Text("Board Analysis") - .font(.headline) - Text("FEN: (processing...)") - .font(.system(.body, design: .monospaced)) - } - .padding() - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.blue.opacity(0.1)) - .cornerRadius(8) - } - Spacer() } - .padding() - .frame(minWidth: 800, minHeight: 600) // Increased window size to fit larger board display + .frame(minWidth: 800, minHeight: 600) } } -#Preview { - ContentView() +struct ContentView_Previews: PreviewProvider { + static var previews: some View { + ContentView() + } } diff --git a/ChessPrism/ChessPrism/ScreenCapture.swift b/ChessPrism/ChessPrism/ScreenCapture.swift index 4a365a5..95aa85d 100644 --- a/ChessPrism/ChessPrism/ScreenCapture.swift +++ b/ChessPrism/ChessPrism/ScreenCapture.swift @@ -1,290 +1,64 @@ import Foundation +import AppKit +import CoreGraphics +import CoreMedia import ScreenCaptureKit -import Vision -import CoreImage +import AVFoundation -class ScreenCapture: NSObject { - private var captureSession: SCStream? - private let queue = DispatchQueue(label: "com.chessprism.screencapture", qos: .userInteractive) - private var lastDetectedCoordinates: [(text: String, boundingBox: CGRect)] = [] - private let boardDetector = BoardDetector() - - // MARK: - Screen Capture Setup - func startCapture() async throws { - print("Starting screen capture...") - // Get available screen content - let availableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) - print("Available content: \(availableContent.displays.count) displays, \(availableContent.windows.count) windows") - - // Find Chess.com app window - let chessWindow = availableContent.windows.first { window in - if let bundleId = window.owningApplication?.bundleIdentifier { - return bundleId == "com.chess.iphone" - } - return false - } - - guard let chessWindow = chessWindow else { - throw ScreenCaptureError.chessWindowNotFound - } - - // Create stream configuration targeting chess window - let filter = SCContentFilter(desktopIndependentWindow: chessWindow) - let configuration = SCStreamConfiguration() - configuration.width = 1920 - configuration.height = 1080 - configuration.pixelFormat = kCVPixelFormatType_32BGRA - configuration.queueDepth = 5 - - // Create and start stream - captureSession = SCStream(filter: filter, configuration: configuration, delegate: nil) - - try await captureSession?.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue) - try await captureSession?.startCapture() - - print("Screen capture started successfully") - NotificationCenter.default.post( - name: .captureStateChanged, - object: nil, - userInfo: ["isCapturing": true] - ) - } - - // MARK: - Frame Processing - private func processFrame(sampleBuffer: CMSampleBuffer) { - guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } - let ciImage = CIImage(cvPixelBuffer: pixelBuffer) - - // Convert CIImage to NSImage for display - let context = CIContext() - if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) { - let size = NSSize(width: cgImage.width, height: cgImage.height) - let nsImage = NSImage(cgImage: cgImage, size: size) - - // Notify subscribers with captured image - NotificationCenter.default.post( - name: .capturedFrame, - object: nil, - userInfo: ["image": nsImage] - ) - } - - // Try pattern-based detection first - if let boardRect = boardDetector.detectBoard(in: ciImage) { - // Add padding to ensure full board capture - let padding = boardRect.width * 0.25 // 25% padding - let paddedRect = CGRect( - x: boardRect.minX - padding, - y: boardRect.minY - padding, - width: boardRect.width + (padding * 2), - height: boardRect.height + (padding * 2) - ) - NotificationCenter.default.post( - name: .boardDetected, - object: nil, - userInfo: ["rect": paddedRect] - ) +class ScreenCapture: NSObject, SCStreamOutput { + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } - // Fallback to coordinate-based detection - let textRequest = VNRecognizeTextRequest { [weak self] request, error in - guard let self = self else { return } - self.handleTextDetection(request: request) - } - textRequest.recognitionLevel = .accurate - textRequest.usesLanguageCorrection = false - - let rectangleRequest = VNDetectRectanglesRequest { [weak self] request, error in - guard let self = self else { return } - self.handleBoardDetection(request: request) - } - rectangleRequest.minimumAspectRatio = 0.8 - rectangleRequest.maximumAspectRatio = 1.2 - rectangleRequest.minimumSize = 0.2 - rectangleRequest.maximumObservations = 3 - - let handler = VNImageRequestHandler(ciImage: ciImage) - try? handler.perform([textRequest, rectangleRequest]) - } - - private func handleTextDetection(request: VNRequest) { - guard let results = request.results as? [VNRecognizedTextObservation] else { return } - - // Filter for chess coordinates (a-h, 1-8) - var coordinates: [(text: String, boundingBox: CGRect)] = [] - for observation in results { - if let candidate = observation.topCandidates(1).first { - let text = candidate.string - if (text.count == 1 && ("a"..."h").contains(text.lowercased())) || - (text.count == 1 && ("1"..."8").contains(text)) { - coordinates.append((text: text, boundingBox: observation.boundingBox)) - } - } - } - - // Store coordinates and notify observers - if !coordinates.isEmpty { - lastDetectedCoordinates = coordinates - NotificationCenter.default.post( - name: .boardCoordinatesDetected, - object: nil, - userInfo: ["coordinates": coordinates] - ) + DispatchQueue.main.async { + let ciImage = CIImage(cvPixelBuffer: pixelBuffer) + let rep = NSCIImageRep(ciImage: ciImage) + let nsImage = NSImage(size: rep.size) + nsImage.addRepresentation(rep) + + self.lastCapturedImage = nsImage } } - // MARK: - Board Detection Handling - private func handleBoardDetection(request: VNRequest) { - guard let results = request.results as? [VNRectangleObservation], - let boardRect = results.first else { return } + private var activeStream: SCStream? + private var lastCapturedImage: NSImage? + + func startCapture() async throws { + guard #available(macOS 12.3, *) else { return } - // Convert rectangle to screen coordinates - var transformedRect = transformRectangle(boardRect) + let content = try await SCShareableContent.current - // Use the stored coordinates - let coordinates = lastDetectedCoordinates - - // If we have coordinates, use them to refine the rectangle - if !coordinates.isEmpty { - // Separate horizontal and vertical coordinates - let horizontalCoords = coordinates.filter { ("a"..."h").contains($0.text.lowercased()) } - let verticalCoords = coordinates.filter { ("1"..."8").contains($0.text) } - - // Get min/max for horizontal coordinates (a-h) - let xCoords = horizontalCoords.map { $0.boundingBox.origin.x } - let minX = xCoords.min() ?? transformedRect.minX - let maxX = xCoords.max() ?? transformedRect.maxX - - // Get min/max for vertical coordinates (1-8) - let yCoords = verticalCoords.map { $0.boundingBox.origin.y } - let minY = yCoords.min() ?? transformedRect.minY - let maxY = yCoords.max() ?? transformedRect.maxY - - // Calculate board boundaries directly from coordinates - let leftMost = horizontalCoords.min { $0.boundingBox.origin.x < $1.boundingBox.origin.x }?.boundingBox.origin.x ?? minX - let rightMost = horizontalCoords.max { $0.boundingBox.origin.x < $1.boundingBox.origin.x }?.boundingBox.origin.x ?? maxX - let bottomMost = verticalCoords.min { $0.boundingBox.origin.y < $1.boundingBox.origin.y }?.boundingBox.origin.y ?? minY - let topMost = verticalCoords.max { $0.boundingBox.origin.y < $1.boundingBox.origin.y }?.boundingBox.origin.y ?? maxY - - // Calculate board dimensions directly from coordinate positions - let boardWidth = rightMost - leftMost - let boardHeight = topMost - bottomMost - - // Find coordinate pairs that are likely on opposite sides of the board - let horizontalPairs = horizontalCoords.flatMap { h1 in - horizontalCoords.map { h2 in - (h1, h2, abs(h1.boundingBox.origin.x - h2.boundingBox.origin.x)) - } - }.sorted { $0.2 > $1.2 } - - let verticalPairs = verticalCoords.flatMap { v1 in - verticalCoords.map { v2 in - (v1, v2, abs(v1.boundingBox.origin.y - v2.boundingBox.origin.y)) - } - }.sorted { $0.2 > $1.2 } - - // Use the most distant pairs to define board boundaries - if let widestHorizontal = horizontalPairs.first, - let tallestVertical = verticalPairs.first { - - // Calculate board dimensions using the most distant coordinates - let boardWidth = widestHorizontal.2 - let boardHeight = tallestVertical.2 - - // Calculate the center of the board - let centerX = (widestHorizontal.0.boundingBox.origin.x + widestHorizontal.1.boundingBox.origin.x) / 2 - let centerY = (tallestVertical.0.boundingBox.origin.y + tallestVertical.1.boundingBox.origin.y) / 2 - - // Add padding proportional to the coordinate spacing - let padding = max(boardWidth, boardHeight) * 0.25 // 25% padding - - // Set rectangle to centered board with padding - transformedRect = CGRect( - x: centerX - (boardWidth / 2) - padding, - y: centerY - (boardHeight / 2) - padding, - width: boardWidth + (padding * 2), - height: boardHeight + (padding * 2) - ) - } - - // Ensure we capture full board with reasonable bounds before aspect ratio adjustment - transformedRect.origin.x = max(-0.1, transformedRect.origin.x) // Allow 10% outside bounds - transformedRect.origin.y = max(-0.1, transformedRect.origin.y) - transformedRect.size.width = min(1.2 - transformedRect.origin.x, transformedRect.size.width) // Allow 20% overflow - transformedRect.size.height = min(1.2 - transformedRect.origin.y, transformedRect.size.height) - - // Apply single aspect ratio adjustment with flexible tolerance - let targetAspectRatio: CGFloat = 1.0 - let currentAspectRatio = transformedRect.size.width / transformedRect.size.height - if abs(currentAspectRatio - targetAspectRatio) > 0.2 { - if currentAspectRatio > targetAspectRatio { - let newHeight = transformedRect.size.width - let heightDiff = newHeight - transformedRect.size.height - transformedRect.origin.y -= heightDiff / 2.0 - transformedRect.size.height = newHeight - } else { - let newWidth = transformedRect.size.height - let widthDiff = newWidth - transformedRect.size.width - transformedRect.origin.x -= widthDiff / 2.0 - transformedRect.size.width = newWidth - } - } + // Get the window from the content + guard let window = content.windows.first(where: { window in + let bundleID = window.owningApplication?.bundleIdentifier ?? "" + let isChessApp = bundleID == "com.chess.iphone" + let hasValidSize = window.frame.width > 100 && window.frame.height > 100 + return isChessApp && window.isOnScreen && hasValidSize + }) else { + throw NSError(domain: "ChessPrism", code: 1, userInfo: [NSLocalizedDescriptionKey: "Chess window not found"]) } - // Notify subscribers - NotificationCenter.default.post( - name: .boardDetected, - object: nil, - userInfo: ["rect": transformedRect] - ) + let filter = SCContentFilter(desktopIndependentWindow: window) + let config = SCStreamConfiguration() + config.width = Int(window.frame.width) + config.height = Int(window.frame.height) + + let stream = SCStream(filter: filter, configuration: config, delegate: nil) + try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: DispatchQueue.global(qos: .userInitiated)) + + // Store the stream before starting it + activeStream = stream + try await stream.startCapture() } - // MARK: - Coordinate Transformation - private func transformRectangle(_ rect: VNRectangleObservation) -> CGRect { - // Vision coordinates are in normalized coordinates with origin at bottom-left - // We need to convert to top-left origin coordinates - return CGRect( - x: rect.boundingBox.origin.x, - y: 1 - rect.boundingBox.origin.y - rect.boundingBox.height, - width: rect.boundingBox.width, - height: rect.boundingBox.height - ) + func stopCapture() async throws { + guard let stream = activeStream else { return } + try await stream.stopCapture() + activeStream = nil } - // MARK: - Capture Control - func stopCapture() { - print("Stopping screen capture...") - captureSession?.stopCapture() - captureSession = nil - NotificationCenter.default.post( - name: .captureStateChanged, - object: nil, - userInfo: ["isCapturing": false] - ) - } - - // MARK: - Error Handling - enum ScreenCaptureError: Error { - case chessWindowNotFound - case streamCreationFailed - case permissionDenied - case captureAlreadyRunning + func getCurrentImage() -> NSImage? { + return lastCapturedImage } } - -// MARK: - SCStreamOutput -extension ScreenCapture: SCStreamOutput { - func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { - guard type == .screen else { return } - processFrame(sampleBuffer: sampleBuffer) - } -} - -// MARK: - Notifications -extension Notification.Name { - static let boardDetected = Notification.Name("com.chessprism.boardDetected") - static let captureStateChanged = Notification.Name("com.chessprism.captureStateChanged") - static let capturedFrame = Notification.Name("com.chessprism.capturedFrame") - static let boardCoordinatesDetected = Notification.Name("com.chessprism.boardCoordinatesDetected") -} diff --git a/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift b/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift index 8de201e..9c9adb1 100644 --- a/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift +++ b/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift @@ -1,157 +1,120 @@ import Foundation -import SwiftUI +import AppKit +import CoreImage import Combine @MainActor class ScreenCaptureViewModel: ObservableObject { - // MARK: - Published Properties + @Published var capturedImage: NSImage? + @Published var croppedBoardImage: NSImage? + @Published var captureError: CaptureError? @Published var isCapturing = false - @Published var captureError: ScreenCaptureError? = nil - @Published var detectedBoardRect: CGRect? = nil - @Published var capturedImage: NSImage? = nil - @Published var croppedBoardImage: NSImage? = nil - // MARK: - Dependencies private let screenCapture = ScreenCapture() - private var cancellables = Set() + private let boardDetector = BoardDetector() + private var captureTask: Task? + // Share CIContext to avoid creating too many Metal command queues + private static let shared = CIContext() + private var context: CIContext { ScreenCaptureViewModel.shared } - // MARK: - Initialization - init() { - setupObservers() + private var detectedBoardRect: CGRect? + + enum CaptureError: LocalizedError { + case chessWindowNotFound + case boardDetectionFailed + + var errorDescription: String? { + switch self { + case .chessWindowNotFound: + return "Chess window not found" + case .boardDetectionFailed: + return "Failed to detect chess board" + } + } } - // MARK: - Screen Capture Management - func startCapture() async { - guard await requestScreenCapturePermission() else { - captureError = .permissionDenied - return - } + func startCapture() { + isCapturing = true + captureError = nil - do { - try await screenCapture.startCapture() - isCapturing = true - } catch let error as ScreenCapture.ScreenCaptureError { - captureError = ScreenCaptureError(from: error) - } catch { - captureError = .unknown(error) + captureTask = Task { + do { + // Start continuous capture + try await screenCapture.startCapture() + + captureLoop: while !Task.isCancelled { + do { + if let image = screenCapture.getCurrentImage() { + try await processImage(image) + } + try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds + } catch is CancellationError { + break captureLoop + } catch { + // Just update the error state but continue capturing + if !Task.isCancelled { + handleCaptureError(error) + } + } + } + + // Clean up + try? await screenCapture.stopCapture() + if isCapturing { + isCapturing = false + } + } catch { + handleCaptureError(error) + isCapturing = false + } } } func stopCapture() { - screenCapture.stopCapture() + captureTask?.cancel() + captureTask = nil isCapturing = false + captureError = nil } - // MARK: - Permission Handling - private func requestScreenCapturePermission() async -> Bool { - return await withCheckedContinuation { continuation in - let status = CGPreflightScreenCaptureAccess() - if status { - continuation.resume(returning: true) - } else { - CGRequestScreenCaptureAccess() - continuation.resume(returning: false) - } - } - } - - // MARK: - Notification Handling - private func setupObservers() { - // Observe board detection - NotificationCenter.default - .publisher(for: .boardDetected) - .receive(on: DispatchQueue.main) - .sink(receiveValue: { [weak self] notification in - guard let rect = notification.userInfo?["rect"] as? CGRect else { return } - self?.detectedBoardRect = rect - }) - .store(in: &cancellables) - - // Observe capture state changes - NotificationCenter.default - .publisher(for: .captureStateChanged) - .receive(on: DispatchQueue.main) - .sink(receiveValue: { [weak self] notification in - guard let isCapturing = notification.userInfo?["isCapturing"] as? Bool else { return } - self?.isCapturing = isCapturing - }) - .store(in: &cancellables) - - // Observe captured frames - NotificationCenter.default - .publisher(for: .capturedFrame) - .receive(on: DispatchQueue.main) - .sink(receiveValue: { [weak self] notification in - guard let image = notification.userInfo?["image"] as? NSImage, - let self = self else { return } - - self.capturedImage = image - - // Crop to detected board area if available - if let rect = self.detectedBoardRect { - let cropped = self.cropImage(image, to: rect) - self.croppedBoardImage = cropped - } - }) - .store(in: &cancellables) - } - - // MARK: - Error Handling - enum ScreenCaptureError: Error, LocalizedError { - case permissionDenied - case chessWindowNotFound - case streamCreationFailed - case captureAlreadyRunning - case unknown(Error) - - init(from error: ScreenCapture.ScreenCaptureError) { - switch error { - case .chessWindowNotFound: - self = .chessWindowNotFound - case .streamCreationFailed: - self = .streamCreationFailed - case .permissionDenied: - self = .permissionDenied - case .captureAlreadyRunning: - self = .captureAlreadyRunning - } - } - - var errorDescription: String? { - switch self { - case .permissionDenied: - return "Screen recording permission is required" - case .chessWindowNotFound: - return "Unable to find Chess window (make sure Chess.com app or Firefox is open)" - case .streamCreationFailed: - return "Failed to create screen capture stream" - case .captureAlreadyRunning: - return "Capture is already running" - case .unknown(let error): - return "An unexpected error occurred: \(error.localizedDescription)" - } - } - } - - private func cropImage(_ image: NSImage, to rect: CGRect) -> NSImage? { + private func processImage(_ image: NSImage) async throws { + // Convert to CIImage for processing guard let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else { - return nil + throw CaptureError.boardDetectionFailed } - // Convert normalized rect to pixel coordinates - let width = CGFloat(cgImage.width) - let height = CGFloat(cgImage.height) - let cropRect = CGRect( - x: rect.origin.x * width, - y: (1 - rect.origin.y - rect.height) * height, // Flip Y coordinate since NSImage uses bottom-left origin - width: rect.width * width, - height: rect.height * height - ) + let ciImage = CIImage(cgImage: cgImage) - guard let croppedCGImage = cgImage.cropping(to: cropRect) else { - return nil + // Detect board + guard let boardRect = boardDetector.detectBoard(in: ciImage) else { + throw CaptureError.boardDetectionFailed } - return NSImage(cgImage: croppedCGImage, size: cropRect.size) + // Update detected rectangle + self.detectedBoardRect = boardRect + + // Crop board image + let croppedImage = ciImage.cropped(to: boardRect) + updateImages(ciImage: ciImage, croppedImage: croppedImage) + } + + private func updateImages(ciImage: CIImage, croppedImage: CIImage) { + if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) { + let capturedNSImage = NSImage(cgImage: cgImage, size: .zero) + self.capturedImage = capturedNSImage + } + + if let cgCroppedImage = context.createCGImage(croppedImage, from: croppedImage.extent) { + let croppedNSImage = NSImage(cgImage: cgCroppedImage, size: .zero) + self.croppedBoardImage = croppedNSImage + } + } + + private func handleCaptureError(_ error: Error) { + if let captureError = error as? CaptureError { + self.captureError = captureError + } else { + self.captureError = .boardDetectionFailed + } } } diff --git a/cline_docs/Info.txt b/cline_docs/Info.txt new file mode 100644 index 0000000..7f03501 --- /dev/null +++ b/cline_docs/Info.txt @@ -0,0 +1,571 @@ + +Command queue creation failed. Worst processes ( + { + Count = 2017; + Process = "pid 25345, ChessPrism"; + }, + { + Count = 4; + Process = "pid 1227, firefox"; + }, + { + Count = 3; + Process = "pid 579, WindowServer"; + }, + { + Count = 3; + Process = "pid 1926, Creative Cloud U"; + }, + { + Count = 2; + Process = "pid 1754, Ollama Helper (G"; + }, + { + Count = 2; + Process = "pid 1928, LogiTune Helper "; + }, + { + Count = 2; + Process = "pid 11425, Xcode"; + }, + { + Count = 1; + Process = "pid 1062, mediaanalysisd"; + }, + { + Count = 1; + Process = "pid 1101, NotificationCent"; + }, + { + Count = 1; + Process = "pid 582, loginwindow"; + }, + { + Count = 1; + Process = "pid 1242, Finder"; + }, + { + Count = 1; + Process = "pid 1100, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 1237, Notes"; + }, + { + Count = 1; + Process = "pid 1222, Terminal"; + }, + { + Count = 1; + Process = "pid 1867, Docker Desktop H"; + }, + { + Count = 1; + Process = "pid 1970, Logi AI Prompt B"; + }, + { + Count = 1; + Process = "pid 1040, ControlCenter"; + }, + { + Count = 1; + Process = "pid 905, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 7637, ControlCenterHel"; + }, + { + Count = 1; + Process = "pid 15435, Code Helper (GPU"; + }, + { + Count = 0; + Process = "pid 1208, avconferenced"; + }, + { + Count = 0; + Process = "pid 1228, VTDecoderXPCServ"; + }, + { + Count = 0; + Process = "pid 1248, Spotlight"; + }, + { + Count = 0; + Process = "pid 1186, replayd"; + }, + { + Count = 0; + Process = "pid 1459, naturallanguaged"; + }, + { + Count = 0; + Process = "pid 577, com.apple.cmio.r"; + }, + { + Count = 0; + Process = "pid 2216, VTEncoderXPCServ"; + }, + { + Count = 0; + Process = "pid 7667, QuickLookUIServi"; + } +) +Command queue creation failed. Worst processes ( + { + Count = 2017; + Process = "pid 25345, ChessPrism"; + }, + { + Count = 4; + Process = "pid 1227, firefox"; + }, + { + Count = 3; + Process = "pid 579, WindowServer"; + }, + { + Count = 3; + Process = "pid 1926, Creative Cloud U"; + }, + { + Count = 2; + Process = "pid 1754, Ollama Helper (G"; + }, + { + Count = 2; + Process = "pid 1928, LogiTune Helper "; + }, + { + Count = 2; + Process = "pid 11425, Xcode"; + }, + { + Count = 1; + Process = "pid 1062, mediaanalysisd"; + }, + { + Count = 1; + Process = "pid 1101, NotificationCent"; + }, + { + Count = 1; + Process = "pid 582, loginwindow"; + }, + { + Count = 1; + Process = "pid 1242, Finder"; + }, + { + Count = 1; + Process = "pid 1100, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 1237, Notes"; + }, + { + Count = 1; + Process = "pid 1222, Terminal"; + }, + { + Count = 1; + Process = "pid 1867, Docker Desktop H"; + }, + { + Count = 1; + Process = "pid 1970, Logi AI Prompt B"; + }, + { + Count = 1; + Process = "pid 1040, ControlCenter"; + }, + { + Count = 1; + Process = "pid 905, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 7637, ControlCenterHel"; + }, + { + Count = 1; + Process = "pid 15435, Code Helper (GPU"; + }, + { + Count = 0; + Process = "pid 1208, avconferenced"; + }, + { + Count = 0; + Process = "pid 1228, VTDecoderXPCServ"; + }, + { + Count = 0; + Process = "pid 1248, Spotlight"; + }, + { + Count = 0; + Process = "pid 1186, replayd"; + }, + { + Count = 0; + Process = "pid 1459, naturallanguaged"; + }, + { + Count = 0; + Process = "pid 577, com.apple.cmio.r"; + }, + { + Count = 0; + Process = "pid 2216, VTEncoderXPCServ"; + }, + { + Count = 0; + Process = "pid 7667, QuickLookUIServi"; + } +) +Command queue creation failed. Worst processes ( + { + Count = 2017; + Process = "pid 25345, ChessPrism"; + }, + { + Count = 4; + Process = "pid 1227, firefox"; + }, + { + Count = 3; + Process = "pid 579, WindowServer"; + }, + { + Count = 3; + Process = "pid 1926, Creative Cloud U"; + }, + { + Count = 2; + Process = "pid 1754, Ollama Helper (G"; + }, + { + Count = 2; + Process = "pid 1928, LogiTune Helper "; + }, + { + Count = 2; + Process = "pid 11425, Xcode"; + }, + { + Count = 1; + Process = "pid 1062, mediaanalysisd"; + }, + { + Count = 1; + Process = "pid 1101, NotificationCent"; + }, + { + Count = 1; + Process = "pid 582, loginwindow"; + }, + { + Count = 1; + Process = "pid 1242, Finder"; + }, + { + Count = 1; + Process = "pid 1100, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 1237, Notes"; + }, + { + Count = 1; + Process = "pid 1222, Terminal"; + }, + { + Count = 1; + Process = "pid 1867, Docker Desktop H"; + }, + { + Count = 1; + Process = "pid 1970, Logi AI Prompt B"; + }, + { + Count = 1; + Process = "pid 1040, ControlCenter"; + }, + { + Count = 1; + Process = "pid 905, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 7637, ControlCenterHel"; + }, + { + Count = 1; + Process = "pid 15435, Code Helper (GPU"; + }, + { + Count = 0; + Process = "pid 1208, avconferenced"; + }, + { + Count = 0; + Process = "pid 1228, VTDecoderXPCServ"; + }, + { + Count = 0; + Process = "pid 1248, Spotlight"; + }, + { + Count = 0; + Process = "pid 1186, replayd"; + }, + { + Count = 0; + Process = "pid 1459, naturallanguaged"; + }, + { + Count = 0; + Process = "pid 577, com.apple.cmio.r"; + }, + { + Count = 0; + Process = "pid 2216, VTEncoderXPCServ"; + }, + { + Count = 0; + Process = "pid 7667, QuickLookUIServi"; + } +) +Command queue creation failed. Worst processes ( + { + Count = 2017; + Process = "pid 25345, ChessPrism"; + }, + { + Count = 4; + Process = "pid 1227, firefox"; + }, + { + Count = 3; + Process = "pid 579, WindowServer"; + }, + { + Count = 3; + Process = "pid 1926, Creative Cloud U"; + }, + { + Count = 2; + Process = "pid 1754, Ollama Helper (G"; + }, + { + Count = 2; + Process = "pid 1928, LogiTune Helper "; + }, + { + Count = 2; + Process = "pid 11425, Xcode"; + }, + { + Count = 1; + Process = "pid 1062, mediaanalysisd"; + }, + { + Count = 1; + Process = "pid 1101, NotificationCent"; + }, + { + Count = 1; + Process = "pid 582, loginwindow"; + }, + { + Count = 1; + Process = "pid 1242, Finder"; + }, + { + Count = 1; + Process = "pid 1100, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 1237, Notes"; + }, + { + Count = 1; + Process = "pid 1222, Terminal"; + }, + { + Count = 1; + Process = "pid 1867, Docker Desktop H"; + }, + { + Count = 1; + Process = "pid 1970, Logi AI Prompt B"; + }, + { + Count = 1; + Process = "pid 1040, ControlCenter"; + }, + { + Count = 1; + Process = "pid 905, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 7637, ControlCenterHel"; + }, + { + Count = 1; + Process = "pid 15435, Code Helper (GPU"; + }, + { + Count = 0; + Process = "pid 1208, avconferenced"; + }, + { + Count = 0; + Process = "pid 1228, VTDecoderXPCServ"; + }, + { + Count = 0; + Process = "pid 1248, Spotlight"; + }, + { + Count = 0; + Process = "pid 1186, replayd"; + }, + { + Count = 0; + Process = "pid 1459, naturallanguaged"; + }, + { + Count = 0; + Process = "pid 577, com.apple.cmio.r"; + }, + { + Count = 0; + Process = "pid 2216, VTEncoderXPCServ"; + }, + { + Count = 0; + Process = "pid 7667, QuickLookUIServi"; + } +) +Command queue creation failed. Worst processes ( + { + Count = 2017; + Process = "pid 25345, ChessPrism"; + }, + { + Count = 4; + Process = "pid 1227, firefox"; + }, + { + Count = 3; + Process = "pid 579, WindowServer"; + }, + { + Count = 3; + Process = "pid 1926, Creative Cloud U"; + }, + { + Count = 2; + Process = "pid 1754, Ollama Helper (G"; + }, + { + Count = 2; + Process = "pid 1928, LogiTune Helper "; + }, + { + Count = 2; + Process = "pid 11425, Xcode"; + }, + { + Count = 1; + Process = "pid 1062, mediaanalysisd"; + }, + { + Count = 1; + Process = "pid 1101, NotificationCent"; + }, + { + Count = 1; + Process = "pid 582, loginwindow"; + }, + { + Count = 1; + Process = "pid 1242, Finder"; + }, + { + Count = 1; + Process = "pid 1100, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 1237, Notes"; + }, + { + Count = 1; + Process = "pid 1222, Terminal"; + }, + { + Count = 1; + Process = "pid 1867, Docker Desktop H"; + }, + { + Count = 1; + Process = "pid 1970, Logi AI Prompt B"; + }, + { + Count = 1; + Process = "pid 1040, ControlCenter"; + }, + { + Count = 1; + Process = "pid 905, iconservicesagen"; + }, + { + Count = 1; + Process = "pid 7637, ControlCenterHel"; + }, + { + Count = 1; + Process = "pid 15435, Code Helper (GPU"; + }, + { + Count = 0; + Process = "pid 1208, avconferenced"; + }, + { + Count = 0; + Process = "pid 1228, VTDecoderXPCServ"; + }, + { + Count = 0; + Process = "pid 1248, Spotlight"; + }, + { + Count = 0; + Process = "pid 1186, replayd"; + }, + { + Count = 0; + Process = "pid 1459, naturallanguaged"; + }, + { + Count = 0; + Process = "pid 577, com.apple.cmio.r"; + }, + { + Count = 0; + Process = "pid 2216, VTEncoderXPCServ"; + }, + { + Count = 0; + Process = "pid 7667, QuickLookUIServi"; + } +) \ No newline at end of file diff --git a/cline_docs/activeContext.md b/cline_docs/activeContext.md index 67f7300..73146f4 100644 --- a/cline_docs/activeContext.md +++ b/cline_docs/activeContext.md @@ -1,50 +1,74 @@ # Active Context ## Current Status -- Screen capture module implemented -- Pattern recognition-based board detection implemented -- Coordinate-based detection retained as fallback -- SwiftUI interface for capture controls added -- Proper resource cleanup implemented -- Error handling system in place +- Screen capture module successfully implemented +- Window detection and capture working correctly +- SwiftUI interface with capture controls functioning +- Error handling system properly managing states +- Resource cleanup implemented +- Metal resource management optimized ## Recent Changes -- Refined pattern recognition approach for chess board detection: - 1. Modified rectangle detection parameters: - - Using 0.3-0.5 aspect ratio for taller rectangles - - Increased minimum size to 0.4 - - Single observation for precision - 2. Improved board extraction: - - Using detected width as reference - - Positioning square board at top of detected area - - Using offsetBy for vertical positioning - 3. Simplified coordinate handling: - - Direct rectangle detection with Vision framework - - Proper coordinate system transformations - - Maintained pattern matching for accuracy +1. Resource Management Optimization: + - Implemented shared CIContext pattern: + * Prevents command queue exhaustion + * Reduces Metal resource usage + * Enables long-running captures + - Proper cleanup on task completion + - Efficient resource utilization -## Current Challenges -- Board detection still not capturing full height -- Need to investigate if issue is with: - 1. Detection parameters - 2. Coordinate transformations - 3. View rendering constraints - 4. Window/display configuration +2. Improved Window Detection: + - Using SCShareableContent for window access + - Precise window identification: + * Exact bundle ID matching (com.chess.iphone) + * Window visibility verification (isOnScreen) + * Size validation (width > 100 && height > 100) + +3. Enhanced Capture System: + - Continuous capture implementation: + * Single persistent capture stream + * Smooth frame processing (no flickering) + * Efficient resource usage + - Proper frame dimensions from window + - Clean task cancellation handling + - Main thread safety for UI updates + +4. UI Simplification: + - Removed debug overlay functionality + - Cleaner, focused interface + - Essential controls only: + * Start/Stop capture + * Full capture view + * Board preview + +5. Error Handling: + - Improved error resilience: + * Continues capturing even if board detection fails + * Only stops on critical errors (e.g., window not found) + * Shows error state without interrupting capture + - Clear error states + - Proper async/await usage + - Task cancellation management + - Thread-safe state updates + +## Current Focus +1. Board Detection: + - Implement chess board recognition + - Handle different board themes + - Process captured frames efficiently ## Next Steps -1. Further refine board detection: - - Test different aspect ratios - - Validate coordinate transformations - - Review view constraints -2. Add board position analysis -3. Implement move detection -4. Create visual overlay system +1. Implement board detection: + - Pattern recognition for chess pieces + - Board coordinate mapping + - Position validation +2. Add position analysis +3. Create move detection system +4. Implement visual overlay 5. Integrate Stockfish engine ## Known Issues -- Board detection showing only bottom portion -- Need to validate pattern detection accuracy -- Need to handle multiple display configurations -- Requires testing with different screen resolutions -- Need to add proper error recovery -- Requires additional permission handling for production +- Board detection not yet implemented +- Need to handle different chess.com themes +- Need to implement piece recognition +- Position analysis pending implementation diff --git a/cline_docs/error b/cline_docs/error index 70c98c6..1bae1d5 100644 --- a/cline_docs/error +++ b/cline_docs/error @@ -1,53 +1,138 @@ -Starting screen capture... -Available content: 1 displays, 32 windows -AddInstanceForFactory: No factory registered for id F8BB1C28-BAE8-11D6-9C31-00039315CD46 -Screen capture started successfully -*** Terminating app due to uncaught exception 'NSUnknownKeyException', reason: '[ valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.' -*** First throw call stack: -( - 0 CoreFoundation 0x0000000189d0ee80 __exceptionPreprocess + 176 - 1 libobjc.A.dylib 0x00000001897f6cd8 objc_exception_throw + 88 - 2 CoreFoundation 0x0000000189d33534 _CFBundleGetValueForInfoKey + 0 - 3 Foundation 0x000000018b6dbd84 -[NSObject(NSKeyValueCoding) valueForUndefinedKey:] + 196 - 4 Foundation 0x000000018ae402fc -[NSObject(NSKeyValueCoding) valueForKey:] + 280 - 5 ChessPrism.debug.dylib 0x0000000100b8f834 $s10ChessPrism13ScreenCaptureC20handleBoardDetection33_6963F6DFCAB5A0A312B3D7446D299968LL7requestySo9VNRequestC_tF + 740 - 6 ChessPrism.debug.dylib 0x0000000100b8e644 $s10ChessPrism13ScreenCaptureC12processFrame33_6963F6DFCAB5A0A312B3D7446D299968LL12sampleBufferySo08CMSampleO3Refa_tFySo9VNRequestC_s5Error_pSgtcfU0_ + 200 - 7 ChessPrism.debug.dylib 0x0000000100b93350 $sSo9VNRequestCs5Error_pSgIeggg_ABSo7NSErrorCSgIeyByy_TR + 140 - 8 Vision 0x000000019f40aab4 -[VNRequest performInContext:error:] + 1244 - 9 Vision 0x000000019f40af08 __73-[VNRequest performInContextAsync:asyncDispatchQueue:asyncDispatchGroup:]_block_invoke + 128 - 10 libdispatch.dylib 0x00000001001d2ac8 _dispatch_block_async_invoke2 + 148 - 11 libdispatch.dylib 0x00000001001be824 _dispatch_client_callout + 20 - 12 libdispatch.dylib 0x00000001001c2350 _dispatch_continuation_pop + 1408 - 13 libdispatch.dylib 0x00000001001c1028 _dispatch_async_redirect_invoke + 616 - 14 libdispatch.dylib 0x00000001001d6b70 _dispatch_root_queue_drain + 404 - 15 libdispatch.dylib 0x00000001001d777c _dispatch_worker_thread2 + 188 - 16 libsystem_pthread.dylib 0x0000000100b090c4 _pthread_wqthread + 228 - 17 libsystem_pthread.dylib 0x0000000100b10cf0 start_wqthread + 8 -) -An uncaught exception was raised -[ valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates. -( - 0 CoreFoundation 0x0000000189d0ee80 __exceptionPreprocess + 176 - 1 libobjc.A.dylib 0x00000001897f6cd8 objc_exception_throw + 88 - 2 CoreFoundation 0x0000000189d33534 _CFBundleGetValueForInfoKey + 0 - 3 Foundation 0x000000018b6dbd84 -[NSObject(NSKeyValueCoding) valueForUndefinedKey:] + 196 - 4 Foundation 0x000000018ae402fc -[NSObject(NSKeyValueCoding) valueForKey:] + 280 - 5 ChessPrism.debug.dylib 0x0000000100b8f834 $s10ChessPrism13ScreenCaptureC20handleBoardDetection33_6963F6DFCAB5A0A312B3D7446D299968LL7requestySo9VNRequestC_tF + 740 - 6 ChessPrism.debug.dylib 0x0000000100b8e644 $s10ChessPrism13ScreenCaptureC12processFrame33_6963F6DFCAB5A0A312B3D7446D299968LL12sampleBufferySo08CMSampleO3Refa_tFySo9VNRequestC_s5Error_pSgtcfU0_ + 200 - 7 ChessPrism.debug.dylib 0x0000000100b93350 $sSo9VNRequestCs5Error_pSgIeggg_ABSo7NSErrorCSgIeyByy_TR + 140 - 8 Vision 0x000000019f40aab4 -[VNRequest performInContext:error:] + 1244 - 9 Vision 0x000000019f40af08 __73-[VNRequest performInContextAsync:asyncDispatchQueue:asyncDispatchGroup:]_block_invoke + 128 - 10 libdispatch.dylib 0x00000001001d2ac8 _dispatch_block_async_invoke2 + 148 - 11 libdispatch.dylib 0x00000001001be824 _dispatch_client_callout + 20 - 12 libdispatch.dylib 0x00000001001c2350 _dispatch_continuation_pop + 1408 - 13 libdispatch.dylib 0x00000001001c1028 _dispatch_async_redirect_invoke + 616 - 14 libdispatch.dylib 0x00000001001d6b70 _dispatch_root_queue_drain + 404 - 15 libdispatch.dylib 0x00000001001d777c _dispatch_worker_thread2 + 188 - 16 libsystem_pthread.dylib 0x0000000100b090c4 _pthread_wqthread + 228 - 17 libsystem_pthread.dylib 0x0000000100b10cf0 start_wqthread + 8 -) -FAULT: NSUnknownKeyException: [ valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.; { - NSTargetObjectUserInfoKey = ""; - NSUnknownUserInfoKey = lastDetectedCoordinates; -} -libc++abi: terminating due to uncaught exception of type NSException \ No newline at end of file +# Error Resolution Log + +## Fixed Issues (2023) + +### ScreenCaptureKit API Updates +1. SCContentFilter Initialization + - Fixed by using correct initializer and parameters: + ```swift + SCContentFilter(display: display, excludingWindows: []) + ``` + - Using SCDisplay object directly (not displayID) + - Correct parameter name: excludingWindows + - Removed incorrect parameters (includingWindows/exceptingWindows) + +2. Stream Output Type + - Fixed type inference issue by explicit declaration: + ```swift + let outputType: SCStreamOutputType = .screen + try stream.addStreamOutput(self, type: outputType, ...) + ``` + - Ensures proper type resolution for .screen member + +### Required Imports +- Added necessary framework imports: + * CoreMedia + * AVFoundation + * ScreenCaptureKit + * CoreGraphics + * AppKit + * Foundation + +### Window Capture Strategy +1. Window Detection + - Precise window identification: + ```swift + let bundleID = window.owningApplication?.bundleIdentifier ?? "" + let isChessApp = bundleID == "com.chess.iphone" + let hasValidSize = window.frame.width > 100 && window.frame.height > 100 + return isChessApp && window.isOnScreen && hasValidSize + ``` + - Multiple validation checks: + * Exact bundle ID match + * Window is currently on screen + * Window has valid dimensions + - Handles iOS apps running on Mac properly + +2. Capture Method + - Implemented continuous capture: + ```swift + // Start once + try await screenCapture.startCapture() + + // Process frames continuously + while !Task.isCancelled { + if let image = screenCapture.getCurrentImage() { + try await processImage(image) + } + try await Task.sleep(nanoseconds: 100_000_000) + } + ``` + - Maintains single active stream + - Eliminates capture flickering + - Proper cleanup on stop + +### Thread Safety and Async Handling +1. Main Actor Isolation + - Added @MainActor to ViewModel class: + ```swift + @MainActor + class ScreenCaptureViewModel: ObservableObject + ``` + - Ensures all @Published property updates happen on main thread + - Proper thread safety for SwiftUI bindings + +2. Async Operation Handling + - Optimized async/await usage: + ```swift + // Only use await for truly async operations + try await screenCapture.startCapture() + try await Task.sleep(nanoseconds: 100_000_000) + ``` + - Removed unnecessary await keywords: + * Non-async error handling + * UI state updates + * Image processing + - Proper @MainActor usage for thread safety + +3. Task and Error Handling + - Improved task cancellation: + ```swift + captureLoop: while !Task.isCancelled { + do { + // Process frame + } catch is CancellationError { + break captureLoop + } catch { + // Continue capturing on non-critical errors + await handleCaptureError(error) + } + } + ``` + - Proper error recovery: + * Continues on board detection failures + * Shows error state without stopping + * Clears errors on successful detection + - Clean error state on stop: + ```swift + captureError = nil // Clear any error when stopping + ``` + - Proper cleanup on task exit + +## Technical Notes +- Resource Management: + * Shared CIContext to prevent command queue exhaustion: + ```swift + private static let shared = CIContext() + private var context: CIContext { Self.shared } + ``` + * Prevents "Command queue creation failed" errors + * Reduces Metal resource usage + * Proper cleanup on task completion + +- Using correct SCContentFilter API with proper parameter names +- Proper type safety throughout the implementation: + * Proper Vision framework result handling: + ```swift + guard let observations = request.results, + !observations.isEmpty else { + return nil + } + + let bestObservation = observations[0] + ``` + * No unnecessary type casting + * Safe array access + * Proper optional handling +- Appropriate framework dependencies +- Follows current ScreenCaptureKit best practices diff --git a/cline_docs/systemPatterns.md b/cline_docs/systemPatterns.md index 0dbdbc3..87bba30 100644 --- a/cline_docs/systemPatterns.md +++ b/cline_docs/systemPatterns.md @@ -1,12 +1,94 @@ # System Patterns +## Window Capture Architecture + +### Window Detection Pattern +1. SCShareableContent Access + - Async/await pattern for content access + - Proper error propagation + - Permission handling + +2. Window Identification + - Multiple validation criteria: + ```swift + let bundleID = window.owningApplication?.bundleIdentifier ?? "" + let isChessApp = bundleID == "com.chess.iphone" + let hasValidSize = window.frame.width > 100 && window.frame.height > 100 + return isChessApp && window.isOnScreen && hasValidSize + ``` + - Fail-fast approach with guard statements + - Clear error states + +### Capture System Pattern +1. Stream Configuration + - Window-specific capture setup + - Frame dimension matching + - Proper delegate handling + +2. Frame Processing + - Main thread safety for UI updates + - Efficient image conversion pipeline + - Resource cleanup + +### Error Handling Pattern +1. Task Management + - Proper cancellation points + - Clean state management + - Resource cleanup + +2. Error States + - Clear error types + - User-friendly messages + - State recovery + +## UI Architecture + +### MVVM Implementation +1. ViewModel + - @MainActor for thread safety + - Published properties for state + - Clear separation of concerns + +2. View Layer + - SwiftUI declarative UI + - State-driven updates + - Error presentation + +### Async Operations +1. Task Management + - Structured concurrency + - Proper cancellation + - State synchronization + +2. State Updates + - Main thread safety + - Clear state transitions + - Error recovery + ## Core Architecture +### Resource Management Patterns +1. Shared CIContext Pattern + - Static shared instance: + ```swift + private static let shared = CIContext() + private var context: CIContext { Self.shared } + ``` + - Benefits: + * Prevents Metal command queue exhaustion + * Reduces resource overhead + * Enables long-running captures + - Implementation: + * Used in BoardDetector and ViewModel + * Proper cleanup on task completion + * Thread-safe access + ### Screen Capture System - Uses ScreenCaptureKit for efficient screen capture - Implements SCStreamOutput protocol for frame processing - Handles capture session lifecycle and cleanup - Manages permissions and error handling +- Optimized resource usage ### Board Detection System Two implemented approaches: diff --git a/cline_docs/techContext.md b/cline_docs/techContext.md index a3a048f..37881b4 100644 --- a/cline_docs/techContext.md +++ b/cline_docs/techContext.md @@ -8,21 +8,38 @@ ## Core Technologies +### Metal Resource Management +- Shared CIContext pattern: + * Static shared instance to prevent command queue exhaustion + * Used across BoardDetector and ViewModel + * Proper cleanup and resource management +- Performance considerations: + * Reduced Metal command queue creation + * Efficient resource utilization + * Support for long-running captures + ### ScreenCaptureKit - System framework for screen capture -- Requires user permissions -- Supports window/display filtering -- Real-time frame capture capabilities - -### Vision Framework -- Used for board and coordinate detection +- Implemented features: + * Window detection using SCShareableContent + * iOS app window capture support + * Real-time frame capture + * Proper error handling - Key components: + * SCShareableContent: Window and display access + * SCContentFilter: Window-specific capture + * SCStream: Frame capture management + * SCStreamOutput: Frame processing + +### Vision Framework (Planned) +- Will be used for board and coordinate detection +- Key components to implement: * VNRecognizeTextRequest: Chess coordinate detection * VNDetectRectanglesRequest: Board boundary detection -- Configuration: +- Planned configuration: * Text recognition level: accurate * Language correction: disabled - * Rectangle aspect ratio: 0.3-0.5 (for taller rectangles) + * Rectangle aspect ratio: 0.3-0.5 * Minimum size: 0.4 * Maximum observations: 1 @@ -58,36 +75,40 @@ ## Technical Constraints -### Board Detection -1. Pattern Recognition - - Detect taller rectangles (0.3-0.5 aspect ratio) - - Extract square board from upper portion - - Use width as reference measurement - - Handle coordinate system transformations +### Window Capture System +1. Window Detection + - Using SCShareableContent for window access + - Multiple validation criteria: + * Bundle ID verification + * Window visibility check + * Size validation + - Error handling for missing windows -2. Coordinate Recognition - - Must detect a-h and 1-8 coordinates - - Handles both light and dark themes - - Requires clear coordinate visibility - - Minimum text size requirements +2. Frame Capture + - Window-specific capture configuration + - Frame dimension matching + - Proper delegate handling + - Resource cleanup -3. Board Boundaries - - Square aspect ratio (1:1) - - Tolerance: 0.3 for dimensions - - Extract from detected area - - Proper coordinate transformations +3. Performance + - Main thread safety for UI updates + - Efficient image conversion + - Proper task cancellation + - Memory management + - Shared CIContext for Metal efficiency -4. Performance - - Frame processing on dedicated queue - - Asynchronous Vision requests - - Memory management for capture session - - Resource cleanup requirements +4. Error Handling + - Clear error types + - User-friendly messages + - State recovery + - Resource cleanup ### System Requirements - macOS 12.0 or later - Screen Capture permissions - Sufficient CPU for real-time processing - Adequate memory for frame buffering +- Metal-capable GPU for image processing ## Dependencies @@ -102,6 +123,7 @@ - Vision.framework - SwiftUI.framework - CoreImage.framework +- Metal.framework (via CIContext) ## Development Guidelines @@ -110,8 +132,10 @@ - Protocol-oriented design - Clear separation of concerns - Comprehensive error handling +- Resource sharing patterns ### Performance Optimization +- Shared CIContext for Metal efficiency - Efficient frame processing - Memory management - Resource cleanup @@ -130,16 +154,19 @@ - Coordinate transformations - Error handling - State management +- Resource management ### Integration Tests - End-to-end workflows - Component interaction - Event propagation +- Resource sharing ### UI Tests - User interaction flows - Error state handling - Visual feedback +- Performance monitoring ## Documentation Requirements @@ -148,27 +175,29 @@ - Parameter descriptions - Return value documentation - Error documentation +- Resource usage documentation ### Architecture Documentation - System overview - Component interaction - Data flow diagrams - State management +- Resource management patterns ## Current Challenges -### Coordinate Systems -1. Understanding - - Different origin points - - Axis directions - - Coordinate spaces - - Transformation requirements +### Resource Management +1. Metal Efficiency + - Command queue management + - Shared context patterns + - Resource cleanup + - Performance monitoring -2. Implementation - - Proper transformations - - Consistent handling - - Validation methods - - Error checking +2. Memory Usage + - Frame buffer management + - Image processing optimization + - Resource pooling + - Cleanup strategies ### Board Detection 1. Full Capture @@ -206,3 +235,4 @@ - Optimize frame processing - Improve error recovery - Enhanced permission handling +- Resource usage monitoring