diff --git a/ChessPrism/ChessPrism/BoardDetector.swift b/ChessPrism/ChessPrism/BoardDetector.swift index d5d99e6..1e8f70b 100644 --- a/ChessPrism/ChessPrism/BoardDetector.swift +++ b/ChessPrism/ChessPrism/BoardDetector.swift @@ -2,46 +2,46 @@ import Foundation import CoreImage import Vision -class BoardDetector { +/// Errors that can occur during board detection and processing +enum BoardDetectionError: Error { + case boardNotFound + case invalidBoardDimensions + case squareExtractionFailed + case imageProcessingFailed +} + +final class BoardDetector { // Share CIContext to avoid creating too many Metal command queues private static let shared = CIContext() private var context: CIContext { BoardDetector.shared } func detectBoard(in image: CIImage) -> CGRect? { - // Configure rectangle detection request let request = VNDetectRectanglesRequest() - request.minimumAspectRatio = 0.8 // Adjusted for standard chess board + request.minimumAspectRatio = 0.8 request.maximumAspectRatio = 1.2 request.minimumSize = 0.4 request.maximumObservations = 1 request.quadratureTolerance = 30 request.minimumConfidence = 0.9 - // Perform the request let requestHandler = VNImageRequestHandler(ciImage: image, options: [:]) do { try requestHandler.perform([request]) } catch { - print("Failed to perform rectangle detection: \(error)") + print("ERROR: Rectangle detection failed - \(error)") return nil } - // Process results guard let observations = request.results, !observations.isEmpty else { return nil } let bestObservation = observations[0] - - // Convert Vision coordinates to CoreImage coordinates let imageSize = image.extent.size let transform = CGAffineTransform(scaleX: imageSize.width, y: imageSize.height) - - // Create normalized rect in CoreImage coordinate space let detectedRect = bestObservation.boundingBox.applying(transform) - // Validate the detected rectangle guard validateDetectedRect(detectedRect, in: image) else { return nil } @@ -49,28 +49,123 @@ class BoardDetector { return detectedRect } + /// Piece recognizer for analyzing extracted squares + private let pieceRecognizer: PieceRecognizer + + /// Initialize with a piece recognizer + init(pieceRecognizer: PieceRecognizer) { + self.pieceRecognizer = pieceRecognizer + } + + /// Extract a specific square from the board image + private func extractSquare(from image: CIImage, in rect: CGRect, at position: BoardPosition) throws -> CGImage { + let squareSize = rect.width / 8 + let x = rect.minX + CGFloat(position.file) * squareSize + let y = rect.minY + CGFloat(position.rank) * squareSize + let squareRect = CGRect(x: x, y: y, width: squareSize, height: squareSize) + + guard image.extent.contains(squareRect) else { + print("ERROR: Square bounds outside image extent at \(position.file),\(position.rank)") + throw BoardDetectionError.squareExtractionFailed + } + + let croppedImage = image.cropped(to: squareRect) + + guard let cgImage = context.createCGImage(croppedImage, from: croppedImage.extent) else { + print("ERROR: Failed to create square image at \(position.file),\(position.rank)") + throw BoardDetectionError.squareExtractionFailed + } + + return cgImage + } + + /// Analyze the board and return the chess position + func analyzeBoard(in image: CIImage) async throws -> ChessPosition { + print("=== ANALYZING BOARD ===") + + guard let boardRect = detectBoard(in: image) else { + print("ERROR: Board detection failed") + throw BoardDetectionError.boardNotFound + } + + let aspectRatio = boardRect.width / boardRect.height + guard boardRect.width > 0, boardRect.height > 0, + abs(1 - aspectRatio) < 0.1 else { + print("ERROR: Invalid board dimensions") + throw BoardDetectionError.invalidBoardDimensions + } + + var position = ChessPosition() + var recognizedPieces = 0 + + // Reset piece counts before starting new scan + pieceRecognizer.resetPieceCounts() + + // Process each square + for rank in 0..<8 { + for file in 0..<8 { + let boardPosition = BoardPosition(file: file, rank: rank) + let squareImage = try extractSquare(from: image, in: boardRect, at: boardPosition) + + // Set the current position being analyzed + pieceRecognizer.currentPosition = boardPosition + + if let piece = try await pieceRecognizer.recognizePiece(from: squareImage) { + position[boardPosition] = piece + recognizedPieces += 1 + } + } + } + + // Validate piece counts after all squares are processed + try pieceRecognizer.validatePieceCounts() + + print("\nPieces recognized: \(recognizedPieces)") + print("\nRecognized position:") + for rank in (0...7).reversed() { + var rankStr = "\(rank + 1) " + for file in 0...7 { + if let piece = position[BoardPosition(file: file, rank: rank)] { + rankStr += "\(piece.color == .white ? "w" : "b")\(piece.type.fenSymbol.uppercased()) " + } else { + rankStr += ".. " + } + } + print(rankStr) + } + print(" a b c d e f g h") + + guard position.isValid else { + print("\nERROR: Invalid chess position") + throw BoardDetectionError.imageProcessingFailed + } + + print("=== BOARD ANALYSIS COMPLETE ===") + return position + } + private func validateDetectedRect(_ rect: CGRect, in image: CIImage) -> Bool { let imageSize = image.extent.size - // Check if rectangle is within image bounds guard image.extent.contains(rect) else { + print("ERROR: Detected rectangle outside image bounds") return false } - // Validate aspect ratio (standard chess board is square) let aspectRatio = rect.width / rect.height guard aspectRatio >= 0.9 && aspectRatio <= 1.1 else { + print("ERROR: Invalid board aspect ratio: \(aspectRatio)") return false } - // Validate size relative to image let minDimension = min(imageSize.width, imageSize.height) let boardSize = max(rect.width, rect.height) - guard boardSize >= minDimension * 0.4 else { + let sizeRatio = boardSize / minDimension + guard sizeRatio >= 0.4 else { + print("ERROR: Board too small relative to image") return false } return true } - } diff --git a/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/analytics/coremldata.bin b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/analytics/coremldata.bin new file mode 100644 index 0000000..20bdbe3 Binary files /dev/null and b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/analytics/coremldata.bin differ diff --git a/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/coremldata.bin b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/coremldata.bin new file mode 100644 index 0000000..cf4c862 Binary files /dev/null and b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/coremldata.bin differ diff --git a/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/metadata.json b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/metadata.json new file mode 100644 index 0000000..aef216c --- /dev/null +++ b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/metadata.json @@ -0,0 +1,86 @@ +[ + { + "metadataOutputVersion" : "3.0", + "outputSchema" : [ + { + "isOptional" : "0", + "formattedType" : "String", + "type" : "String", + "name" : "target", + "shortDescription" : "" + }, + { + "isOptional" : "0", + "keyType" : "String", + "formattedType" : "Dictionary (String → Double)", + "type" : "Dictionary", + "name" : "targetProbability", + "shortDescription" : "" + } + ], + "modelParameters" : [ + + ], + "author" : "Chris Haulmark", + "specificationVersion" : 8, + "isUpdatable" : "0", + "stateSchema" : [ + + ], + "availability" : { + "macOS" : "14.0", + "tvOS" : "17.0", + "visionOS" : "1.0", + "watchOS" : "unavailable", + "iOS" : "17.0", + "macCatalyst" : "17.0" + }, + "modelType" : { + "name" : "MLModelType_imageClassifier", + "structure" : [ + { + "name" : "MLModelType_visionFeaturePrint" + }, + { + "name" : "MLModelType_glmClassifier" + } + ] + }, + "inputSchema" : [ + { + "height" : "360", + "colorspace" : "BGR", + "isOptional" : "0", + "width" : "360", + "isColor" : "1", + "formattedType" : "Image (Color 360 × 360)", + "hasSizeFlexibility" : "0", + "type" : "Image", + "shortDescription" : "", + "name" : "image" + } + ], + "classLabels" : [ + "black_bishop", + "black_king", + "black_knight", + "black_pawn", + "black_queen", + "black_rook", + "white_bishop", + "white_king", + "white_knight", + "white_pawn", + "white_queen", + "white_rook" + ], + "generatedClassName" : "ChessPieceClassifier", + "userDefinedMetadata" : { + "com.apple.createml.version" : "15.3.0", + "com.apple.createml.app.tag" : "150.3", + "com.apple.coreml.model.preview.type" : "imageClassifier", + "com.apple.createml.app.version" : "6.1" + }, + "method" : "predict" + } +] \ No newline at end of file diff --git a/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/model0/coremldata.bin b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/model0/coremldata.bin new file mode 100644 index 0000000..3ec9c2d Binary files /dev/null and b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/model0/coremldata.bin differ diff --git a/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/model1/coremldata.bin b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/model1/coremldata.bin new file mode 100644 index 0000000..1b027f9 Binary files /dev/null and b/ChessPrism/ChessPrism/ChessPieceClassifier.mlmodelc/model1/coremldata.bin differ diff --git a/ChessPrism/ChessPrism/ContentView.swift b/ChessPrism/ChessPrism/ContentView.swift index d7c36d1..50e18eb 100644 --- a/ChessPrism/ChessPrism/ContentView.swift +++ b/ChessPrism/ChessPrism/ContentView.swift @@ -1,6 +1,84 @@ import SwiftUI import AppKit +// MARK: - Chess Position View Components + +struct ChessPieceView: View { + let piece: ChessPiece + + var body: some View { + Text(pieceSymbol) + .font(.system(size: 24, weight: .bold)) + .foregroundColor(piece.color == .white ? .white : .black) + } + + private var pieceSymbol: String { + switch piece.type { + case .pawn: return "♟" + case .knight: return "♞" + case .bishop: return "♝" + case .rook: return "♜" + case .queen: return "♛" + case .king: return "♚" + } + } +} + +struct ChessboardSquareView: View { + let isLightSquare: Bool + let piece: ChessPiece? + + var body: some View { + ZStack { + Rectangle() + .fill(isLightSquare ? Color.white : Color.gray) + .frame(width: 40, height: 40) + + if let piece = piece { + ChessPieceView(piece: piece) + } + } + } +} + +struct ChessboardView: View { + let position: ChessPosition? + + var body: some View { + VStack(spacing: 0) { + ForEach((0..<8).reversed(), id: \.self) { rank in + HStack(spacing: 0) { + ForEach(0..<8, id: \.self) { file in + let boardPosition = BoardPosition(file: file, rank: rank) + ChessboardSquareView( + isLightSquare: (rank + file) % 2 == 0, + piece: position?[boardPosition] + ) + } + } + } + } + .border(Color.black, width: 1) + } +} + +struct AnalysisStatusView: View { + let isAnalyzing: Bool + let confidence: Double + + var body: some View { + VStack(spacing: 8) { + if isAnalyzing { + ProgressView("Analyzing position...") + } else { + Text("Recognition Confidence: \(Int(confidence * 100))%") + .foregroundColor(confidence > 0.7 ? .green : .orange) + } + } + .padding() + } +} + class CustomNSView: NSView { private static let invisibleCursor: NSCursor = { let image = NSImage(size: NSSize(width: 1, height: 1)) @@ -84,8 +162,116 @@ struct CaptureStatusButton: View { } } +struct CaptureView: View { + @ObservedObject var viewModel: ScreenCaptureViewModel + + var body: some View { + HStack { + VStack { + Text("Full Capture") + if let image = viewModel.capturedImage { + Image(nsImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxWidth: 400) + } else { + Text("No capture available") + .foregroundColor(.gray) + } + } + + Divider() + + VStack { + Text("Chessboard Preview") + if let image = viewModel.croppedBoardImage { + Image(nsImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxWidth: 400) + .overlay( + HiddenCursorView() + .allowsHitTesting(true) + ) + } else { + Text("No board detected") + .foregroundColor(.gray) + } + + if viewModel.snapshotTaken { + Divider() + .padding(.vertical) + + Text("Latest Snapshot") + if let snapshot = viewModel.latestSnapshot { + Image(nsImage: snapshot) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxWidth: 400) + .padding() + .background(Color.black.opacity(0.1)) + .cornerRadius(8) + } + } + } + } + .padding() + } +} + +struct AnalysisView: View { + @ObservedObject var viewModel: ScreenCaptureViewModel + + var body: some View { + VStack { + if let position = viewModel.currentPosition { + HStack(alignment: .top, spacing: 20) { + VStack(alignment: .leading) { + Text("Current Position") + .font(.headline) + ChessboardView(position: position) + .padding() + .background(Color.white) + .cornerRadius(8) + .shadow(radius: 2) + + AnalysisStatusView( + isAnalyzing: viewModel.isAnalyzing, + confidence: viewModel.recognitionConfidence + ) + } + + VStack(alignment: .leading) { + Text("Position Details") + .font(.headline) + Text("FEN: \(position.fen)") + .font(.system(.body, design: .monospaced)) + .padding(.vertical) + + if viewModel.captureError == .recognitionFailed { + Text("Recognition Error") + .foregroundColor(.red) + .padding() + } + } + .padding() + .background(Color.gray.opacity(0.1)) + .cornerRadius(8) + } + } else { + Text("No position detected") + .foregroundColor(.gray) + } + + Spacer() + } + .padding() + } +} + struct ContentView: View { @StateObject private var viewModel = ScreenCaptureViewModel() + @State private var selectedTab = 0 var body: some View { VStack { @@ -110,11 +296,9 @@ struct ContentView: View { .disabled(!viewModel.isCapturing || !viewModel.isBoardDetected) } .onAppear { - // Start monitoring for chess boards when view appears viewModel.startMonitoring() } .onDisappear { - // Stop monitoring when view disappears viewModel.stopMonitoring() } @@ -125,61 +309,22 @@ struct ContentView: View { .padding() } - // Image display - HStack { - VStack { - Text("Full Capture") - if let image = viewModel.capturedImage { - Image(nsImage: image) - .resizable() - .aspectRatio(contentMode: .fit) - .frame(maxWidth: 400) - } else { - Text("No capture available") - .foregroundColor(.gray) + // Main content area with tabs + TabView(selection: $selectedTab) { + CaptureView(viewModel: viewModel) + .tabItem { + Label("Capture", systemImage: "camera") } - } + .tag(0) - Divider() - - VStack { - Text("Chessboard Preview") - if let image = viewModel.croppedBoardImage { - Image(nsImage: image) - .resizable() - .aspectRatio(contentMode: .fit) - .frame(maxWidth: 400) - .overlay( - HiddenCursorView() - .allowsHitTesting(true) - ) - } else { - Text("No board detected") - .foregroundColor(.gray) + AnalysisView(viewModel: viewModel) + .tabItem { + Label("Analysis", systemImage: "magnifyingglass") } - - if viewModel.snapshotTaken { - Divider() - .padding(.vertical) - - Text("Latest Snapshot") - if let snapshot = viewModel.latestSnapshot { - Image(nsImage: snapshot) - .resizable() - .aspectRatio(contentMode: .fit) - .frame(maxWidth: 400) - .padding() - .background(Color.black.opacity(0.1)) - .cornerRadius(8) - } - } - } + .tag(1) } - .padding() - - Spacer() } - .frame(minWidth: 800, minHeight: 600) + .frame(minWidth: 1000, minHeight: 700) } } diff --git a/ChessPrism/ChessPrism/Models/ChessPosition.swift b/ChessPrism/ChessPrism/Models/ChessPosition.swift new file mode 100644 index 0000000..c026f33 --- /dev/null +++ b/ChessPrism/ChessPrism/Models/ChessPosition.swift @@ -0,0 +1,274 @@ +import Foundation + +/// Represents a chess piece color +enum PieceColor: String { + case white + case black +} + +/// Represents a chess piece type +enum PieceType: String { + case pawn + case knight + case bishop + case rook + case queen + case king + + /// FEN notation for the piece + var fenSymbol: String { + switch self { + case .pawn: return "p" + case .knight: return "n" + case .bishop: return "b" + case .rook: return "r" + case .queen: return "q" + case .king: return "k" + } + } +} + +/// Represents a chess piece with its type and color +struct ChessPiece: Equatable { + let type: PieceType + let color: PieceColor + + /// FEN notation for the piece (uppercase for white, lowercase for black) + var fenSymbol: String { + let symbol = type.fenSymbol + return color == .white ? symbol.uppercased() : symbol + } +} + +/// Represents a position on the chess board +struct BoardPosition: Equatable { + let file: Int // 0-7 for a-h + let rank: Int // 0-7 for 1-8 + + /// Initialize from algebraic notation (e.g., "e4") + init?(algebraic: String) { + guard algebraic.count == 2, + let file = algebraic.first?.asciiValue, + let rank = algebraic.last?.wholeNumberValue, + file >= UInt8(ascii: "a"), file <= UInt8(ascii: "h"), + rank >= 1, rank <= 8 else { + return nil + } + + self.file = Int(file - UInt8(ascii: "a")) + self.rank = rank - 1 + } + + /// Convert to algebraic notation + var algebraic: String { + let fileChar = Character(UnicodeScalar(UInt8(ascii: "a") + UInt8(file))) + return "\(fileChar)\(rank + 1)" + } + + /// Validate if the position is within bounds + var isValid: Bool { + file >= 0 && file < 8 && rank >= 0 && rank < 8 + } +} + +/// Represents a complete chess position +struct ChessPosition { + /// 8x8 grid representing the board state, nil means empty square + private var board: [[ChessPiece?]] + + /// Initialize an empty board + init() { + board = Array(repeating: Array(repeating: nil, count: 8), count: 8) + } + + /// Initialize from FEN string + init?(fen: String) { + self.init() + + let components = fen.components(separatedBy: " ") + guard components.count >= 1 else { return nil } + + let ranks = components[0].components(separatedBy: "/") + guard ranks.count == 8 else { return nil } + + for (rankIndex, rank) in ranks.enumerated() { + var fileIndex = 0 + + for char in rank { + if let number = Int(String(char)) { + fileIndex += number + } else { + guard fileIndex < 8 else { return nil } + + let color: PieceColor = char.isUppercase ? .white : .black + let lowerChar = char.lowercased() + + guard let type = pieceTypeFromFen(String(lowerChar)) else { return nil } + + board[7 - rankIndex][fileIndex] = ChessPiece(type: type, color: color) + fileIndex += 1 + } + } + + guard fileIndex == 8 else { return nil } + } + } + + /// Convert position to FEN string (piece placement only) + var fen: String { + var result = "" + + for rankIndex in (0...7).reversed() { + var emptyCount = 0 + + for fileIndex in 0...7 { + if let piece = board[rankIndex][fileIndex] { + if emptyCount > 0 { + result += String(emptyCount) + emptyCount = 0 + } + result += piece.fenSymbol + } else { + emptyCount += 1 + } + } + + if emptyCount > 0 { + result += String(emptyCount) + } + + if rankIndex > 0 { + result += "/" + } + } + + return result + } + + /// Get piece at position + subscript(position: BoardPosition) -> ChessPiece? { + get { + guard position.isValid else { return nil } + return board[position.rank][position.file] + } + set { + guard position.isValid else { return } + board[position.rank][position.file] = newValue + } + } + + /// Get piece at algebraic position + subscript(algebraic: String) -> ChessPiece? { + get { + guard let position = BoardPosition(algebraic: algebraic) else { return nil } + return self[position] + } + set { + guard let position = BoardPosition(algebraic: algebraic) else { return } + self[position] = newValue + } + } + + /// Validate if the position is legal + var isValid: Bool { + print("\n=== VALIDATING CHESS POSITION ===") + var whitePieces = [PieceType: Int]() + var blackPieces = [PieceType: Int]() + + // Count all pieces + for rank in 0...7 { + for file in 0...7 { + if let piece = board[rank][file] { + if piece.color == .white { + whitePieces[piece.type, default: 0] += 1 + } else { + blackPieces[piece.type, default: 0] += 1 + } + + // Check pawns on invalid ranks + if piece.type == .pawn && (rank == 0 || rank == 7) { + print("ERROR: Pawn found on first/last rank") + return false + } + } + } + } + + // Print piece counts + print("White pieces:") + for (type, count) in whitePieces { + print("- \(type): \(count)") + } + print("Black pieces:") + for (type, count) in blackPieces { + print("- \(type): \(count)") + } + + // Validate piece counts + let whiteTotal = whitePieces.values.reduce(0, +) + let blackTotal = blackPieces.values.reduce(0, +) + + if whiteTotal > 16 { + print("ERROR: Too many white pieces (\(whiteTotal))") + return false + } + if blackTotal > 16 { + print("ERROR: Too many black pieces (\(blackTotal))") + return false + } + + // Validate kings + if whitePieces[.king] ?? 0 != 1 { + print("ERROR: Invalid number of white kings (\(whitePieces[.king] ?? 0))") + return false + } + if blackPieces[.king] ?? 0 != 1 { + print("ERROR: Invalid number of black kings (\(blackPieces[.king] ?? 0))") + return false + } + + // Validate pawns + if whitePieces[.pawn] ?? 0 > 8 { + print("ERROR: Too many white pawns (\(whitePieces[.pawn] ?? 0))") + return false + } + if blackPieces[.pawn] ?? 0 > 8 { + print("ERROR: Too many black pawns (\(blackPieces[.pawn] ?? 0))") + return false + } + + // Validate other pieces + for pieceType in [PieceType.queen, .rook, .bishop, .knight] { + if whitePieces[pieceType] ?? 0 > 2 { + print("ERROR: Too many white \(pieceType)s (\(whitePieces[pieceType] ?? 0))") + return false + } + if blackPieces[pieceType] ?? 0 > 2 { + print("ERROR: Too many black \(pieceType)s (\(blackPieces[pieceType] ?? 0))") + return false + } + } + + print("Position validation successful") + return true + } + + /// Helper function to convert FEN piece symbol to PieceType + private func pieceTypeFromFen(_ symbol: String) -> PieceType? { + switch symbol { + case "p": return .pawn + case "n": return .knight + case "b": return .bishop + case "r": return .rook + case "q": return .queen + case "k": return .king + default: return nil + } + } + + /// Initialize with the standard starting position + static var startingPosition: ChessPosition { + // swiftlint:disable:next force_unwrapping + ChessPosition(fen: "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR")! + } +} diff --git a/ChessPrism/ChessPrism/Recognition/PieceRecognizer.swift b/ChessPrism/ChessPrism/Recognition/PieceRecognizer.swift new file mode 100644 index 0000000..4dceafd --- /dev/null +++ b/ChessPrism/ChessPrism/Recognition/PieceRecognizer.swift @@ -0,0 +1,397 @@ +import Foundation +import Vision +import CoreML +import CoreImage + +/// Errors that can occur during piece recognition +enum PieceRecognitionError: Error { + case invalidImageDimensions + case modelLoadError + case recognitionFailed(String) + case lowConfidence + case invalidInput + case invalidPosition(String) +} + +/// A class responsible for recognizing chess pieces from images +final class PieceRecognizer { + // MARK: - Properties + + /// Vision model for piece classification + private let vnModel: VNCoreMLModel + + /// Shared CIContext for image processing + private static let ciContext = CIContext() + + /// Piece counts for validation + private var whitePieceCount: [PieceType: Int] = [:] + private var blackPieceCount: [PieceType: Int] = [:] + + // MARK: - Initialization + + init() throws { + print("=== INITIALIZING PIECE RECOGNIZER ===") + + let bundle = Bundle.main + + // Load model from bundle + guard let modelURL = bundle.url(forResource: "ChessPieceClassifier", withExtension: "mlmodelc") else { + print("ERROR: Model not found in bundle at \(bundle.bundlePath)") + throw PieceRecognitionError.modelLoadError + } + + do { + let config = MLModelConfiguration() + config.computeUnits = .all + let model = try MLModel(contentsOf: modelURL, configuration: config) + self.vnModel = try VNCoreMLModel(for: model) + print("Model loaded successfully from: \(modelURL.path)") + } catch { + print("ERROR: Failed to load model - \(error)") + throw PieceRecognitionError.modelLoadError + } + + resetPieceCounts() + } + + // MARK: - Recognition Methods + + /// Reset piece counts for new position + func resetPieceCounts() { + whitePieceCount = [ + .king: 0, + .queen: 0, + .rook: 0, + .bishop: 0, + .knight: 0, + .pawn: 0 + ] + blackPieceCount = [ + .king: 0, + .queen: 0, + .rook: 0, + .bishop: 0, + .knight: 0, + .pawn: 0 + ] + } + + /// Validate piece counts and ensure the position is legal + func validatePieceCounts() throws { + var errors: [String] = [] + + // Check white pieces + if whitePieceCount[.king] != 1 { + let count = whitePieceCount[.king] ?? 0 + errors.append("Invalid white king count: \(count)") + } + if let count = whitePieceCount[.queen], count > 1 { + errors.append("Too many white queens: \(count)") + } + if let count = whitePieceCount[.rook], count > 2 { + errors.append("Too many white rooks: \(count)") + } + if let count = whitePieceCount[.bishop], count > 2 { + errors.append("Too many white bishops: \(count)") + } + if let count = whitePieceCount[.knight], count > 2 { + errors.append("Too many white knights: \(count)") + } + if let count = whitePieceCount[.pawn], count > 8 { + errors.append("Too many white pawns: \(count)") + } + + // Check black pieces + if blackPieceCount[.king] ?? 0 != 1 { + let count = blackPieceCount[.king] ?? 0 + errors.append("Invalid black king count: \(count)") + } + if let count = blackPieceCount[.queen], count > 1 { + errors.append("Too many black queens: \(count)") + } + if let count = blackPieceCount[.rook], count > 2 { + errors.append("Too many black rooks: \(count)") + } + if let count = blackPieceCount[.bishop], count > 2 { + errors.append("Too many black bishops: \(count)") + } + if let count = blackPieceCount[.knight], count > 2 { + errors.append("Too many black knights: \(count)") + } + if let count = blackPieceCount[.pawn], count > 8 { + errors.append("Too many black pawns: \(count)") + } + + if !errors.isEmpty { + throw PieceRecognitionError.invalidPosition(errors.joined(separator: ", ")) + } + } + + /// Update piece count + private func updatePieceCount(piece: ChessPiece) { + if piece.color == .white { + whitePieceCount[piece.type] = (whitePieceCount[piece.type] ?? 0) + 1 + } else { + blackPieceCount[piece.type] = (blackPieceCount[piece.type] ?? 0) + 1 + } + } + + /// Check if adding this piece would exceed limits + private func wouldExceedLimits(_ piece: ChessPiece) -> Bool { + let count = piece.color == .white ? whitePieceCount[piece.type] ?? 0 : blackPieceCount[piece.type] ?? 0 + switch piece.type { + case .king: return count >= 1 + case .queen: return count >= 1 + case .rook, .bishop, .knight: return count >= 2 + case .pawn: return count >= 8 + } + } + + /// Recognize a chess piece from a square image + /// - Parameters: + /// - image: CGImage of the chess square + /// - completion: Callback with result (ChessPiece if recognized, nil if empty) + /// - Throws: PieceRecognitionError + func recognizePiece(from image: CGImage) async throws -> ChessPiece? { + // Validate image dimensions + guard image.width > 0, image.height > 0, + abs(1 - Float(image.width) / Float(image.height)) < 0.1 else { + print("ERROR: Invalid square dimensions \(image.width)x\(image.height)") + throw PieceRecognitionError.invalidImageDimensions + } + + let handler = VNImageRequestHandler(cgImage: image) + var classificationResults: [VNClassificationObservation]? + var classificationError: Error? + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let request = VNCoreMLRequest(model: vnModel) { request, error in + if let error = error { + classificationError = error + continuation.resume(throwing: error) + return + } + classificationResults = request.results as? [VNClassificationObservation] + continuation.resume() + } + request.imageCropAndScaleOption = .centerCrop + + do { + try handler.perform([request]) + } catch { + continuation.resume(throwing: error) + } + } + + if let error = classificationError { + print("ERROR: Classification failed - \(error)") + throw PieceRecognitionError.recognitionFailed(error.localizedDescription) + } + + guard let results = classificationResults, + let topResult = results.first else { + print("ERROR: No classification results") + throw PieceRecognitionError.recognitionFailed("No results") + } + + // Print all results to help diagnose recognition issues + if let pos = currentPosition { + print("\nClassification results for \(String(describing: pos)):") + } else { + print("\nClassification results:") + } + for result in results.prefix(3) { + print("- \(result.identifier): \(result.confidence)") + } + + // Check for empty squares + if topResult.identifier == "empty_dark" || topResult.identifier == "empty_light" { + if topResult.confidence > 0.9 { + if let pos = currentPosition { + print("\nEmpty square confirmed at \(String(describing: pos))") + } + return nil + } + } + + // Get confidence ratio between top predictions + let secondBestConfidence = results.count > 1 ? results[1].confidence : 0 + let confidenceRatio = topResult.confidence / (secondBestConfidence + Float.ulpOfOne) + + // Adjust confidence based on position-specific knowledge + let adjustedConfidence = adjustConfidence(topResult.confidence, + for: topResult.identifier, + at: currentPosition) + + // Much stricter piece recognition: + // 1. Must have very high confidence (>0.98) + // 2. Must have strong separation from second best (>5.0 ratio) + // 3. Must make sense for position + // 4. Must not exceed piece limits + if adjustedConfidence > 0.98 && confidenceRatio > 5.0 { + if let piece = try? createPiece(from: topResult.identifier) { + if isValidPieceForPosition(piece, at: currentPosition) && !wouldExceedLimits(piece) { + updatePieceCount(piece: piece) + return piece + } + } + } + + // Log classification details + if let pos = currentPosition { + print("\nClassification rejected at \(String(describing: pos)):") + } + print("Top result: \(topResult.identifier) (\(topResult.confidence))") + if results.count > 1 { + print("Second best: \(results[1].identifier) (\(results[1].confidence))") + print("Confidence ratio: \(confidenceRatio)") + } + print("Adjusted confidence: \(adjustedConfidence)") + + return nil // Default to empty for unclear cases + } + + /// Position where this piece is being recognized + var currentPosition: BoardPosition? + + /// Adjust confidence based on position-specific knowledge + private func adjustConfidence(_ confidence: Float, for identifier: String, at position: BoardPosition?) -> Float { + guard let position = currentPosition else { return confidence } + + // Parse the piece info + let components = identifier.split(separator: "_") + guard components.count == 2, + let color = PieceColor(rawValue: String(components[0])), + let type = PieceType(rawValue: String(components[1])) else { + return confidence + } + + var adjustment: Float = 0.0 + + // Back rank pieces are more likely to be correct + if position.isBackRank(for: color) { + // Corners should be rooks + if type == .rook && position.isEdgeFile { + adjustment += 0.1 + } + // Next to corners should be knights + if type == .knight && (position.file == 1 || position.file == 6) { + adjustment += 0.1 + } + // Next to knights should be bishops + if type == .bishop && (position.file == 2 || position.file == 5) { + adjustment += 0.1 + } + // Center should be king/queen + if (type == .king || type == .queen) && position.isCenterFile { + adjustment += 0.1 + } + } + + // Pawns are more likely on their starting ranks + if type == .pawn { + if (color == .white && position.rank == 1) || + (color == .black && position.rank == 6) { + adjustment += 0.1 + } + } + + return min(1.0, confidence + adjustment) + } + + /// Validate if a piece makes sense for its position + private func isValidPieceForPosition(_ piece: ChessPiece, at position: BoardPosition?) -> Bool { + guard let pos = position else { return true } + + // Basic position validation + switch piece.type { + case .king: + // Kings can't be on the first or last rank of opponent's side + if piece.color == .white && pos.rank == 7 { return false } + if piece.color == .black && pos.rank == 0 { return false } + + case .pawn: + // Pawns can't be on first or last rank + if pos.rank == 0 || pos.rank == 7 { return false } + // White pawns can't be behind their starting rank + if piece.color == .white && pos.rank > 6 { return false } + // Black pawns can't be behind their starting rank + if piece.color == .black && pos.rank < 1 { return false } + + default: + // Other pieces can move freely + break + } + + return true + } + + /// Helper to create a chess piece from a classification label + private func createPiece(from identifier: String) throws -> ChessPiece { + // Validate it's not an empty square + guard !identifier.starts(with: "empty_") else { + print("ERROR: Cannot create piece from empty square label: \(identifier)") + throw PieceRecognitionError.invalidInput + } + + let components = identifier.split(separator: "_") + guard components.count == 2, + let color = PieceColor(rawValue: String(components[0])), + let type = PieceType(rawValue: String(components[1])) else { + print("ERROR: Invalid piece label format: \(identifier)") + throw PieceRecognitionError.invalidInput + } + return ChessPiece(type: type, color: color) + } + + /// Preprocess an image for recognition + /// - Parameter image: Input CGImage + /// - Returns: Preprocessed CGImage + func preprocessImage(_ image: CGImage) throws -> CGImage { + let ciImage = CIImage(cgImage: image) + + // Apply preprocessing filters + let processed = ciImage + .applyingFilter("CIColorControls", parameters: [ + kCIInputContrastKey: 1.1, + kCIInputBrightnessKey: 0.0, + kCIInputSaturationKey: 1.1 + ]) + .applyingFilter("CIUnsharpMask", parameters: [ + kCIInputRadiusKey: 1.0, + kCIInputIntensityKey: 0.5 + ]) + + // Convert back to CGImage + guard let outputImage = Self.ciContext.createCGImage(processed, from: processed.extent) else { + throw PieceRecognitionError.invalidInput + } + + return outputImage + } +} + +// MARK: - BoardPosition Extensions + +extension BoardPosition { + /// Initialize from file and rank indices + init(file: Int, rank: Int) { + self.file = file + self.rank = rank + } + + /// Whether this position is on the edge of the board (files a or h) + var isEdgeFile: Bool { + return file == 0 || file == 7 + } + + /// Whether this position is on the back rank for the given color + func isBackRank(for color: PieceColor) -> Bool { + return (color == .white && rank == 0) || (color == .black && rank == 7) + } + + /// Whether this position is in the center files (d or e) + var isCenterFile: Bool { + return file == 3 || file == 4 + } +} diff --git a/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift b/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift index bf943ef..41e8337 100644 --- a/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift +++ b/ChessPrism/ChessPrism/ScreenCaptureViewModel.swift @@ -13,10 +13,14 @@ class ScreenCaptureViewModel: ObservableObject { @Published var isAutoCapturing = true // Default to auto-capture mode @Published var snapshotTaken = false // Track if snapshot was taken @Published var latestSnapshot: NSImage? // Make snapshot accessible to view + @Published var currentPosition: ChessPosition? // Current chess position + @Published var recognitionConfidence: Double = 0.0 // Recognition confidence + @Published var isAnalyzing = false // Track analysis state private let captureManager = ScreenCapture() // For actual capture private let monitorManager = ScreenCapture() // For monitoring - private let boardDetector = BoardDetector() + private let pieceRecognizer: PieceRecognizer + private let boardDetector: BoardDetector private var captureTask: Task? private var monitorTask: Task? // Share CIContext to avoid creating too many Metal command queues @@ -31,6 +35,8 @@ class ScreenCaptureViewModel: ObservableObject { case boardDetectionFailed case noBoardDetected case snapshotFailed + case recognitionFailed + case invalidPosition var errorDescription: String? { switch self { @@ -42,40 +48,61 @@ class ScreenCaptureViewModel: ObservableObject { return "No chess board detected" case .snapshotFailed: return "Failed to take snapshot" + case .recognitionFailed: + return "Failed to recognize pieces" + case .invalidPosition: + return "Invalid chess position detected" } } } - func takeSnapshot() async { - // Stop current capture - try? await captureManager.stopCapture() - - // Start new capture without cursor + init() { + // Initialize piece recognizer do { + pieceRecognizer = try PieceRecognizer() + boardDetector = BoardDetector(pieceRecognizer: pieceRecognizer) + } catch { + fatalError("Failed to initialize piece recognizer: \(error)") + } + } + + func takeSnapshot() async { + print("=== SCAN BUTTON PRESSED ===") + do { + try await captureManager.stopCapture() try await captureManager.startCapture(excludeCursor: true) - // Wait a brief moment for the capture to stabilize try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds if let currentImage = captureManager.getCurrentImage() { - // Process the image to get the cropped board - try await processImage(currentImage) + print("Captured image: \(currentImage.size)") + try await processImage(currentImage, analyzePosition: true) - // Store the cropped board as snapshot if let boardImage = croppedBoardImage { latestSnapshot = boardImage snapshotTaken = true + + if let position = currentPosition { + print("Successfully detected position") + } else { + print("ERROR: Failed to detect position") + captureError = .recognitionFailed + } } else { + print("ERROR: Failed to crop board image") captureError = .snapshotFailed + currentPosition = nil } } else { + print("ERROR: Failed to capture image") captureError = .snapshotFailed + currentPosition = nil } - // Restart normal capture try await captureManager.startCapture(excludeCursor: false) } catch { + print("ERROR: Snapshot failed - \(error)") captureError = .snapshotFailed - // Ensure we restart normal capture even if snapshot fails + currentPosition = nil try? await captureManager.startCapture(excludeCursor: false) } } @@ -137,6 +164,8 @@ class ScreenCaptureViewModel: ObservableObject { captureError = nil snapshotTaken = false latestSnapshot = nil + currentPosition = nil + recognitionConfidence = 0.0 do { try await captureManager.startCapture() @@ -150,7 +179,7 @@ class ScreenCaptureViewModel: ObservableObject { captureLoop: while !Task.isCancelled { do { if let image = captureManager.getCurrentImage() { - try await processImage(image) + try await processImage(image, analyzePosition: false) } try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds } catch is CancellationError { @@ -182,6 +211,8 @@ class ScreenCaptureViewModel: ObservableObject { isBoardDetected = false snapshotTaken = false latestSnapshot = nil + currentPosition = nil + recognitionConfidence = 0.0 // Clean up capture session Task { @@ -189,29 +220,47 @@ class ScreenCaptureViewModel: ObservableObject { } } - private func processImage(_ image: NSImage) async throws { - // Convert to CIImage for processing + private func processImage(_ image: NSImage, analyzePosition: Bool = false) async throws { guard let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else { + print("ERROR: Failed to convert image for processing") throw CaptureError.boardDetectionFailed } let ciImage = CIImage(cgImage: cgImage) - // Detect board if let boardRect = boardDetector.detectBoard(in: ciImage) { - // Board detected isBoardDetected = true self.detectedBoardRect = boardRect - // Crop board image let croppedImage = ciImage.cropped(to: boardRect) updateImages(ciImage: ciImage, croppedImage: croppedImage) + + if analyzePosition && !isAnalyzing { + isAnalyzing = true + do { + print("Analyzing board position...") + let position = try await boardDetector.analyzeBoard(in: ciImage) + currentPosition = position + recognitionConfidence = 1.0 + isAnalyzing = false + } catch { + print("ERROR: Position analysis failed - \(error)") + isAnalyzing = false + currentPosition = nil + recognitionConfidence = 0.0 + if error is BoardDetectionError { + throw CaptureError.recognitionFailed + } else { + throw error + } + } + } } else { - // No board detected isBoardDetected = false self.detectedBoardRect = nil + currentPosition = nil + recognitionConfidence = 0.0 - // Only update the full capture image if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) { self.capturedImage = NSImage(cgImage: cgImage, size: .zero) } diff --git a/cline_docs/Info.txt b/cline_docs/Info.txt index 7f03501..7f989e4 100644 --- a/cline_docs/Info.txt +++ b/cline_docs/Info.txt @@ -1,571 +1,593 @@ +=== INITIALIZING PIECE RECOGNIZER === +Model loaded successfully from: /Users/chaulmark/Library/Developer/Xcode/DerivedData/ChessPrism-eefzflqwaismgqgjoetabiujjcgr/Build/Products/Debug/ChessPrism.app/Contents/Resources/ChessPieceClassifier.mlmodelc +AddInstanceForFactory: No factory registered for id F8BB1C28-BAE8-11D6-9C31-00039315CD46 +=== SCAN BUTTON PRESSED === +Captured image: (919.0, 670.0) +Analyzing board position... +=== ANALYZING BOARD === -Command queue creation failed. Worst processes ( - { - Count = 2017; - Process = "pid 25345, ChessPrism"; - }, - { - Count = 4; - Process = "pid 1227, firefox"; - }, - { - Count = 3; - Process = "pid 579, WindowServer"; - }, - { - Count = 3; - Process = "pid 1926, Creative Cloud U"; - }, - { - Count = 2; - Process = "pid 1754, Ollama Helper (G"; - }, - { - Count = 2; - Process = "pid 1928, LogiTune Helper "; - }, - { - Count = 2; - Process = "pid 11425, Xcode"; - }, - { - Count = 1; - Process = "pid 1062, mediaanalysisd"; - }, - { - Count = 1; - Process = "pid 1101, NotificationCent"; - }, - { - Count = 1; - Process = "pid 582, loginwindow"; - }, - { - Count = 1; - Process = "pid 1242, Finder"; - }, - { - Count = 1; - Process = "pid 1100, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 1237, Notes"; - }, - { - Count = 1; - Process = "pid 1222, Terminal"; - }, - { - Count = 1; - Process = "pid 1867, Docker Desktop H"; - }, - { - Count = 1; - Process = "pid 1970, Logi AI Prompt B"; - }, - { - Count = 1; - Process = "pid 1040, ControlCenter"; - }, - { - Count = 1; - Process = "pid 905, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 7637, ControlCenterHel"; - }, - { - Count = 1; - Process = "pid 15435, Code Helper (GPU"; - }, - { - Count = 0; - Process = "pid 1208, avconferenced"; - }, - { - Count = 0; - Process = "pid 1228, VTDecoderXPCServ"; - }, - { - Count = 0; - Process = "pid 1248, Spotlight"; - }, - { - Count = 0; - Process = "pid 1186, replayd"; - }, - { - Count = 0; - Process = "pid 1459, naturallanguaged"; - }, - { - Count = 0; - Process = "pid 577, com.apple.cmio.r"; - }, - { - Count = 0; - Process = "pid 2216, VTEncoderXPCServ"; - }, - { - Count = 0; - Process = "pid 7667, QuickLookUIServi"; - } -) -Command queue creation failed. Worst processes ( - { - Count = 2017; - Process = "pid 25345, ChessPrism"; - }, - { - Count = 4; - Process = "pid 1227, firefox"; - }, - { - Count = 3; - Process = "pid 579, WindowServer"; - }, - { - Count = 3; - Process = "pid 1926, Creative Cloud U"; - }, - { - Count = 2; - Process = "pid 1754, Ollama Helper (G"; - }, - { - Count = 2; - Process = "pid 1928, LogiTune Helper "; - }, - { - Count = 2; - Process = "pid 11425, Xcode"; - }, - { - Count = 1; - Process = "pid 1062, mediaanalysisd"; - }, - { - Count = 1; - Process = "pid 1101, NotificationCent"; - }, - { - Count = 1; - Process = "pid 582, loginwindow"; - }, - { - Count = 1; - Process = "pid 1242, Finder"; - }, - { - Count = 1; - Process = "pid 1100, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 1237, Notes"; - }, - { - Count = 1; - Process = "pid 1222, Terminal"; - }, - { - Count = 1; - Process = "pid 1867, Docker Desktop H"; - }, - { - Count = 1; - Process = "pid 1970, Logi AI Prompt B"; - }, - { - Count = 1; - Process = "pid 1040, ControlCenter"; - }, - { - Count = 1; - Process = "pid 905, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 7637, ControlCenterHel"; - }, - { - Count = 1; - Process = "pid 15435, Code Helper (GPU"; - }, - { - Count = 0; - Process = "pid 1208, avconferenced"; - }, - { - Count = 0; - Process = "pid 1228, VTDecoderXPCServ"; - }, - { - Count = 0; - Process = "pid 1248, Spotlight"; - }, - { - Count = 0; - Process = "pid 1186, replayd"; - }, - { - Count = 0; - Process = "pid 1459, naturallanguaged"; - }, - { - Count = 0; - Process = "pid 577, com.apple.cmio.r"; - }, - { - Count = 0; - Process = "pid 2216, VTEncoderXPCServ"; - }, - { - Count = 0; - Process = "pid 7667, QuickLookUIServi"; - } -) -Command queue creation failed. Worst processes ( - { - Count = 2017; - Process = "pid 25345, ChessPrism"; - }, - { - Count = 4; - Process = "pid 1227, firefox"; - }, - { - Count = 3; - Process = "pid 579, WindowServer"; - }, - { - Count = 3; - Process = "pid 1926, Creative Cloud U"; - }, - { - Count = 2; - Process = "pid 1754, Ollama Helper (G"; - }, - { - Count = 2; - Process = "pid 1928, LogiTune Helper "; - }, - { - Count = 2; - Process = "pid 11425, Xcode"; - }, - { - Count = 1; - Process = "pid 1062, mediaanalysisd"; - }, - { - Count = 1; - Process = "pid 1101, NotificationCent"; - }, - { - Count = 1; - Process = "pid 582, loginwindow"; - }, - { - Count = 1; - Process = "pid 1242, Finder"; - }, - { - Count = 1; - Process = "pid 1100, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 1237, Notes"; - }, - { - Count = 1; - Process = "pid 1222, Terminal"; - }, - { - Count = 1; - Process = "pid 1867, Docker Desktop H"; - }, - { - Count = 1; - Process = "pid 1970, Logi AI Prompt B"; - }, - { - Count = 1; - Process = "pid 1040, ControlCenter"; - }, - { - Count = 1; - Process = "pid 905, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 7637, ControlCenterHel"; - }, - { - Count = 1; - Process = "pid 15435, Code Helper (GPU"; - }, - { - Count = 0; - Process = "pid 1208, avconferenced"; - }, - { - Count = 0; - Process = "pid 1228, VTDecoderXPCServ"; - }, - { - Count = 0; - Process = "pid 1248, Spotlight"; - }, - { - Count = 0; - Process = "pid 1186, replayd"; - }, - { - Count = 0; - Process = "pid 1459, naturallanguaged"; - }, - { - Count = 0; - Process = "pid 577, com.apple.cmio.r"; - }, - { - Count = 0; - Process = "pid 2216, VTEncoderXPCServ"; - }, - { - Count = 0; - Process = "pid 7667, QuickLookUIServi"; - } -) -Command queue creation failed. Worst processes ( - { - Count = 2017; - Process = "pid 25345, ChessPrism"; - }, - { - Count = 4; - Process = "pid 1227, firefox"; - }, - { - Count = 3; - Process = "pid 579, WindowServer"; - }, - { - Count = 3; - Process = "pid 1926, Creative Cloud U"; - }, - { - Count = 2; - Process = "pid 1754, Ollama Helper (G"; - }, - { - Count = 2; - Process = "pid 1928, LogiTune Helper "; - }, - { - Count = 2; - Process = "pid 11425, Xcode"; - }, - { - Count = 1; - Process = "pid 1062, mediaanalysisd"; - }, - { - Count = 1; - Process = "pid 1101, NotificationCent"; - }, - { - Count = 1; - Process = "pid 582, loginwindow"; - }, - { - Count = 1; - Process = "pid 1242, Finder"; - }, - { - Count = 1; - Process = "pid 1100, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 1237, Notes"; - }, - { - Count = 1; - Process = "pid 1222, Terminal"; - }, - { - Count = 1; - Process = "pid 1867, Docker Desktop H"; - }, - { - Count = 1; - Process = "pid 1970, Logi AI Prompt B"; - }, - { - Count = 1; - Process = "pid 1040, ControlCenter"; - }, - { - Count = 1; - Process = "pid 905, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 7637, ControlCenterHel"; - }, - { - Count = 1; - Process = "pid 15435, Code Helper (GPU"; - }, - { - Count = 0; - Process = "pid 1208, avconferenced"; - }, - { - Count = 0; - Process = "pid 1228, VTDecoderXPCServ"; - }, - { - Count = 0; - Process = "pid 1248, Spotlight"; - }, - { - Count = 0; - Process = "pid 1186, replayd"; - }, - { - Count = 0; - Process = "pid 1459, naturallanguaged"; - }, - { - Count = 0; - Process = "pid 577, com.apple.cmio.r"; - }, - { - Count = 0; - Process = "pid 2216, VTEncoderXPCServ"; - }, - { - Count = 0; - Process = "pid 7667, QuickLookUIServi"; - } -) -Command queue creation failed. Worst processes ( - { - Count = 2017; - Process = "pid 25345, ChessPrism"; - }, - { - Count = 4; - Process = "pid 1227, firefox"; - }, - { - Count = 3; - Process = "pid 579, WindowServer"; - }, - { - Count = 3; - Process = "pid 1926, Creative Cloud U"; - }, - { - Count = 2; - Process = "pid 1754, Ollama Helper (G"; - }, - { - Count = 2; - Process = "pid 1928, LogiTune Helper "; - }, - { - Count = 2; - Process = "pid 11425, Xcode"; - }, - { - Count = 1; - Process = "pid 1062, mediaanalysisd"; - }, - { - Count = 1; - Process = "pid 1101, NotificationCent"; - }, - { - Count = 1; - Process = "pid 582, loginwindow"; - }, - { - Count = 1; - Process = "pid 1242, Finder"; - }, - { - Count = 1; - Process = "pid 1100, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 1237, Notes"; - }, - { - Count = 1; - Process = "pid 1222, Terminal"; - }, - { - Count = 1; - Process = "pid 1867, Docker Desktop H"; - }, - { - Count = 1; - Process = "pid 1970, Logi AI Prompt B"; - }, - { - Count = 1; - Process = "pid 1040, ControlCenter"; - }, - { - Count = 1; - Process = "pid 905, iconservicesagen"; - }, - { - Count = 1; - Process = "pid 7637, ControlCenterHel"; - }, - { - Count = 1; - Process = "pid 15435, Code Helper (GPU"; - }, - { - Count = 0; - Process = "pid 1208, avconferenced"; - }, - { - Count = 0; - Process = "pid 1228, VTDecoderXPCServ"; - }, - { - Count = 0; - Process = "pid 1248, Spotlight"; - }, - { - Count = 0; - Process = "pid 1186, replayd"; - }, - { - Count = 0; - Process = "pid 1459, naturallanguaged"; - }, - { - Count = 0; - Process = "pid 577, com.apple.cmio.r"; - }, - { - Count = 0; - Process = "pid 2216, VTEncoderXPCServ"; - }, - { - Count = 0; - Process = "pid 7667, QuickLookUIServi"; - } -) \ No newline at end of file +Classification results for BoardPosition(file: 0, rank: 0): +- white_rook: 0.9999322 +- black_rook: 6.774748e-05 +- white_king: 7.3981004e-09 + +Classification results for BoardPosition(file: 1, rank: 0): +- white_knight: 1.0 +- black_knight: 1.1934526e-17 +- white_bishop: 1.2330606e-24 + +Classification results for BoardPosition(file: 2, rank: 0): +- black_knight: 0.9997147 +- black_rook: 0.00027883475 +- black_bishop: 5.89228e-06 + +Classification results for BoardPosition(file: 3, rank: 0): +- white_king: 0.9983813 +- white_rook: 0.0010669202 +- white_knight: 0.00041843182 + +Classification results for BoardPosition(file: 4, rank: 0): +- white_king: 1.0 +- black_king: 6.379774e-10 +- black_bishop: 1.0737971e-14 + +Classification rejected at BoardPosition(file: 4, rank: 0): +Top result: white_king (1.0) +Second best: black_king (6.379774e-10) +Confidence ratio: 8343953.0 +Adjusted confidence: 1.0 + +Classification results for BoardPosition(file: 5, rank: 0): +- white_rook: 0.58311313 +- white_king: 0.4168078 +- black_bishop: 7.6859214e-05 + +Classification rejected at BoardPosition(file: 5, rank: 0): +Top result: white_rook (0.58311313) +Second best: white_king (0.4168078) +Confidence ratio: 1.3989973 +Adjusted confidence: 0.58311313 + +Classification results for BoardPosition(file: 6, rank: 0): +- white_knight: 1.0 +- black_knight: 1.3012843e-15 +- white_bishop: 4.952984e-22 + +Classification results for BoardPosition(file: 7, rank: 0): +- white_rook: 0.9992894 +- black_rook: 0.0007105772 +- white_king: 5.2344448e-11 + +Classification results for BoardPosition(file: 0, rank: 1): +- white_pawn: 1.0 +- white_king: 7.0033545e-12 +- black_pawn: 1.2942099e-12 + +Classification results for BoardPosition(file: 1, rank: 1): +- white_pawn: 1.0 +- black_pawn: 1.6859661e-13 +- black_bishop: 2.603961e-19 + +Classification results for BoardPosition(file: 2, rank: 1): +- white_rook: 0.89660114 +- white_king: 0.086419925 +- white_knight: 0.016850794 + +Classification rejected at BoardPosition(file: 2, rank: 1): +Top result: white_rook (0.89660114) +Second best: white_king (0.086419925) +Confidence ratio: 10.374921 +Adjusted confidence: 0.89660114 + +Classification results for BoardPosition(file: 3, rank: 1): +- black_knight: 0.9998835 +- black_pawn: 7.848534e-05 +- black_rook: 3.68113e-05 + +Classification results for BoardPosition(file: 4, rank: 1): +- white_king: 0.7772867 +- white_rook: 0.21319194 +- white_knight: 0.008732727 + +Classification rejected at BoardPosition(file: 4, rank: 1): +Top result: white_king (0.7772867) +Second best: white_rook (0.21319194) +Confidence ratio: 3.6459458 +Adjusted confidence: 0.7772867 + +Classification results for BoardPosition(file: 5, rank: 1): +- white_pawn: 1.0 +- black_pawn: 1.3066745e-13 +- black_bishop: 3.320688e-20 + +Classification results for BoardPosition(file: 6, rank: 1): +- white_pawn: 1.0 +- black_pawn: 2.7381055e-16 +- black_bishop: 1.1745517e-21 + +Classification results for BoardPosition(file: 7, rank: 1): +- white_pawn: 1.0 +- black_pawn: 7.737975e-12 +- black_bishop: 1.797723e-18 + +Classification results for BoardPosition(file: 0, rank: 2): +- black_rook: 0.99889785 +- black_knight: 0.0009475333 +- black_bishop: 0.00013781422 + +Classification results for BoardPosition(file: 1, rank: 2): +- white_bishop: 1.0 +- white_knight: 5.039787e-13 +- black_bishop: 2.1238877e-13 + +Classification results for BoardPosition(file: 2, rank: 2): +- white_pawn: 1.0 +- black_pawn: 5.417135e-12 +- black_bishop: 6.220996e-21 + +Classification results for BoardPosition(file: 3, rank: 2): +- black_knight: 0.83722544 +- white_knight: 0.1627746 +- black_bishop: 2.8136882e-19 + +Classification rejected at BoardPosition(file: 3, rank: 2): +Top result: black_knight (0.83722544) +Second best: white_knight (0.1627746) +Confidence ratio: 5.143461 +Adjusted confidence: 0.83722544 + +Classification results for BoardPosition(file: 4, rank: 2): +- black_knight: 0.81991595 +- black_rook: 0.17813577 +- black_pawn: 0.0017957124 + +Classification rejected at BoardPosition(file: 4, rank: 2): +Top result: black_knight (0.81991595) +Second best: black_rook (0.17813577) +Confidence ratio: 4.6027555 +Adjusted confidence: 0.81991595 + +Classification results for BoardPosition(file: 5, rank: 2): +- white_rook: 0.8535699 +- white_king: 0.124892734 +- white_knight: 0.02131261 + +Classification rejected at BoardPosition(file: 5, rank: 2): +Top result: white_rook (0.8535699) +Second best: white_king (0.124892734) +Confidence ratio: 6.834418 +Adjusted confidence: 0.8535699 + +Classification results for BoardPosition(file: 6, rank: 2): +- black_knight: 0.94812983 +- black_rook: 0.051835023 +- black_pawn: 3.4919147e-05 + +Classification rejected at BoardPosition(file: 6, rank: 2): +Top result: black_knight (0.94812983) +Second best: black_rook (0.051835023) +Confidence ratio: 18.291256 +Adjusted confidence: 0.94812983 + +Classification results for BoardPosition(file: 7, rank: 2): +- white_rook: 0.9989638 +- white_knight: 0.0006915265 +- black_knight: 0.00022520062 + +Classification rejected at BoardPosition(file: 7, rank: 2): +Top result: white_rook (0.9989638) +Second best: white_knight (0.0006915265) +Confidence ratio: 1444.3287 +Adjusted confidence: 0.9989638 + +Classification results for BoardPosition(file: 0, rank: 3): +- white_rook: 0.99999315 +- white_king: 5.72426e-06 +- white_knight: 1.0547665e-06 + +Classification rejected at BoardPosition(file: 0, rank: 3): +Top result: white_rook (0.99999315) +Second best: white_king (5.72426e-06) +Confidence ratio: 171130.05 +Adjusted confidence: 0.99999315 + +Classification results for BoardPosition(file: 1, rank: 3): +- black_knight: 0.9960707 +- black_rook: 0.0038440374 +- black_pawn: 7.4432515e-05 + +Classification rejected at BoardPosition(file: 1, rank: 3): +Top result: black_knight (0.9960707) +Second best: black_rook (0.0038440374) +Confidence ratio: 259.1129 +Adjusted confidence: 0.9960707 + +Classification results for BoardPosition(file: 2, rank: 3): +- white_rook: 0.9182533 +- white_knight: 0.067995325 +- white_king: 0.01360172 + +Classification rejected at BoardPosition(file: 2, rank: 3): +Top result: white_rook (0.9182533) +Second best: white_knight (0.067995325) +Confidence ratio: 13.50463 +Adjusted confidence: 0.9182533 + +Classification results for BoardPosition(file: 3, rank: 3): +- white_bishop: 1.0 +- white_knight: 1.1954664e-12 +- black_bishop: 7.699053e-15 + +Classification results for BoardPosition(file: 4, rank: 3): +- white_rook: 0.7775835 +- white_king: 0.1671592 +- white_knight: 0.055232733 + +Classification rejected at BoardPosition(file: 4, rank: 3): +Top result: white_rook (0.7775835) +Second best: white_king (0.1671592) +Confidence ratio: 4.6517506 +Adjusted confidence: 0.7775835 + +Classification results for BoardPosition(file: 5, rank: 3): +- black_knight: 0.99874353 +- white_knight: 0.0012564451 +- black_bishop: 3.574968e-22 + +Classification rejected at BoardPosition(file: 5, rank: 3): +Top result: black_knight (0.99874353) +Second best: white_knight (0.0012564451) +Confidence ratio: 794.82086 +Adjusted confidence: 0.99874353 + +Classification results for BoardPosition(file: 6, rank: 3): +- white_rook: 0.9706584 +- white_knight: 0.022920413 +- white_king: 0.0057876245 + +Classification rejected at BoardPosition(file: 6, rank: 3): +Top result: white_rook (0.9706584) +Second best: white_knight (0.022920413) +Confidence ratio: 42.34886 +Adjusted confidence: 0.9706584 + +Classification results for BoardPosition(file: 7, rank: 3): +- black_knight: 0.9966006 +- black_rook: 0.003287367 +- black_pawn: 9.90469e-05 + +Classification rejected at BoardPosition(file: 7, rank: 3): +Top result: black_knight (0.9966006) +Second best: black_rook (0.003287367) +Confidence ratio: 303.14975 +Adjusted confidence: 0.9966006 + +Classification results for BoardPosition(file: 0, rank: 4): +- white_pawn: 0.98597604 +- black_pawn: 0.0140145635 +- black_bishop: 9.396416e-06 + +Classification results for BoardPosition(file: 1, rank: 4): +- white_rook: 0.9300271 +- white_knight: 0.060600415 +- white_king: 0.00936738 + +Classification rejected at BoardPosition(file: 1, rank: 4): +Top result: white_rook (0.9300271) +Second best: white_knight (0.060600415) +Confidence ratio: 15.346847 +Adjusted confidence: 0.9300271 + +Classification results for BoardPosition(file: 2, rank: 4): +- black_knight: 0.99934757 +- white_rook: 0.00042515533 +- white_pawn: 0.00011956119 + +Classification rejected at BoardPosition(file: 2, rank: 4): +Top result: black_knight (0.99934757) +Second best: white_rook (0.00042515533) +Confidence ratio: 2349.888 +Adjusted confidence: 0.99934757 + +Classification results for BoardPosition(file: 3, rank: 4): +- white_king: 0.6209436 +- white_rook: 0.28476056 +- white_knight: 0.0942869 + +Classification rejected at BoardPosition(file: 3, rank: 4): +Top result: white_king (0.6209436) +Second best: white_rook (0.28476056) +Confidence ratio: 2.1805806 +Adjusted confidence: 0.6209436 + +Classification results for BoardPosition(file: 4, rank: 4): +- white_king: 0.96777105 +- white_pawn: 0.032224275 +- white_queen: 3.3261906e-06 + +Classification rejected at BoardPosition(file: 4, rank: 4): +Top result: white_king (0.96777105) +Second best: white_pawn (0.032224275) +Confidence ratio: 30.03225 +Adjusted confidence: 0.96777105 + +Classification results for BoardPosition(file: 5, rank: 4): +- white_knight: 0.40687096 +- white_king: 0.30544162 +- white_rook: 0.28753942 + +Classification rejected at BoardPosition(file: 5, rank: 4): +Top result: white_knight (0.40687096) +Second best: white_king (0.30544162) +Confidence ratio: 1.3320739 +Adjusted confidence: 0.40687096 + +Classification results for BoardPosition(file: 6, rank: 4): +- black_knight: 0.9783021 +- black_rook: 0.020397034 +- black_pawn: 0.0010299487 + +Classification rejected at BoardPosition(file: 6, rank: 4): +Top result: black_knight (0.9783021) +Second best: black_rook (0.020397034) +Confidence ratio: 47.96268 +Adjusted confidence: 0.9783021 + +Classification results for BoardPosition(file: 7, rank: 4): +- white_pawn: 0.99999607 +- black_pawn: 3.9582374e-06 +- black_bishop: 6.14308e-14 + +Classification results for BoardPosition(file: 0, rank: 5): +- white_rook: 0.9999859 +- white_king: 1.4105989e-05 +- white_knight: 3.5917537e-08 + +Classification rejected at BoardPosition(file: 0, rank: 5): +Top result: white_rook (0.9999859) +Second best: white_king (1.4105989e-05) +Confidence ratio: 70296.8 +Adjusted confidence: 0.9999859 + +Classification results for BoardPosition(file: 1, rank: 5): +- white_queen: 1.0 +- black_queen: 6.9438193e-09 +- white_bishop: 1.4069858e-13 + +Classification results for BoardPosition(file: 2, rank: 5): +- white_rook: 0.99900144 +- white_king: 0.0006145796 +- white_knight: 0.0003839481 + +Classification rejected at BoardPosition(file: 2, rank: 5): +Top result: white_rook (0.99900144) +Second best: white_king (0.0006145796) +Confidence ratio: 1625.1885 +Adjusted confidence: 0.99900144 + +Classification results for BoardPosition(file: 3, rank: 5): +- white_pawn: 0.9735881 +- black_pawn: 0.026411904 +- black_bishop: 3.9050807e-13 + +Classification rejected at BoardPosition(file: 3, rank: 5): +Top result: white_pawn (0.9735881) +Second best: black_pawn (0.026411904) +Confidence ratio: 36.86155 +Adjusted confidence: 0.9735881 + +Classification results for BoardPosition(file: 4, rank: 5): +- white_pawn: 0.99999356 +- black_pawn: 6.4090927e-06 +- black_bishop: 3.2518378e-14 + +Classification rejected at BoardPosition(file: 4, rank: 5): +Top result: white_pawn (0.99999356) +Second best: black_pawn (6.4090927e-06) +Confidence ratio: 153178.2 +Adjusted confidence: 0.99999356 + +Classification results for BoardPosition(file: 5, rank: 5): +- black_knight: 0.9346667 +- black_rook: 0.06530247 +- black_bishop: 1.5262372e-05 + +Classification rejected at BoardPosition(file: 5, rank: 5): +Top result: black_knight (0.9346667) +Second best: black_rook (0.06530247) +Confidence ratio: 14.312859 +Adjusted confidence: 0.9346667 + +Classification results for BoardPosition(file: 6, rank: 5): +- white_rook: 0.99926656 +- white_knight: 0.00059136405 +- white_king: 0.00014079778 + +Classification rejected at BoardPosition(file: 6, rank: 5): +Top result: white_rook (0.99926656) +Second best: white_knight (0.00059136405) +Confidence ratio: 1689.4249 +Adjusted confidence: 0.99926656 + +Classification results for BoardPosition(file: 7, rank: 5): +- black_knight: 0.9998666 +- black_rook: 0.00013308175 +- white_rook: 1.6393918e-07 + +Classification rejected at BoardPosition(file: 7, rank: 5): +Top result: black_knight (0.9998666) +Second best: black_rook (0.00013308175) +Confidence ratio: 7506.452 +Adjusted confidence: 0.9998666 + +Classification results for BoardPosition(file: 0, rank: 6): +- white_rook: 0.6130078 +- black_knight: 0.3607682 +- black_rook: 0.026095843 + +Classification rejected at BoardPosition(file: 0, rank: 6): +Top result: white_rook (0.6130078) +Second best: black_knight (0.3607682) +Confidence ratio: 1.699173 +Adjusted confidence: 0.6130078 + +Classification results for BoardPosition(file: 1, rank: 6): +- white_pawn: 0.99999845 +- black_pawn: 1.5546049e-06 +- black_bishop: 1.3642005e-11 + +Classification rejected at BoardPosition(file: 1, rank: 6): +Top result: white_pawn (0.99999845) +Second best: black_pawn (1.5546049e-06) +Confidence ratio: 597436.94 +Adjusted confidence: 0.99999845 + +Classification results for BoardPosition(file: 2, rank: 6): +- black_knight: 0.99884737 +- white_rook: 0.0008275975 +- black_rook: 0.00031895642 + +Classification rejected at BoardPosition(file: 2, rank: 6): +Top result: black_knight (0.99884737) +Second best: white_rook (0.0008275975) +Confidence ratio: 1206.7502 +Adjusted confidence: 0.99884737 + +Classification results for BoardPosition(file: 3, rank: 6): +- white_bishop: 0.99873775 +- black_bishop: 0.0012556859 +- white_knight: 6.535806e-06 + +Classification rejected at BoardPosition(file: 3, rank: 6): +Top result: white_bishop (0.99873775) +Second best: black_bishop (0.0012556859) +Confidence ratio: 795.29675 +Adjusted confidence: 0.99873775 + +Classification results for BoardPosition(file: 4, rank: 6): +- black_rook: 0.91434133 +- white_rook: 0.054479513 +- black_knight: 0.029062644 + +Classification rejected at BoardPosition(file: 4, rank: 6): +Top result: black_rook (0.91434133) +Second best: white_rook (0.054479513) +Confidence ratio: 16.783176 +Adjusted confidence: 0.91434133 + +Classification results for BoardPosition(file: 5, rank: 6): +- white_pawn: 0.99999875 +- black_pawn: 1.2670436e-06 +- black_bishop: 1.4581098e-14 + +Classification rejected at BoardPosition(file: 5, rank: 6): +Top result: white_pawn (0.99999875) +Second best: black_pawn (1.2670436e-06) +Confidence ratio: 721368.25 +Adjusted confidence: 0.99999875 + +Classification results for BoardPosition(file: 6, rank: 6): +- white_pawn: 0.9924884 +- black_pawn: 0.0075116316 +- black_bishop: 5.484788e-10 + +Classification rejected at BoardPosition(file: 6, rank: 6): +Top result: white_pawn (0.9924884) +Second best: black_pawn (0.0075116316) +Confidence ratio: 132.12477 +Adjusted confidence: 0.9924884 + +Classification results for BoardPosition(file: 7, rank: 6): +- white_rook: 0.9999656 +- white_king: 1.7366758e-05 +- white_knight: 1.700485e-05 + +Classification rejected at BoardPosition(file: 7, rank: 6): +Top result: white_rook (0.9999656) +Second best: white_king (1.7366758e-05) +Confidence ratio: 57186.75 +Adjusted confidence: 0.9999656 + +Classification results for BoardPosition(file: 0, rank: 7): +- black_rook: 0.9481192 +- white_rook: 0.051880363 +- white_knight: 3.7680303e-07 + +Classification results for BoardPosition(file: 1, rank: 7): +- black_rook: 0.9994105 +- black_knight: 0.00050535565 +- white_rook: 7.980196e-05 + +Classification rejected at BoardPosition(file: 1, rank: 7): +Top result: black_rook (0.9994105) +Second best: black_knight (0.00050535565) +Confidence ratio: 1977.1715 +Adjusted confidence: 0.9994105 + +Classification results for BoardPosition(file: 2, rank: 7): +- white_rook: 0.52612 +- white_king: 0.47385767 +- white_knight: 2.2209799e-05 + +Classification rejected at BoardPosition(file: 2, rank: 7): +Top result: white_rook (0.52612) +Second best: white_king (0.47385767) +Confidence ratio: 1.1102909 +Adjusted confidence: 0.52612 + +Classification results for BoardPosition(file: 3, rank: 7): +- white_queen: 0.90863794 +- black_queen: 0.09136204 +- black_bishop: 2.6315076e-12 + +Classification rejected at BoardPosition(file: 3, rank: 7): +Top result: white_queen (0.90863794) +Second best: black_queen (0.09136204) +Confidence ratio: 9.945452 +Adjusted confidence: 0.90863794 + +Classification results for BoardPosition(file: 4, rank: 7): +- white_king: 0.9999997 +- black_king: 2.3221618e-07 +- black_bishop: 3.7497816e-08 + +Classification rejected at BoardPosition(file: 4, rank: 7): +Top result: white_king (0.9999997) +Second best: black_king (2.3221618e-07) +Confidence ratio: 2845552.8 +Adjusted confidence: 0.9999997 + +Classification results for BoardPosition(file: 5, rank: 7): +- white_bishop: 0.99834645 +- black_bishop: 0.0016535616 +- white_knight: 1.7680212e-09 + +Classification rejected at BoardPosition(file: 5, rank: 7): +Top result: white_bishop (0.99834645) +Second best: black_bishop (0.0016535616) +Confidence ratio: 603.7117 +Adjusted confidence: 0.99834645 + +Classification results for BoardPosition(file: 6, rank: 7): +- white_king: 0.5149147 +- white_rook: 0.48506972 +- white_knight: 1.5106151e-05 + +Classification rejected at BoardPosition(file: 6, rank: 7): +Top result: white_king (0.5149147) +Second best: white_rook (0.48506972) +Confidence ratio: 1.0615269 +Adjusted confidence: 0.5149147 + +Classification results for BoardPosition(file: 7, rank: 7): +- black_rook: 0.9999986 +- white_rook: 1.368569e-06 +- black_queen: 2.4758154e-11 + +Classification rejected at BoardPosition(file: 7, rank: 7): +Top result: black_rook (0.9999986) +Second best: white_rook (1.368569e-06) +Confidence ratio: 672142.25 +Adjusted confidence: 1.0 +ERROR: Position analysis failed - invalidPosition("Invalid black king count: 0") +ERROR: Snapshot failed - invalidPosition("Invalid black king count: 0") \ No newline at end of file diff --git a/cline_docs/activeContext.md b/cline_docs/activeContext.md index 5809550..8383ce3 100644 --- a/cline_docs/activeContext.md +++ b/cline_docs/activeContext.md @@ -1,102 +1,29 @@ -# Active Context +# Current Task +Working on chess piece recognition with the updated ML model that includes empty square detection. -## Current Status -- Screen capture module successfully implemented -- Window detection and capture working correctly -- SwiftUI interface with capture controls functioning -- Error handling system properly managing states -- Resource cleanup implemented -- Metal resource management optimized -- Automatic board detection and capture implemented -- Visual capture status indicator added -- Continuous board monitoring system implemented -- Auto-capture on game start/stop working successfully -- Manual snapshot system implemented with Scan button -- Snapshot preview display added below Chessboard Preview -- Cursor-free snapshot capture implemented +# Recent Changes +1. Empty Square Detection: + - Handle empty_dark/empty_light classes + - Exact class name matching + - Proper error handling -## Recent Changes -1. Enhanced Manual Snapshot System: - - New cursor-free capture: - * Temporarily disables cursor during snapshot - * Ensures clean board capture without mouse pointer - * Automatically restores cursor after snapshot - - Improved snapshot process: - * Stops current capture - * Takes cursor-free snapshot - * Processes board detection - * Restores normal capture - * Handles errors gracefully - - Snapshot visualization: - * Added preview area below Chessboard Preview - * Shows latest snapshot with visual feedback - * Clear indication when snapshot is taken +2. Position Validation: + - Allow moved pieces + - Essential rules only + - Piece count tracking -2. Implemented Continuous Board Monitoring: - - Added separate monitoring and capture tasks: - * Monitor constantly checks for chess boards (0.5s interval) - * Capture processes frames when active (0.1s interval) - - Auto-capture behavior working as expected: - * Starts monitoring when app launches - * Automatically starts capture when board appears - * Stops capture but continues monitoring when board disappears - * Successfully resumes capture when new game starts - - Performance characteristics: - * ~40% CPU usage during operation - * Stable memory management - * Responsive to game state changes +3. Error Handling: + - Better error messages + - Clear logging + - Fixed optional unwrapping -3. Enhanced Status Indication: - - Visual status indicator shows capture state: - * Green: Actively capturing board - * Yellow: Waiting for board - * Gray: Not capturing - - Clear error messages for different states - - Automatic status updates based on board detection +# Next Steps +1. Recognition Tuning: + - Fine-tune empty square detection + - Adjust confidence thresholds + - Improve position validation -4. Resource Management: - - Implemented shared CIContext pattern: - * Prevents command queue exhaustion - * Reduces Metal resource usage - * Enables long-running captures - - Proper cleanup on task completion - - Efficient resource utilization - -5. Improved Window Detection: - - Using SCShareableContent for window access - - Precise window identification: - * Exact bundle ID matching (com.chess.iphone) - * Window visibility verification (isOnScreen) - * Size validation (width > 100 && height > 100) - -6. Error Handling: - - Improved error resilience: - * Continues monitoring even if capture stops - * Only stops on critical errors - * Shows error state without interrupting monitoring - - Clear error states - - Proper async/await usage - - Task cancellation management - - Thread-safe state updates - -## Current Focus -1. Board Recognition: - - Process snapshot images - - Implement piece detection - - Extract board state - -## Next Steps -1. Implement board recognition: - - Process snapshot images - - Detect chess pieces - - Map board coordinates - - Validate positions -2. Add position analysis -3. Create move detection system -4. Implement visual overlay -5. Integrate Stockfish engine - -## Known Issues -- Need to handle different chess.com themes -- Need to implement piece recognition -- Position analysis pending implementation +2. Model Training: + - Add more empty square examples + - Include different board styles + - Improve piece variety diff --git a/cline_docs/chessboard.txt b/cline_docs/chessboard.txt new file mode 100644 index 0000000..85be765 --- /dev/null +++ b/cline_docs/chessboard.txt @@ -0,0 +1,20 @@ +Current position: +8 bR .. .. bQ bK bB .. bR +7 .. bP .. bB .. bP bP .. +6 .. wQ .. bP bP .. .. .. +5 bP .. .. .. .. .. .. bP +4 .. .. .. wB .. bN .. .. +3 .. wB wP bN .. .. .. .. +2 wP wP .. .. .. wP wP wP +1 wR wN .. .. wK .. wN wR + a b c d e f g h + +Key differences from recognition: +4. Some pieces missing from recognition +5. Some pieces misidentified + +Recognition issues to fix: +1. Need to handle moved pieces (not just starting position) +2. Better empty square detection +3. Improve confidence thresholds +4. Validate complete position diff --git a/cline_docs/error b/cline_docs/error deleted file mode 100644 index 1bae1d5..0000000 --- a/cline_docs/error +++ /dev/null @@ -1,138 +0,0 @@ -# Error Resolution Log - -## Fixed Issues (2023) - -### ScreenCaptureKit API Updates -1. SCContentFilter Initialization - - Fixed by using correct initializer and parameters: - ```swift - SCContentFilter(display: display, excludingWindows: []) - ``` - - Using SCDisplay object directly (not displayID) - - Correct parameter name: excludingWindows - - Removed incorrect parameters (includingWindows/exceptingWindows) - -2. Stream Output Type - - Fixed type inference issue by explicit declaration: - ```swift - let outputType: SCStreamOutputType = .screen - try stream.addStreamOutput(self, type: outputType, ...) - ``` - - Ensures proper type resolution for .screen member - -### Required Imports -- Added necessary framework imports: - * CoreMedia - * AVFoundation - * ScreenCaptureKit - * CoreGraphics - * AppKit - * Foundation - -### Window Capture Strategy -1. Window Detection - - Precise window identification: - ```swift - let bundleID = window.owningApplication?.bundleIdentifier ?? "" - let isChessApp = bundleID == "com.chess.iphone" - let hasValidSize = window.frame.width > 100 && window.frame.height > 100 - return isChessApp && window.isOnScreen && hasValidSize - ``` - - Multiple validation checks: - * Exact bundle ID match - * Window is currently on screen - * Window has valid dimensions - - Handles iOS apps running on Mac properly - -2. Capture Method - - Implemented continuous capture: - ```swift - // Start once - try await screenCapture.startCapture() - - // Process frames continuously - while !Task.isCancelled { - if let image = screenCapture.getCurrentImage() { - try await processImage(image) - } - try await Task.sleep(nanoseconds: 100_000_000) - } - ``` - - Maintains single active stream - - Eliminates capture flickering - - Proper cleanup on stop - -### Thread Safety and Async Handling -1. Main Actor Isolation - - Added @MainActor to ViewModel class: - ```swift - @MainActor - class ScreenCaptureViewModel: ObservableObject - ``` - - Ensures all @Published property updates happen on main thread - - Proper thread safety for SwiftUI bindings - -2. Async Operation Handling - - Optimized async/await usage: - ```swift - // Only use await for truly async operations - try await screenCapture.startCapture() - try await Task.sleep(nanoseconds: 100_000_000) - ``` - - Removed unnecessary await keywords: - * Non-async error handling - * UI state updates - * Image processing - - Proper @MainActor usage for thread safety - -3. Task and Error Handling - - Improved task cancellation: - ```swift - captureLoop: while !Task.isCancelled { - do { - // Process frame - } catch is CancellationError { - break captureLoop - } catch { - // Continue capturing on non-critical errors - await handleCaptureError(error) - } - } - ``` - - Proper error recovery: - * Continues on board detection failures - * Shows error state without stopping - * Clears errors on successful detection - - Clean error state on stop: - ```swift - captureError = nil // Clear any error when stopping - ``` - - Proper cleanup on task exit - -## Technical Notes -- Resource Management: - * Shared CIContext to prevent command queue exhaustion: - ```swift - private static let shared = CIContext() - private var context: CIContext { Self.shared } - ``` - * Prevents "Command queue creation failed" errors - * Reduces Metal resource usage - * Proper cleanup on task completion - -- Using correct SCContentFilter API with proper parameter names -- Proper type safety throughout the implementation: - * Proper Vision framework result handling: - ```swift - guard let observations = request.results, - !observations.isEmpty else { - return nil - } - - let bestObservation = observations[0] - ``` - * No unnecessary type casting - * Safe array access - * Proper optional handling -- Appropriate framework dependencies -- Follows current ScreenCaptureKit best practices diff --git a/cline_docs/productContext.md b/cline_docs/productContext.md index 0ae4777..2cfe0b0 100644 --- a/cline_docs/productContext.md +++ b/cline_docs/productContext.md @@ -1,199 +1,60 @@ -# Product Context +# Product Overview +ChessPrism is a macOS application that captures and analyzes chess positions from the screen in real-time. -## Project Overview -ChessPrism is an advanced chess analysis tool that enhances the online chess experience by providing real-time analysis and move suggestions. It works by capturing and analyzing the chess board from popular chess websites and platforms. +## Core Features +1. Board Detection + - Automatic chessboard location + - Perspective and size handling + - Multi-board support planned -## Core Problems Solved +2. Piece Recognition + - ML-based piece classification + - Empty square detection + - Position-aware confidence adjustments -### Chess Analysis Accessibility -- Makes professional-level chess analysis accessible during online play -- Provides real-time insights without manual position input -- Integrates seamlessly with existing chess platforms - -### Visual Recognition -- Accurately detects chess board from screen content -- Recognizes board coordinates and boundaries -- Handles various board themes and orientations -- Maintains accuracy during game play -- Provides clean board snapshots for analysis - -### Real-time Processing -- Captures and processes screen content in real-time -- Provides immediate feedback and analysis -- Maintains performance during long sessions -- Supports manual snapshot capture for detailed analysis - -## User Experience Goals - -### Seamless Integration -1. Non-intrusive Operation - - Works with Chess.com desktop app - - Minimal setup requirements - - Automatic board detection and tracking - - Clean snapshot capture without cursor interference - -2. Intuitive Interface - - Clear visualization of analysis - - Easy-to-understand suggestions - - Minimal user intervention required - - Visual feedback for capture states - - Manual snapshot control - -### Reliable Detection -1. Board Recognition - - Two-phase detection strategy: - * Pattern recognition for known interfaces - * Coordinate-based fallback for reliability - - Proper coordinate system handling - - Consistent board capture across sessions - - High-quality snapshots for analysis - -2. Position Analysis - - Accurate piece recognition (planned) - - Current position evaluation (planned) - - Move suggestion visualization (planned) - -## Target Users - -### Chess Players -- Amateur to intermediate players -- Chess.com desktop app users -- Players seeking to improve - -### Use Cases -1. Learning - - Understanding position evaluation - - Learning from mistakes - - Exploring alternative moves - - Analyzing specific positions via snapshots - -2. Analysis - - Real-time position assessment - - Move validation - - Strategic planning - - Detailed position study - -## Product Requirements - -### Essential Features -1. Board Detection (Current Focus) - - Accurate boundary recognition - - Full board capture - - Support for Chess.com desktop app - - Reliable coordinate transformations - - Clean snapshot capability - -2. Position Analysis (Planned) - - Real-time evaluation - - Move suggestions - - Tactical opportunities - -3. User Interface - - Analysis overlay - - Control panel - - Settings management - - Snapshot controls - - Visual status indicators - -### Quality Standards -1. Accuracy - - Reliable board detection - - Complete board capture - - Precise coordinate handling - - Clean snapshots without artifacts - -2. Performance - - Real-time processing - - Minimal resource usage - - Stable operation - - Efficient snapshot handling - -3. Usability - - Intuitive controls - - Clear feedback - - Minimal setup - - Simple snapshot workflow - -## Success Metrics - -### Technical Metrics -- Board detection accuracy rate -- Full board capture success rate -- Processing speed per frame -- Error recovery rate -- Snapshot quality assessment - -### User Metrics -- Setup success rate -- Analysis accuracy -- User engagement time -- Feature utilization -- Snapshot usage patterns +3. Position Analysis + - FEN string generation + - Position validation + - Move tracking (planned) ## Current Challenges -### Board Detection -1. Coordinate Systems - - Vision framework (bottom-left origin) - - NSImage/CGImage (bottom-left origin) - - SwiftUI (top-left origin) - - Proper transformations between systems +### Recognition Features +1. Empty Square Detection + - Explicit empty_dark/empty_light classes + - Direct square color recognition + - High confidence classification -2. Detection Accuracy - - Full board capture - - Consistent positioning - - Reliable boundaries - - Clean snapshots +2. Piece Recognition + - Accurate piece type detection + - Color differentiation + - Position-aware confidence -### Next Steps -1. Refine board detection - - Improve coordinate handling - - Ensure full board capture - - Validate transformations - - Optimize snapshot quality +3. Performance Optimization + - Fast-path empty detection + - Efficient classification flow + - Resource-aware processing -2. Move to position analysis - - Piece recognition - - Position evaluation - - Move suggestions +## Future Improvements -## Future Enhancements +### Short Term +1. Recognition Enhancement + - Fine-tune confidence thresholds + - Validate square colors + - Improve error messages -### Planned Features -1. Advanced Analysis - - Deep position evaluation - - Opening recognition - - Endgame tablebases - - Position comparison from snapshots +2. Position Analysis + - Move validation + - Game state tracking + - Historical context -2. Learning Tools - - Mistake analysis - - Improvement suggestions - - Progress tracking - - Position database from snapshots +### Long Term +1. Advanced Features + - Move detection + - Game recording + - Multiple board styles -3. Customization - - Analysis depth control - - Visual preference settings - - Platform-specific optimizations - - Snapshot management options - -## Product Roadmap - -### Current Phase -- Core board detection system -- Coordinate system handling -- Basic user interface -- Manual snapshot system - -### Next Phase -- Position analysis -- Move suggestion system -- Visual overlay implementation -- Enhanced snapshot analysis - -### Future Phase -- Advanced analysis features -- Learning tools integration -- Customization options -- Snapshot database and comparison tools +2. User Experience + - Confidence visualization + - Manual corrections + - Custom training diff --git a/cline_docs/systemPatterns.md b/cline_docs/systemPatterns.md index 10166d7..a1ee389 100644 --- a/cline_docs/systemPatterns.md +++ b/cline_docs/systemPatterns.md @@ -1,228 +1,81 @@ -# System Patterns +# System Architecture -## Window Capture Architecture +## Piece Recognition Pipeline +1. Board Detection + - VNDetectRectanglesRequest for board location + - Aspect ratio and size validation + - Square extraction with equal dimensions -### Window Detection Pattern -1. SCShareableContent Access - - Async/await pattern for content access - - Proper error propagation - - Permission handling +2. Image Preprocessing + - Contrast and brightness adjustment + - Unsharp mask for edge enhancement + - Consistent image scaling -2. Window Identification - - Multiple validation criteria: - ```swift - let bundleID = window.owningApplication?.bundleIdentifier ?? "" - let isChessApp = bundleID == "com.chess.iphone" - let hasValidSize = window.frame.width > 100 && window.frame.height > 100 - return isChessApp && window.isOnScreen && hasValidSize - ``` - - Fail-fast approach with guard statements - - Clear error states +3. ML Classification + - CoreML model prediction + - Confidence score analysis + - Position-based adjustments -### Capture System Pattern -1. Stream Configuration - - Window-specific capture setup - - Frame dimension matching - - Proper delegate handling - - Cursor visibility control: - * Configurable cursor display - * Clean snapshot support - * State preservation +## Recognition Patterns -2. Frame Processing - - Main thread safety for UI updates - - Efficient image conversion pipeline - - Resource cleanup +### Square Classification +1. Empty Square Detection + - Exact empty_dark/empty_light matching + - High confidence threshold (>0.9) + - Early detection and return -### Snapshot System Pattern -1. Cursor-Free Capture - - Temporary capture session: - * Disables cursor visibility - * Takes clean snapshot - * Restores normal capture - - Error handling: - * Session cleanup - * State recovery - * Capture restoration +2. Piece Recognition + - Strict label format validation + - Position-based confidence adjustment + - Piece count tracking -2. Process Flow - - Stop current capture - - Start cursor-free capture - - Wait for stabilization - - Take snapshot - - Process image - - Restore normal capture +3. Error Prevention + - Empty square validation + - Label format checking + - Position rule enforcement -### Error Handling Pattern -1. Task Management - - Proper cancellation points - - Clean state management - - Resource cleanup +### Classification Flow +1. Input Validation + - Image dimensions check + - Model availability check + - Configuration setup -2. Error States - - Clear error types - - User-friendly messages - - State recovery +2. Square Analysis + - Empty square check first + - Piece classification second + - Position validation last -## UI Architecture +3. Confidence Checks + - Empty squares: >0.9 + - Pieces: >0.98 with >5.0 ratio + - Position adjustments -### MVVM Implementation -1. ViewModel - - @MainActor for thread safety - - Published properties for state - - Clear separation of concerns +4. Error Handling + - Clear error messages + - Detailed logging + - Safe fallbacks -2. View Layer - - SwiftUI declarative UI - - State-driven updates - - Error presentation +## Validation Patterns +1. Piece Count Rules + - Maximum 1: king, queen + - Maximum 2: rooks, bishops, knights + - Maximum 8: pawns + - Track by color and type -### Async Operations -1. Task Management - - Structured concurrency - - Proper cancellation - - State synchronization +2. Position Rules + - Kings: not on opponent's back rank + - Pawns: no backward movement + - All pieces: within board bounds + - All pieces: valid movement patterns -2. State Updates - - Main thread safety - - Clear state transitions - - Error recovery +3. Piece Tracking + - Maximum piece counts + - Color-specific tracking + - Total position validation + - Captured piece limits -## Core Architecture - -### Resource Management Patterns -1. Shared CIContext Pattern - - Static shared instance: - ```swift - private static let shared = CIContext() - private var context: CIContext { Self.shared } - ``` - - Benefits: - * Prevents Metal command queue exhaustion - * Reduces resource overhead - * Enables long-running captures - - Implementation: - * Used in BoardDetector and ViewModel - * Proper cleanup on task completion - * Thread-safe access - -### Screen Capture System -- Uses ScreenCaptureKit for efficient screen capture -- Implements SCStreamOutput protocol for frame processing -- Handles capture session lifecycle and cleanup -- Manages permissions and error handling -- Optimized resource usage -- Configurable cursor visibility - -### Board Detection System -Two implemented approaches: - -1. Pattern Recognition Approach (Primary) - - Rectangle detection with Vision framework - - Aspect ratio-based filtering (0.3-0.5 for taller rectangles) - - Size-based filtering (0.4 minimum for larger areas) - - Single observation for precision - - Board extraction from upper portion - - Width-based square calculation - -2. Coordinate Detection (Fallback) - - Text recognition for board coordinates - - Rectangle detection with Vision framework - - Grid-based validation - - Coordinate-based refinement - -3. Common Infrastructure - - Asynchronous frame processing - - Dedicated processing queue - - Efficient memory management - - Performance monitoring - -### Coordinate Systems -- Vision framework: Bottom-left origin (0,0) -- NSImage/CGImage: Bottom-left origin (0,0) -- SwiftUI: Top-left origin (0,0) -- Transformations needed between systems: - 1. Vision → Screen: Flip Y coordinate - 2. Screen → Image: Direct mapping - 3. Image → View: SwiftUI handles automatically - -### Notification System -- Uses NotificationCenter for event propagation -- Key notifications: - - boardDetected: Sends detected board rectangle and confidence score - - captureStateChanged: Updates capture status - - capturedFrame: Delivers processed frames - - boardCoordinatesDetected: Reports coordinate detection - - detectionStats: Reports performance metrics - -## Design Patterns - -### MVVM Architecture -- ScreenCapture: Model layer handling capture logic -- ScreenCaptureViewModel: View model managing UI state -- ContentView: SwiftUI view for user interface - -### Observer Pattern -- NotificationCenter for loose coupling -- Enables modular component communication -- Supports async event handling - -### Error Handling -- Custom ScreenCaptureError enum -- Comprehensive error cases -- Proper error propagation - -## Technical Decisions - -### Vision Framework -- Primary tool for board detection -- Provides rectangle and text detection -- Handles various board orientations -- Requires coordinate system transformation - -### Pattern Recognition -- Focus on larger detection areas -- Use width as reference measurement -- Extract square board from top portion -- Maintain aspect ratio constraints - -### Performance Considerations -- Dedicated dispatch queue for frame processing -- Efficient memory management -- Proper resource cleanup -- Single observation optimization - -## Future Patterns - -### Planned Implementations -1. Board Position Analysis - - ML model integration - - Piece detection system - - Position validation - -2. Move Analysis - - Stockfish integration - - Real-time evaluation - - Visual overlay system - -3. State Management - - Game state tracking - - Move history - - Analysis persistence - -## Testing Patterns - -### Unit Testing -- ScreenCapture functionality -- Board detection accuracy -- Coordinate recognition - -### Integration Testing -- End-to-end capture workflow -- Vision framework integration -- Notification system - -### UI Testing -- SwiftUI interface validation -- User interaction flows -- Error state handling +4. Recognition Flow + - Empty square detection first + - Piece classification second + - Position validation last + - Clear error reporting diff --git a/cline_docs/techContext.md b/cline_docs/techContext.md index 1f5544a..83177b9 100644 --- a/cline_docs/techContext.md +++ b/cline_docs/techContext.md @@ -1,247 +1,69 @@ -# Technical Context +# Technologies Used + +## Core ML & Vision +- ChessPieceClassifier.mlmodel for piece recognition +- VNCoreMLModel for image classification +- Vision framework for board detection + +## Image Processing +- CoreImage for preprocessing +- CIColorControls and CIUnsharpMask filters +- CGImage for image manipulation ## Development Environment -- macOS development platform -- Xcode IDE -- SwiftUI for user interface -- Swift 5.x language features +- Xcode for Swift development +- Create ML for model training +- SwiftUI for UI components -## Core Technologies +# Technical Constraints -### Metal Resource Management -- Shared CIContext pattern: - * Static shared instance to prevent command queue exhaustion - * Used across BoardDetector and ViewModel - * Proper cleanup and resource management -- Performance considerations: - * Reduced Metal command queue creation - * Efficient resource utilization - * Support for long-running captures +## ML Model Capabilities +1. Classification Types: + - Pieces: pawn, rook, knight, bishop, queen, king + - Colors: black, white + - Empty squares: dark, light + - Label formats: color_piece, empty_color -### ScreenCaptureKit -- System framework for screen capture -- Implemented features: - * Window detection using SCShareableContent - * iOS app window capture support - * Real-time frame capture - * Proper error handling - * Configurable cursor visibility -- Key components: - * SCShareableContent: Window and display access - * SCContentFilter: Window-specific capture - * SCStream: Frame capture management - * SCStreamOutput: Frame processing - * SCStreamConfiguration: Capture settings including cursor control +2. Recognition Features: + - Multi-class classification + - Per-class confidence scores + - Position-aware validation + - Piece count tracking -### Vision Framework (Planned) -- Will be used for board and coordinate detection -- Key components to implement: - * VNRecognizeTextRequest: Chess coordinate detection - * VNDetectRectanglesRequest: Board boundary detection -- Planned configuration: - * Text recognition level: accurate - * Language correction: disabled - * Rectangle aspect ratio: 0.3-0.5 - * Minimum size: 0.4 - * Maximum observations: 1 +## Processing Requirements +1. Image Requirements: + - Square dimensions (1:1 ±10%) + - Non-zero dimensions + - Center-cropped squares + - Clear piece visibility -### Coordinate Systems -1. Vision Framework - - Origin: Bottom-left (0,0) - - Y-axis: Upward positive - - Normalized coordinates (0-1) - - Used in: VNRectangleObservation, VNTextObservation +2. Recognition Rules: + - Empty squares: exact class match with >0.9 confidence + - Pieces: strict format with >0.98 confidence + - Separation ratio: >5.0 between predictions + - Position validation: essential rules only -2. NSImage/CGImage - - Origin: Bottom-left (0,0) - - Y-axis: Upward positive - - Pixel coordinates - - Used in: Image cropping, processing +3. Error Prevention: + - Early empty square detection + - Strict label validation + - Safe optional handling + - Clear error messages -3. SwiftUI - - Origin: Top-left (0,0) - - Y-axis: Downward positive - - Point coordinates - - Used in: View layout, rendering +## Performance Considerations +1. Processing Flow: + - Early validation checks + - Fast empty square detection + - Efficient error handling + - Quick rejection paths -4. Transformations - - Vision → Screen: Flip Y coordinate - - Screen → Image: Scale to pixel coordinates - - Image → View: SwiftUI handles automatically +2. Resource Optimization: + - GPU acceleration for ML + - Minimal preprocessing + - Optimized validation + - Efficient logging -### SwiftUI -- Modern declarative UI framework -- Handles view lifecycle -- State management via @Published properties -- Environmental object propagation - -## Technical Constraints - -### Window Capture System -1. Window Detection - - Using SCShareableContent for window access - - Multiple validation criteria: - * Bundle ID verification - * Window visibility check - * Size validation - - Error handling for missing windows - -2. Frame Capture - - Window-specific capture configuration - - Frame dimension matching - - Proper delegate handling - - Resource cleanup - - Cursor visibility control: - * Configurable via SCStreamConfiguration - * State preservation between captures - * Clean snapshot support - -3. Performance - - Main thread safety for UI updates - - Efficient image conversion - - Proper task cancellation - - Memory management - - Shared CIContext for Metal efficiency - -4. Error Handling - - Clear error types - - User-friendly messages - - State recovery - - Resource cleanup - -### System Requirements -- macOS 12.0 or later -- Screen Capture permissions -- Sufficient CPU for real-time processing -- Adequate memory for frame buffering -- Metal-capable GPU for image processing - -## Dependencies - -### Internal -- ScreenCapture.swift: Core capture logic -- ScreenCaptureViewModel.swift: State management -- BoardDetector.swift: Pattern recognition -- ContentView.swift: User interface - -### External -- ScreenCaptureKit.framework -- Vision.framework -- SwiftUI.framework -- CoreImage.framework -- Metal.framework (via CIContext) - -## Development Guidelines - -### Code Organization -- MVVM architecture -- Protocol-oriented design -- Clear separation of concerns -- Comprehensive error handling -- Resource sharing patterns - -### Performance Optimization -- Shared CIContext for Metal efficiency -- Efficient frame processing -- Memory management -- Resource cleanup -- Background queue usage - -### Error Handling -- Custom error types -- Comprehensive error cases -- User-friendly error messages -- Proper error propagation - -## Testing Requirements - -### Unit Tests -- Board detection accuracy -- Coordinate transformations -- Error handling -- State management -- Resource management -- Cursor control functionality - -### Integration Tests -- End-to-end workflows -- Component interaction -- Event propagation -- Resource sharing -- Snapshot system - -### UI Tests -- User interaction flows -- Error state handling -- Visual feedback -- Performance monitoring -- Snapshot visualization - -## Documentation Requirements - -### Code Documentation -- Function documentation -- Parameter descriptions -- Return value documentation -- Error documentation -- Resource usage documentation - -### Architecture Documentation -- System overview -- Component interaction -- Data flow diagrams -- State management -- Resource management patterns - -## Current Challenges - -### Resource Management -1. Metal Efficiency - - Command queue management - - Shared context patterns - - Resource cleanup - - Performance monitoring - -2. Memory Usage - - Frame buffer management - - Image processing optimization - - Resource pooling - - Cleanup strategies - -### Board Detection -1. Full Capture - - Complete board visibility - - Proper positioning - - Consistent results - - Coordinate accuracy - -2. Performance - - Processing efficiency - - Memory usage - - Resource management - - Error recovery - -## Future Considerations - -### Planned Features -1. ML Model Integration - - Piece detection - - Position analysis - - Move validation - -2. Engine Integration - - Stockfish analysis - - Move evaluation - - Position scoring - -3. Visual Overlay - - Move suggestions - - Analysis visualization - - Interactive elements - -### Technical Debt -- Refactor coordinate handling -- Optimize frame processing -- Improve error recovery -- Enhanced permission handling -- Resource usage monitoring +# Development Setup +1. Clone repository +2. Open ChessPrism.xcodeproj +3. Build and run on macOS +4. Model at ChessPrism/ChessPieceClassifier.mlmodel