docs(workflow): add comprehensive git workflow standards #1
9 changed files with 1151 additions and 798 deletions
|
|
@ -1,194 +1,76 @@
|
|||
import Foundation
|
||||
import Vision
|
||||
import CoreImage
|
||||
import Vision
|
||||
|
||||
class BoardDetector {
|
||||
private let context = CIContext()
|
||||
|
||||
// Known chess interface patterns
|
||||
private struct ChessPattern {
|
||||
let interfaceType: ChessInterface
|
||||
let boardColor: CGColor
|
||||
let squareSize: CGFloat
|
||||
let cornerPattern: [CGPoint]
|
||||
}
|
||||
|
||||
enum ChessInterface {
|
||||
case chesscom
|
||||
case lichess
|
||||
}
|
||||
|
||||
// Chess.com and Lichess patterns
|
||||
private let patterns: [ChessPattern] = [
|
||||
// Chess.com light theme pattern
|
||||
ChessPattern(
|
||||
interfaceType: .chesscom,
|
||||
boardColor: CGColor(red: 0.82, green: 0.82, blue: 0.82, alpha: 1.0), // Light squares
|
||||
squareSize: 45.0, // Default square size
|
||||
cornerPattern: [
|
||||
CGPoint(x: 0, y: 0),
|
||||
CGPoint(x: 45, y: 0),
|
||||
CGPoint(x: 0, y: 45),
|
||||
CGPoint(x: 45, y: 45)
|
||||
]
|
||||
),
|
||||
// Lichess light theme pattern
|
||||
ChessPattern(
|
||||
interfaceType: .lichess,
|
||||
boardColor: CGColor(red: 0.93, green: 0.93, blue: 0.83, alpha: 1.0), // Light squares
|
||||
squareSize: 40.0, // Default square size
|
||||
cornerPattern: [
|
||||
CGPoint(x: 0, y: 0),
|
||||
CGPoint(x: 40, y: 0),
|
||||
CGPoint(x: 0, y: 40),
|
||||
CGPoint(x: 40, y: 40)
|
||||
]
|
||||
)
|
||||
]
|
||||
|
||||
// MARK: - Pattern Detection
|
||||
// Share CIContext to avoid creating too many Metal command queues
|
||||
private static let shared = CIContext()
|
||||
private var context: CIContext { BoardDetector.shared }
|
||||
|
||||
func detectBoard(in image: CIImage) -> CGRect? {
|
||||
// 1. Color-based detection
|
||||
let colorMatches = detectColorPatterns(in: image)
|
||||
// Configure rectangle detection request
|
||||
let request = VNDetectRectanglesRequest()
|
||||
request.minimumAspectRatio = 0.8 // Adjusted for standard chess board
|
||||
request.maximumAspectRatio = 1.2
|
||||
request.minimumSize = 0.4
|
||||
request.maximumObservations = 1
|
||||
request.quadratureTolerance = 30
|
||||
request.minimumConfidence = 0.9
|
||||
|
||||
// 2. Grid pattern detection
|
||||
let gridMatches = detectGridPattern(in: image, colorMatches: colorMatches)
|
||||
|
||||
// 3. Corner validation
|
||||
if let bestMatch = validateCorners(in: image, candidates: gridMatches) {
|
||||
// Extract square board from detected area
|
||||
let width = bestMatch.width
|
||||
let height = bestMatch.height
|
||||
|
||||
// Calculate the board position
|
||||
// Vision coordinates are bottom-left origin, so we need to:
|
||||
// 1. Keep the same width
|
||||
// 2. Use the top portion of the detected area
|
||||
// 3. Maintain square aspect ratio
|
||||
return CGRect(
|
||||
x: bestMatch.minX,
|
||||
y: bestMatch.maxY - width, // Position at top of detected area
|
||||
width: width,
|
||||
height: width
|
||||
)
|
||||
// Perform the request
|
||||
let requestHandler = VNImageRequestHandler(ciImage: image, options: [:])
|
||||
do {
|
||||
try requestHandler.perform([request])
|
||||
} catch {
|
||||
print("Failed to perform rectangle detection: \(error)")
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
private func detectColorPatterns(in image: CIImage) -> [CGRect] {
|
||||
var matches: [CGRect] = []
|
||||
|
||||
for pattern in patterns {
|
||||
// Apply color adjustments to enhance pattern detection
|
||||
let colorControls = CIFilter(name: "CIColorControls")
|
||||
colorControls?.setValue(image, forKey: kCIInputImageKey)
|
||||
colorControls?.setValue(1.2, forKey: kCIInputSaturationKey) // Increase saturation
|
||||
colorControls?.setValue(0.2, forKey: kCIInputBrightnessKey) // Adjust brightness
|
||||
colorControls?.setValue(1.1, forKey: kCIInputContrastKey) // Increase contrast
|
||||
|
||||
guard let adjustedImage = colorControls?.outputImage else { continue }
|
||||
|
||||
// Detect rectangles
|
||||
var rectangles: [VNRectangleObservation] = []
|
||||
let request = VNDetectRectanglesRequest()
|
||||
request.minimumAspectRatio = 0.3 // Allow taller rectangles
|
||||
request.maximumAspectRatio = 0.5
|
||||
request.minimumSize = 0.4 // Look for larger areas
|
||||
request.maximumObservations = 1
|
||||
|
||||
let handler = VNImageRequestHandler(ciImage: adjustedImage)
|
||||
try? handler.perform([request])
|
||||
|
||||
if let results = request.results as? [VNRectangleObservation] {
|
||||
matches.append(contentsOf: results.map { $0.boundingBox })
|
||||
}
|
||||
// Process results
|
||||
guard let observations = request.results,
|
||||
!observations.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
|
||||
return matches
|
||||
}
|
||||
let bestObservation = observations[0]
|
||||
|
||||
private func detectGridPattern(in image: CIImage, colorMatches: [CGRect]) -> [CGRect] {
|
||||
var gridMatches: [CGRect] = []
|
||||
// Convert Vision coordinates to CoreImage coordinates
|
||||
let imageSize = image.extent.size
|
||||
let transform = CGAffineTransform(scaleX: imageSize.width, y: imageSize.height)
|
||||
|
||||
for rect in colorMatches {
|
||||
// Create edge detection filter
|
||||
let edgeFilter = CIFilter(name: "CIEdgeWork")
|
||||
edgeFilter?.setValue(image.cropped(to: rect), forKey: kCIInputImageKey)
|
||||
edgeFilter?.setValue(1.0, forKey: kCIInputRadiusKey)
|
||||
// Create normalized rect in CoreImage coordinate space
|
||||
let detectedRect = bestObservation.boundingBox.applying(transform)
|
||||
|
||||
guard let edgeImage = edgeFilter?.outputImage else { continue }
|
||||
|
||||
// Detect rectangles in edge image
|
||||
var rectangles: [VNRectangleObservation] = []
|
||||
let request = VNDetectRectanglesRequest()
|
||||
request.minimumAspectRatio = 0.3
|
||||
request.maximumAspectRatio = 0.5
|
||||
request.minimumSize = 0.4
|
||||
request.maximumObservations = 1
|
||||
|
||||
let handler = VNImageRequestHandler(ciImage: edgeImage)
|
||||
try? handler.perform([request])
|
||||
|
||||
if let results = request.results as? [VNRectangleObservation] {
|
||||
for observation in results {
|
||||
if validateGridDimensions(observation.boundingBox, squareSize: rect.width / 8) {
|
||||
gridMatches.append(observation.boundingBox)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Validate the detected rectangle
|
||||
guard validateDetectedRect(detectedRect, in: image) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
return gridMatches
|
||||
return detectedRect
|
||||
}
|
||||
|
||||
private func validateGridDimensions(_ rect: CGRect, squareSize: CGFloat) -> Bool {
|
||||
// Check if dimensions match 8x8 grid with some tolerance
|
||||
let expectedSize = squareSize * 8
|
||||
let tolerance: CGFloat = 0.3 // More flexible tolerance
|
||||
private func validateDetectedRect(_ rect: CGRect, in image: CIImage) -> Bool {
|
||||
let imageSize = image.extent.size
|
||||
|
||||
// Only validate width since height detection is unreliable
|
||||
let widthMatch = abs(rect.width - expectedSize) <= (expectedSize * tolerance)
|
||||
|
||||
return widthMatch
|
||||
}
|
||||
|
||||
private func validateCorners(in image: CIImage, candidates: [CGRect]) -> CGRect? {
|
||||
var bestMatch: (rect: CGRect, score: Float)?
|
||||
|
||||
for rect in candidates {
|
||||
var totalScore: Float = 0
|
||||
|
||||
// Extract corners
|
||||
let corners = [
|
||||
CGPoint(x: rect.minX, y: rect.minY),
|
||||
CGPoint(x: rect.maxX, y: rect.minY),
|
||||
CGPoint(x: rect.minX, y: rect.maxY),
|
||||
CGPoint(x: rect.maxX, y: rect.maxY)
|
||||
]
|
||||
|
||||
// Compare with pattern corners
|
||||
for pattern in patterns {
|
||||
var patternScore: Float = 0
|
||||
|
||||
for (corner, patternCorner) in zip(corners, pattern.cornerPattern) {
|
||||
let distance = hypot(
|
||||
corner.x - patternCorner.x,
|
||||
corner.y - patternCorner.y
|
||||
)
|
||||
patternScore += Float(1.0 / (1.0 + distance))
|
||||
}
|
||||
|
||||
totalScore = max(totalScore, patternScore / Float(corners.count))
|
||||
}
|
||||
|
||||
// Update best match if score is higher
|
||||
if totalScore > (bestMatch?.score ?? 0) {
|
||||
bestMatch = (rect, totalScore)
|
||||
}
|
||||
// Check if rectangle is within image bounds
|
||||
guard image.extent.contains(rect) else {
|
||||
return false
|
||||
}
|
||||
|
||||
return bestMatch?.rect
|
||||
// Validate aspect ratio (standard chess board is square)
|
||||
let aspectRatio = rect.width / rect.height
|
||||
guard aspectRatio >= 0.9 && aspectRatio <= 1.1 else {
|
||||
return false
|
||||
}
|
||||
|
||||
// Validate size relative to image
|
||||
let minDimension = min(imageSize.width, imageSize.height)
|
||||
let boardSize = max(rect.width, rect.height)
|
||||
guard boardSize >= minDimension * 0.4 else {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,21 +2,16 @@ import SwiftUI
|
|||
|
||||
struct ContentView: View {
|
||||
@StateObject private var viewModel = ScreenCaptureViewModel()
|
||||
@State private var showDebugOverlay = false
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 20) {
|
||||
// Screen Capture Controls
|
||||
VStack {
|
||||
// Capture controls
|
||||
HStack {
|
||||
Button(action: {
|
||||
Task {
|
||||
await viewModel.startCapture()
|
||||
}
|
||||
viewModel.startCapture()
|
||||
}) {
|
||||
Text("Start Capture")
|
||||
.padding()
|
||||
.background(viewModel.isCapturing ? Color.gray : Color.blue)
|
||||
.foregroundColor(.white)
|
||||
.cornerRadius(8)
|
||||
}
|
||||
.disabled(viewModel.isCapturing)
|
||||
|
||||
|
|
@ -24,112 +19,59 @@ struct ContentView: View {
|
|||
viewModel.stopCapture()
|
||||
}) {
|
||||
Text("Stop Capture")
|
||||
.padding()
|
||||
.background(!viewModel.isCapturing ? Color.gray : Color.red)
|
||||
.foregroundColor(.white)
|
||||
.cornerRadius(8)
|
||||
}
|
||||
.disabled(!viewModel.isCapturing)
|
||||
}
|
||||
.padding()
|
||||
|
||||
// Board Detection Visualization
|
||||
if let rect = viewModel.detectedBoardRect {
|
||||
GeometryReader { geometry in
|
||||
Rectangle()
|
||||
.stroke(Color.green, lineWidth: 2)
|
||||
.frame(
|
||||
width: rect.width * geometry.size.width,
|
||||
height: rect.height * geometry.size.height
|
||||
)
|
||||
.position(
|
||||
x: rect.midX * geometry.size.width,
|
||||
y: rect.midY * geometry.size.height
|
||||
)
|
||||
}
|
||||
.background(Color.black.opacity(0.1))
|
||||
}
|
||||
|
||||
// Error Display
|
||||
// Error display
|
||||
if let error = viewModel.captureError {
|
||||
VStack(spacing: 10) {
|
||||
Text(error.localizedDescription)
|
||||
.font(.headline)
|
||||
.foregroundColor(.red)
|
||||
.multilineTextAlignment(.center)
|
||||
|
||||
if case .chessWindowNotFound = error {
|
||||
VStack(alignment: .leading, spacing: 5) {
|
||||
Text("Requirements:")
|
||||
.font(.subheadline)
|
||||
.bold()
|
||||
Text("• Chess.com app or Firefox must be running")
|
||||
Text("• A chess game must be open")
|
||||
Text("• The game window must be visible")
|
||||
}
|
||||
.padding()
|
||||
.background(Color.red.opacity(0.1))
|
||||
.cornerRadius(8)
|
||||
}
|
||||
}
|
||||
.padding()
|
||||
.frame(maxWidth: .infinity)
|
||||
Text(error.localizedDescription)
|
||||
.foregroundColor(.red)
|
||||
.padding()
|
||||
}
|
||||
|
||||
// Image Display Section
|
||||
HStack(spacing: 20) {
|
||||
// Full Capture Display
|
||||
if let image = viewModel.capturedImage {
|
||||
VStack {
|
||||
Text("Full Capture")
|
||||
.font(.caption)
|
||||
// Image display
|
||||
HStack {
|
||||
VStack {
|
||||
Text("Full Capture")
|
||||
if let image = viewModel.capturedImage {
|
||||
Image(nsImage: image)
|
||||
.resizable()
|
||||
.aspectRatio(contentMode: .fit)
|
||||
.frame(height: 400) // Fixed height to show full board
|
||||
.frame(maxWidth: 400)
|
||||
} else {
|
||||
Text("No capture available")
|
||||
.foregroundColor(.gray)
|
||||
}
|
||||
.padding()
|
||||
.background(Color.black.opacity(0.05))
|
||||
.cornerRadius(8)
|
||||
}
|
||||
|
||||
// Cropped Board Display
|
||||
if let cropped = viewModel.croppedBoardImage {
|
||||
VStack {
|
||||
Text("Chessboard")
|
||||
.font(.caption)
|
||||
Image(nsImage: cropped)
|
||||
Divider()
|
||||
|
||||
VStack {
|
||||
Text("Chessboard Preview")
|
||||
if let image = viewModel.croppedBoardImage {
|
||||
Image(nsImage: image)
|
||||
.resizable()
|
||||
.aspectRatio(contentMode: .fit)
|
||||
.frame(height: 400) // Match the board height
|
||||
.frame(maxWidth: 400)
|
||||
.aspectRatio(contentMode: .fit)
|
||||
} else {
|
||||
Text("No board detected")
|
||||
.foregroundColor(.gray)
|
||||
}
|
||||
.padding()
|
||||
.background(Color.black.opacity(0.05))
|
||||
.cornerRadius(8)
|
||||
}
|
||||
}
|
||||
.padding()
|
||||
|
||||
// Vision Model Output Placeholder
|
||||
if viewModel.croppedBoardImage != nil {
|
||||
VStack(alignment: .leading) {
|
||||
Text("Board Analysis")
|
||||
.font(.headline)
|
||||
Text("FEN: (processing...)")
|
||||
.font(.system(.body, design: .monospaced))
|
||||
}
|
||||
.padding()
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.background(Color.blue.opacity(0.1))
|
||||
.cornerRadius(8)
|
||||
}
|
||||
|
||||
Spacer()
|
||||
}
|
||||
.padding()
|
||||
.frame(minWidth: 800, minHeight: 600) // Increased window size to fit larger board display
|
||||
.frame(minWidth: 800, minHeight: 600)
|
||||
}
|
||||
}
|
||||
|
||||
#Preview {
|
||||
ContentView()
|
||||
struct ContentView_Previews: PreviewProvider {
|
||||
static var previews: some View {
|
||||
ContentView()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,290 +1,64 @@
|
|||
import Foundation
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
import CoreMedia
|
||||
import ScreenCaptureKit
|
||||
import Vision
|
||||
import CoreImage
|
||||
import AVFoundation
|
||||
|
||||
class ScreenCapture: NSObject {
|
||||
private var captureSession: SCStream?
|
||||
private let queue = DispatchQueue(label: "com.chessprism.screencapture", qos: .userInteractive)
|
||||
private var lastDetectedCoordinates: [(text: String, boundingBox: CGRect)] = []
|
||||
private let boardDetector = BoardDetector()
|
||||
|
||||
// MARK: - Screen Capture Setup
|
||||
func startCapture() async throws {
|
||||
print("Starting screen capture...")
|
||||
// Get available screen content
|
||||
let availableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
|
||||
print("Available content: \(availableContent.displays.count) displays, \(availableContent.windows.count) windows")
|
||||
|
||||
// Find Chess.com app window
|
||||
let chessWindow = availableContent.windows.first { window in
|
||||
if let bundleId = window.owningApplication?.bundleIdentifier {
|
||||
return bundleId == "com.chess.iphone"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
guard let chessWindow = chessWindow else {
|
||||
throw ScreenCaptureError.chessWindowNotFound
|
||||
}
|
||||
|
||||
// Create stream configuration targeting chess window
|
||||
let filter = SCContentFilter(desktopIndependentWindow: chessWindow)
|
||||
let configuration = SCStreamConfiguration()
|
||||
configuration.width = 1920
|
||||
configuration.height = 1080
|
||||
configuration.pixelFormat = kCVPixelFormatType_32BGRA
|
||||
configuration.queueDepth = 5
|
||||
|
||||
// Create and start stream
|
||||
captureSession = SCStream(filter: filter, configuration: configuration, delegate: nil)
|
||||
|
||||
try await captureSession?.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
|
||||
try await captureSession?.startCapture()
|
||||
|
||||
print("Screen capture started successfully")
|
||||
NotificationCenter.default.post(
|
||||
name: .captureStateChanged,
|
||||
object: nil,
|
||||
userInfo: ["isCapturing": true]
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Frame Processing
|
||||
private func processFrame(sampleBuffer: CMSampleBuffer) {
|
||||
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
||||
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
|
||||
|
||||
// Convert CIImage to NSImage for display
|
||||
let context = CIContext()
|
||||
if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) {
|
||||
let size = NSSize(width: cgImage.width, height: cgImage.height)
|
||||
let nsImage = NSImage(cgImage: cgImage, size: size)
|
||||
|
||||
// Notify subscribers with captured image
|
||||
NotificationCenter.default.post(
|
||||
name: .capturedFrame,
|
||||
object: nil,
|
||||
userInfo: ["image": nsImage]
|
||||
)
|
||||
}
|
||||
|
||||
// Try pattern-based detection first
|
||||
if let boardRect = boardDetector.detectBoard(in: ciImage) {
|
||||
// Add padding to ensure full board capture
|
||||
let padding = boardRect.width * 0.25 // 25% padding
|
||||
let paddedRect = CGRect(
|
||||
x: boardRect.minX - padding,
|
||||
y: boardRect.minY - padding,
|
||||
width: boardRect.width + (padding * 2),
|
||||
height: boardRect.height + (padding * 2)
|
||||
)
|
||||
NotificationCenter.default.post(
|
||||
name: .boardDetected,
|
||||
object: nil,
|
||||
userInfo: ["rect": paddedRect]
|
||||
)
|
||||
class ScreenCapture: NSObject, SCStreamOutput {
|
||||
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
||||
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
|
||||
return
|
||||
}
|
||||
|
||||
// Fallback to coordinate-based detection
|
||||
let textRequest = VNRecognizeTextRequest { [weak self] request, error in
|
||||
guard let self = self else { return }
|
||||
self.handleTextDetection(request: request)
|
||||
}
|
||||
textRequest.recognitionLevel = .accurate
|
||||
textRequest.usesLanguageCorrection = false
|
||||
DispatchQueue.main.async {
|
||||
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
|
||||
let rep = NSCIImageRep(ciImage: ciImage)
|
||||
let nsImage = NSImage(size: rep.size)
|
||||
nsImage.addRepresentation(rep)
|
||||
|
||||
let rectangleRequest = VNDetectRectanglesRequest { [weak self] request, error in
|
||||
guard let self = self else { return }
|
||||
self.handleBoardDetection(request: request)
|
||||
}
|
||||
rectangleRequest.minimumAspectRatio = 0.8
|
||||
rectangleRequest.maximumAspectRatio = 1.2
|
||||
rectangleRequest.minimumSize = 0.2
|
||||
rectangleRequest.maximumObservations = 3
|
||||
|
||||
let handler = VNImageRequestHandler(ciImage: ciImage)
|
||||
try? handler.perform([textRequest, rectangleRequest])
|
||||
}
|
||||
|
||||
private func handleTextDetection(request: VNRequest) {
|
||||
guard let results = request.results as? [VNRecognizedTextObservation] else { return }
|
||||
|
||||
// Filter for chess coordinates (a-h, 1-8)
|
||||
var coordinates: [(text: String, boundingBox: CGRect)] = []
|
||||
for observation in results {
|
||||
if let candidate = observation.topCandidates(1).first {
|
||||
let text = candidate.string
|
||||
if (text.count == 1 && ("a"..."h").contains(text.lowercased())) ||
|
||||
(text.count == 1 && ("1"..."8").contains(text)) {
|
||||
coordinates.append((text: text, boundingBox: observation.boundingBox))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store coordinates and notify observers
|
||||
if !coordinates.isEmpty {
|
||||
lastDetectedCoordinates = coordinates
|
||||
NotificationCenter.default.post(
|
||||
name: .boardCoordinatesDetected,
|
||||
object: nil,
|
||||
userInfo: ["coordinates": coordinates]
|
||||
)
|
||||
self.lastCapturedImage = nsImage
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Board Detection Handling
|
||||
private func handleBoardDetection(request: VNRequest) {
|
||||
guard let results = request.results as? [VNRectangleObservation],
|
||||
let boardRect = results.first else { return }
|
||||
private var activeStream: SCStream?
|
||||
private var lastCapturedImage: NSImage?
|
||||
|
||||
// Convert rectangle to screen coordinates
|
||||
var transformedRect = transformRectangle(boardRect)
|
||||
func startCapture() async throws {
|
||||
guard #available(macOS 12.3, *) else { return }
|
||||
|
||||
// Use the stored coordinates
|
||||
let coordinates = lastDetectedCoordinates
|
||||
let content = try await SCShareableContent.current
|
||||
|
||||
// If we have coordinates, use them to refine the rectangle
|
||||
if !coordinates.isEmpty {
|
||||
// Separate horizontal and vertical coordinates
|
||||
let horizontalCoords = coordinates.filter { ("a"..."h").contains($0.text.lowercased()) }
|
||||
let verticalCoords = coordinates.filter { ("1"..."8").contains($0.text) }
|
||||
|
||||
// Get min/max for horizontal coordinates (a-h)
|
||||
let xCoords = horizontalCoords.map { $0.boundingBox.origin.x }
|
||||
let minX = xCoords.min() ?? transformedRect.minX
|
||||
let maxX = xCoords.max() ?? transformedRect.maxX
|
||||
|
||||
// Get min/max for vertical coordinates (1-8)
|
||||
let yCoords = verticalCoords.map { $0.boundingBox.origin.y }
|
||||
let minY = yCoords.min() ?? transformedRect.minY
|
||||
let maxY = yCoords.max() ?? transformedRect.maxY
|
||||
|
||||
// Calculate board boundaries directly from coordinates
|
||||
let leftMost = horizontalCoords.min { $0.boundingBox.origin.x < $1.boundingBox.origin.x }?.boundingBox.origin.x ?? minX
|
||||
let rightMost = horizontalCoords.max { $0.boundingBox.origin.x < $1.boundingBox.origin.x }?.boundingBox.origin.x ?? maxX
|
||||
let bottomMost = verticalCoords.min { $0.boundingBox.origin.y < $1.boundingBox.origin.y }?.boundingBox.origin.y ?? minY
|
||||
let topMost = verticalCoords.max { $0.boundingBox.origin.y < $1.boundingBox.origin.y }?.boundingBox.origin.y ?? maxY
|
||||
|
||||
// Calculate board dimensions directly from coordinate positions
|
||||
let boardWidth = rightMost - leftMost
|
||||
let boardHeight = topMost - bottomMost
|
||||
|
||||
// Find coordinate pairs that are likely on opposite sides of the board
|
||||
let horizontalPairs = horizontalCoords.flatMap { h1 in
|
||||
horizontalCoords.map { h2 in
|
||||
(h1, h2, abs(h1.boundingBox.origin.x - h2.boundingBox.origin.x))
|
||||
}
|
||||
}.sorted { $0.2 > $1.2 }
|
||||
|
||||
let verticalPairs = verticalCoords.flatMap { v1 in
|
||||
verticalCoords.map { v2 in
|
||||
(v1, v2, abs(v1.boundingBox.origin.y - v2.boundingBox.origin.y))
|
||||
}
|
||||
}.sorted { $0.2 > $1.2 }
|
||||
|
||||
// Use the most distant pairs to define board boundaries
|
||||
if let widestHorizontal = horizontalPairs.first,
|
||||
let tallestVertical = verticalPairs.first {
|
||||
|
||||
// Calculate board dimensions using the most distant coordinates
|
||||
let boardWidth = widestHorizontal.2
|
||||
let boardHeight = tallestVertical.2
|
||||
|
||||
// Calculate the center of the board
|
||||
let centerX = (widestHorizontal.0.boundingBox.origin.x + widestHorizontal.1.boundingBox.origin.x) / 2
|
||||
let centerY = (tallestVertical.0.boundingBox.origin.y + tallestVertical.1.boundingBox.origin.y) / 2
|
||||
|
||||
// Add padding proportional to the coordinate spacing
|
||||
let padding = max(boardWidth, boardHeight) * 0.25 // 25% padding
|
||||
|
||||
// Set rectangle to centered board with padding
|
||||
transformedRect = CGRect(
|
||||
x: centerX - (boardWidth / 2) - padding,
|
||||
y: centerY - (boardHeight / 2) - padding,
|
||||
width: boardWidth + (padding * 2),
|
||||
height: boardHeight + (padding * 2)
|
||||
)
|
||||
}
|
||||
|
||||
// Ensure we capture full board with reasonable bounds before aspect ratio adjustment
|
||||
transformedRect.origin.x = max(-0.1, transformedRect.origin.x) // Allow 10% outside bounds
|
||||
transformedRect.origin.y = max(-0.1, transformedRect.origin.y)
|
||||
transformedRect.size.width = min(1.2 - transformedRect.origin.x, transformedRect.size.width) // Allow 20% overflow
|
||||
transformedRect.size.height = min(1.2 - transformedRect.origin.y, transformedRect.size.height)
|
||||
|
||||
// Apply single aspect ratio adjustment with flexible tolerance
|
||||
let targetAspectRatio: CGFloat = 1.0
|
||||
let currentAspectRatio = transformedRect.size.width / transformedRect.size.height
|
||||
if abs(currentAspectRatio - targetAspectRatio) > 0.2 {
|
||||
if currentAspectRatio > targetAspectRatio {
|
||||
let newHeight = transformedRect.size.width
|
||||
let heightDiff = newHeight - transformedRect.size.height
|
||||
transformedRect.origin.y -= heightDiff / 2.0
|
||||
transformedRect.size.height = newHeight
|
||||
} else {
|
||||
let newWidth = transformedRect.size.height
|
||||
let widthDiff = newWidth - transformedRect.size.width
|
||||
transformedRect.origin.x -= widthDiff / 2.0
|
||||
transformedRect.size.width = newWidth
|
||||
}
|
||||
}
|
||||
// Get the window from the content
|
||||
guard let window = content.windows.first(where: { window in
|
||||
let bundleID = window.owningApplication?.bundleIdentifier ?? ""
|
||||
let isChessApp = bundleID == "com.chess.iphone"
|
||||
let hasValidSize = window.frame.width > 100 && window.frame.height > 100
|
||||
return isChessApp && window.isOnScreen && hasValidSize
|
||||
}) else {
|
||||
throw NSError(domain: "ChessPrism", code: 1, userInfo: [NSLocalizedDescriptionKey: "Chess window not found"])
|
||||
}
|
||||
|
||||
// Notify subscribers
|
||||
NotificationCenter.default.post(
|
||||
name: .boardDetected,
|
||||
object: nil,
|
||||
userInfo: ["rect": transformedRect]
|
||||
)
|
||||
let filter = SCContentFilter(desktopIndependentWindow: window)
|
||||
let config = SCStreamConfiguration()
|
||||
config.width = Int(window.frame.width)
|
||||
config.height = Int(window.frame.height)
|
||||
|
||||
let stream = SCStream(filter: filter, configuration: config, delegate: nil)
|
||||
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: DispatchQueue.global(qos: .userInitiated))
|
||||
|
||||
// Store the stream before starting it
|
||||
activeStream = stream
|
||||
try await stream.startCapture()
|
||||
}
|
||||
|
||||
// MARK: - Coordinate Transformation
|
||||
private func transformRectangle(_ rect: VNRectangleObservation) -> CGRect {
|
||||
// Vision coordinates are in normalized coordinates with origin at bottom-left
|
||||
// We need to convert to top-left origin coordinates
|
||||
return CGRect(
|
||||
x: rect.boundingBox.origin.x,
|
||||
y: 1 - rect.boundingBox.origin.y - rect.boundingBox.height,
|
||||
width: rect.boundingBox.width,
|
||||
height: rect.boundingBox.height
|
||||
)
|
||||
func stopCapture() async throws {
|
||||
guard let stream = activeStream else { return }
|
||||
try await stream.stopCapture()
|
||||
activeStream = nil
|
||||
}
|
||||
|
||||
// MARK: - Capture Control
|
||||
func stopCapture() {
|
||||
print("Stopping screen capture...")
|
||||
captureSession?.stopCapture()
|
||||
captureSession = nil
|
||||
NotificationCenter.default.post(
|
||||
name: .captureStateChanged,
|
||||
object: nil,
|
||||
userInfo: ["isCapturing": false]
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Error Handling
|
||||
enum ScreenCaptureError: Error {
|
||||
case chessWindowNotFound
|
||||
case streamCreationFailed
|
||||
case permissionDenied
|
||||
case captureAlreadyRunning
|
||||
func getCurrentImage() -> NSImage? {
|
||||
return lastCapturedImage
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - SCStreamOutput
|
||||
extension ScreenCapture: SCStreamOutput {
|
||||
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
||||
guard type == .screen else { return }
|
||||
processFrame(sampleBuffer: sampleBuffer)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Notifications
|
||||
extension Notification.Name {
|
||||
static let boardDetected = Notification.Name("com.chessprism.boardDetected")
|
||||
static let captureStateChanged = Notification.Name("com.chessprism.captureStateChanged")
|
||||
static let capturedFrame = Notification.Name("com.chessprism.capturedFrame")
|
||||
static let boardCoordinatesDetected = Notification.Name("com.chessprism.boardCoordinatesDetected")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,157 +1,120 @@
|
|||
import Foundation
|
||||
import SwiftUI
|
||||
import AppKit
|
||||
import CoreImage
|
||||
import Combine
|
||||
|
||||
@MainActor
|
||||
class ScreenCaptureViewModel: ObservableObject {
|
||||
// MARK: - Published Properties
|
||||
@Published var capturedImage: NSImage?
|
||||
@Published var croppedBoardImage: NSImage?
|
||||
@Published var captureError: CaptureError?
|
||||
@Published var isCapturing = false
|
||||
@Published var captureError: ScreenCaptureError? = nil
|
||||
@Published var detectedBoardRect: CGRect? = nil
|
||||
@Published var capturedImage: NSImage? = nil
|
||||
@Published var croppedBoardImage: NSImage? = nil
|
||||
|
||||
// MARK: - Dependencies
|
||||
private let screenCapture = ScreenCapture()
|
||||
private var cancellables = Set<AnyCancellable>()
|
||||
private let boardDetector = BoardDetector()
|
||||
private var captureTask: Task<Void, Never>?
|
||||
// Share CIContext to avoid creating too many Metal command queues
|
||||
private static let shared = CIContext()
|
||||
private var context: CIContext { ScreenCaptureViewModel.shared }
|
||||
|
||||
// MARK: - Initialization
|
||||
init() {
|
||||
setupObservers()
|
||||
private var detectedBoardRect: CGRect?
|
||||
|
||||
enum CaptureError: LocalizedError {
|
||||
case chessWindowNotFound
|
||||
case boardDetectionFailed
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .chessWindowNotFound:
|
||||
return "Chess window not found"
|
||||
case .boardDetectionFailed:
|
||||
return "Failed to detect chess board"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Screen Capture Management
|
||||
func startCapture() async {
|
||||
guard await requestScreenCapturePermission() else {
|
||||
captureError = .permissionDenied
|
||||
return
|
||||
}
|
||||
func startCapture() {
|
||||
isCapturing = true
|
||||
captureError = nil
|
||||
|
||||
do {
|
||||
try await screenCapture.startCapture()
|
||||
isCapturing = true
|
||||
} catch let error as ScreenCapture.ScreenCaptureError {
|
||||
captureError = ScreenCaptureError(from: error)
|
||||
} catch {
|
||||
captureError = .unknown(error)
|
||||
captureTask = Task {
|
||||
do {
|
||||
// Start continuous capture
|
||||
try await screenCapture.startCapture()
|
||||
|
||||
captureLoop: while !Task.isCancelled {
|
||||
do {
|
||||
if let image = screenCapture.getCurrentImage() {
|
||||
try await processImage(image)
|
||||
}
|
||||
try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
|
||||
} catch is CancellationError {
|
||||
break captureLoop
|
||||
} catch {
|
||||
// Just update the error state but continue capturing
|
||||
if !Task.isCancelled {
|
||||
handleCaptureError(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
try? await screenCapture.stopCapture()
|
||||
if isCapturing {
|
||||
isCapturing = false
|
||||
}
|
||||
} catch {
|
||||
handleCaptureError(error)
|
||||
isCapturing = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stopCapture() {
|
||||
screenCapture.stopCapture()
|
||||
captureTask?.cancel()
|
||||
captureTask = nil
|
||||
isCapturing = false
|
||||
captureError = nil
|
||||
}
|
||||
|
||||
// MARK: - Permission Handling
|
||||
private func requestScreenCapturePermission() async -> Bool {
|
||||
return await withCheckedContinuation { continuation in
|
||||
let status = CGPreflightScreenCaptureAccess()
|
||||
if status {
|
||||
continuation.resume(returning: true)
|
||||
} else {
|
||||
CGRequestScreenCaptureAccess()
|
||||
continuation.resume(returning: false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Notification Handling
|
||||
private func setupObservers() {
|
||||
// Observe board detection
|
||||
NotificationCenter.default
|
||||
.publisher(for: .boardDetected)
|
||||
.receive(on: DispatchQueue.main)
|
||||
.sink(receiveValue: { [weak self] notification in
|
||||
guard let rect = notification.userInfo?["rect"] as? CGRect else { return }
|
||||
self?.detectedBoardRect = rect
|
||||
})
|
||||
.store(in: &cancellables)
|
||||
|
||||
// Observe capture state changes
|
||||
NotificationCenter.default
|
||||
.publisher(for: .captureStateChanged)
|
||||
.receive(on: DispatchQueue.main)
|
||||
.sink(receiveValue: { [weak self] notification in
|
||||
guard let isCapturing = notification.userInfo?["isCapturing"] as? Bool else { return }
|
||||
self?.isCapturing = isCapturing
|
||||
})
|
||||
.store(in: &cancellables)
|
||||
|
||||
// Observe captured frames
|
||||
NotificationCenter.default
|
||||
.publisher(for: .capturedFrame)
|
||||
.receive(on: DispatchQueue.main)
|
||||
.sink(receiveValue: { [weak self] notification in
|
||||
guard let image = notification.userInfo?["image"] as? NSImage,
|
||||
let self = self else { return }
|
||||
|
||||
self.capturedImage = image
|
||||
|
||||
// Crop to detected board area if available
|
||||
if let rect = self.detectedBoardRect {
|
||||
let cropped = self.cropImage(image, to: rect)
|
||||
self.croppedBoardImage = cropped
|
||||
}
|
||||
})
|
||||
.store(in: &cancellables)
|
||||
}
|
||||
|
||||
// MARK: - Error Handling
|
||||
enum ScreenCaptureError: Error, LocalizedError {
|
||||
case permissionDenied
|
||||
case chessWindowNotFound
|
||||
case streamCreationFailed
|
||||
case captureAlreadyRunning
|
||||
case unknown(Error)
|
||||
|
||||
init(from error: ScreenCapture.ScreenCaptureError) {
|
||||
switch error {
|
||||
case .chessWindowNotFound:
|
||||
self = .chessWindowNotFound
|
||||
case .streamCreationFailed:
|
||||
self = .streamCreationFailed
|
||||
case .permissionDenied:
|
||||
self = .permissionDenied
|
||||
case .captureAlreadyRunning:
|
||||
self = .captureAlreadyRunning
|
||||
}
|
||||
}
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .permissionDenied:
|
||||
return "Screen recording permission is required"
|
||||
case .chessWindowNotFound:
|
||||
return "Unable to find Chess window (make sure Chess.com app or Firefox is open)"
|
||||
case .streamCreationFailed:
|
||||
return "Failed to create screen capture stream"
|
||||
case .captureAlreadyRunning:
|
||||
return "Capture is already running"
|
||||
case .unknown(let error):
|
||||
return "An unexpected error occurred: \(error.localizedDescription)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func cropImage(_ image: NSImage, to rect: CGRect) -> NSImage? {
|
||||
private func processImage(_ image: NSImage) async throws {
|
||||
// Convert to CIImage for processing
|
||||
guard let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else {
|
||||
return nil
|
||||
throw CaptureError.boardDetectionFailed
|
||||
}
|
||||
|
||||
// Convert normalized rect to pixel coordinates
|
||||
let width = CGFloat(cgImage.width)
|
||||
let height = CGFloat(cgImage.height)
|
||||
let cropRect = CGRect(
|
||||
x: rect.origin.x * width,
|
||||
y: (1 - rect.origin.y - rect.height) * height, // Flip Y coordinate since NSImage uses bottom-left origin
|
||||
width: rect.width * width,
|
||||
height: rect.height * height
|
||||
)
|
||||
let ciImage = CIImage(cgImage: cgImage)
|
||||
|
||||
guard let croppedCGImage = cgImage.cropping(to: cropRect) else {
|
||||
return nil
|
||||
// Detect board
|
||||
guard let boardRect = boardDetector.detectBoard(in: ciImage) else {
|
||||
throw CaptureError.boardDetectionFailed
|
||||
}
|
||||
|
||||
return NSImage(cgImage: croppedCGImage, size: cropRect.size)
|
||||
// Update detected rectangle
|
||||
self.detectedBoardRect = boardRect
|
||||
|
||||
// Crop board image
|
||||
let croppedImage = ciImage.cropped(to: boardRect)
|
||||
updateImages(ciImage: ciImage, croppedImage: croppedImage)
|
||||
}
|
||||
|
||||
private func updateImages(ciImage: CIImage, croppedImage: CIImage) {
|
||||
if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) {
|
||||
let capturedNSImage = NSImage(cgImage: cgImage, size: .zero)
|
||||
self.capturedImage = capturedNSImage
|
||||
}
|
||||
|
||||
if let cgCroppedImage = context.createCGImage(croppedImage, from: croppedImage.extent) {
|
||||
let croppedNSImage = NSImage(cgImage: cgCroppedImage, size: .zero)
|
||||
self.croppedBoardImage = croppedNSImage
|
||||
}
|
||||
}
|
||||
|
||||
private func handleCaptureError(_ error: Error) {
|
||||
if let captureError = error as? CaptureError {
|
||||
self.captureError = captureError
|
||||
} else {
|
||||
self.captureError = .boardDetectionFailed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
571
cline_docs/Info.txt
Normal file
571
cline_docs/Info.txt
Normal file
|
|
@ -0,0 +1,571 @@
|
|||
|
||||
Command queue creation failed. Worst processes (
|
||||
{
|
||||
Count = 2017;
|
||||
Process = "pid 25345, ChessPrism";
|
||||
},
|
||||
{
|
||||
Count = 4;
|
||||
Process = "pid 1227, firefox";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 579, WindowServer";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 1926, Creative Cloud U";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1754, Ollama Helper (G";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1928, LogiTune Helper ";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 11425, Xcode";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1062, mediaanalysisd";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1101, NotificationCent";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 582, loginwindow";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1242, Finder";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1100, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1237, Notes";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1222, Terminal";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1867, Docker Desktop H";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1970, Logi AI Prompt B";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1040, ControlCenter";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 905, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 7637, ControlCenterHel";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 15435, Code Helper (GPU";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1208, avconferenced";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1228, VTDecoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1248, Spotlight";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1186, replayd";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1459, naturallanguaged";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 577, com.apple.cmio.r";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 2216, VTEncoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 7667, QuickLookUIServi";
|
||||
}
|
||||
)
|
||||
Command queue creation failed. Worst processes (
|
||||
{
|
||||
Count = 2017;
|
||||
Process = "pid 25345, ChessPrism";
|
||||
},
|
||||
{
|
||||
Count = 4;
|
||||
Process = "pid 1227, firefox";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 579, WindowServer";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 1926, Creative Cloud U";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1754, Ollama Helper (G";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1928, LogiTune Helper ";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 11425, Xcode";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1062, mediaanalysisd";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1101, NotificationCent";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 582, loginwindow";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1242, Finder";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1100, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1237, Notes";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1222, Terminal";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1867, Docker Desktop H";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1970, Logi AI Prompt B";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1040, ControlCenter";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 905, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 7637, ControlCenterHel";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 15435, Code Helper (GPU";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1208, avconferenced";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1228, VTDecoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1248, Spotlight";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1186, replayd";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1459, naturallanguaged";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 577, com.apple.cmio.r";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 2216, VTEncoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 7667, QuickLookUIServi";
|
||||
}
|
||||
)
|
||||
Command queue creation failed. Worst processes (
|
||||
{
|
||||
Count = 2017;
|
||||
Process = "pid 25345, ChessPrism";
|
||||
},
|
||||
{
|
||||
Count = 4;
|
||||
Process = "pid 1227, firefox";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 579, WindowServer";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 1926, Creative Cloud U";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1754, Ollama Helper (G";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1928, LogiTune Helper ";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 11425, Xcode";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1062, mediaanalysisd";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1101, NotificationCent";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 582, loginwindow";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1242, Finder";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1100, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1237, Notes";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1222, Terminal";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1867, Docker Desktop H";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1970, Logi AI Prompt B";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1040, ControlCenter";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 905, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 7637, ControlCenterHel";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 15435, Code Helper (GPU";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1208, avconferenced";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1228, VTDecoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1248, Spotlight";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1186, replayd";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1459, naturallanguaged";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 577, com.apple.cmio.r";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 2216, VTEncoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 7667, QuickLookUIServi";
|
||||
}
|
||||
)
|
||||
Command queue creation failed. Worst processes (
|
||||
{
|
||||
Count = 2017;
|
||||
Process = "pid 25345, ChessPrism";
|
||||
},
|
||||
{
|
||||
Count = 4;
|
||||
Process = "pid 1227, firefox";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 579, WindowServer";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 1926, Creative Cloud U";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1754, Ollama Helper (G";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1928, LogiTune Helper ";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 11425, Xcode";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1062, mediaanalysisd";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1101, NotificationCent";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 582, loginwindow";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1242, Finder";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1100, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1237, Notes";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1222, Terminal";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1867, Docker Desktop H";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1970, Logi AI Prompt B";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1040, ControlCenter";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 905, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 7637, ControlCenterHel";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 15435, Code Helper (GPU";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1208, avconferenced";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1228, VTDecoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1248, Spotlight";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1186, replayd";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1459, naturallanguaged";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 577, com.apple.cmio.r";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 2216, VTEncoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 7667, QuickLookUIServi";
|
||||
}
|
||||
)
|
||||
Command queue creation failed. Worst processes (
|
||||
{
|
||||
Count = 2017;
|
||||
Process = "pid 25345, ChessPrism";
|
||||
},
|
||||
{
|
||||
Count = 4;
|
||||
Process = "pid 1227, firefox";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 579, WindowServer";
|
||||
},
|
||||
{
|
||||
Count = 3;
|
||||
Process = "pid 1926, Creative Cloud U";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1754, Ollama Helper (G";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 1928, LogiTune Helper ";
|
||||
},
|
||||
{
|
||||
Count = 2;
|
||||
Process = "pid 11425, Xcode";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1062, mediaanalysisd";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1101, NotificationCent";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 582, loginwindow";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1242, Finder";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1100, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1237, Notes";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1222, Terminal";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1867, Docker Desktop H";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1970, Logi AI Prompt B";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 1040, ControlCenter";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 905, iconservicesagen";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 7637, ControlCenterHel";
|
||||
},
|
||||
{
|
||||
Count = 1;
|
||||
Process = "pid 15435, Code Helper (GPU";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1208, avconferenced";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1228, VTDecoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1248, Spotlight";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1186, replayd";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 1459, naturallanguaged";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 577, com.apple.cmio.r";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 2216, VTEncoderXPCServ";
|
||||
},
|
||||
{
|
||||
Count = 0;
|
||||
Process = "pid 7667, QuickLookUIServi";
|
||||
}
|
||||
)
|
||||
|
|
@ -1,50 +1,74 @@
|
|||
# Active Context
|
||||
|
||||
## Current Status
|
||||
- Screen capture module implemented
|
||||
- Pattern recognition-based board detection implemented
|
||||
- Coordinate-based detection retained as fallback
|
||||
- SwiftUI interface for capture controls added
|
||||
- Proper resource cleanup implemented
|
||||
- Error handling system in place
|
||||
- Screen capture module successfully implemented
|
||||
- Window detection and capture working correctly
|
||||
- SwiftUI interface with capture controls functioning
|
||||
- Error handling system properly managing states
|
||||
- Resource cleanup implemented
|
||||
- Metal resource management optimized
|
||||
|
||||
## Recent Changes
|
||||
- Refined pattern recognition approach for chess board detection:
|
||||
1. Modified rectangle detection parameters:
|
||||
- Using 0.3-0.5 aspect ratio for taller rectangles
|
||||
- Increased minimum size to 0.4
|
||||
- Single observation for precision
|
||||
2. Improved board extraction:
|
||||
- Using detected width as reference
|
||||
- Positioning square board at top of detected area
|
||||
- Using offsetBy for vertical positioning
|
||||
3. Simplified coordinate handling:
|
||||
- Direct rectangle detection with Vision framework
|
||||
- Proper coordinate system transformations
|
||||
- Maintained pattern matching for accuracy
|
||||
1. Resource Management Optimization:
|
||||
- Implemented shared CIContext pattern:
|
||||
* Prevents command queue exhaustion
|
||||
* Reduces Metal resource usage
|
||||
* Enables long-running captures
|
||||
- Proper cleanup on task completion
|
||||
- Efficient resource utilization
|
||||
|
||||
## Current Challenges
|
||||
- Board detection still not capturing full height
|
||||
- Need to investigate if issue is with:
|
||||
1. Detection parameters
|
||||
2. Coordinate transformations
|
||||
3. View rendering constraints
|
||||
4. Window/display configuration
|
||||
2. Improved Window Detection:
|
||||
- Using SCShareableContent for window access
|
||||
- Precise window identification:
|
||||
* Exact bundle ID matching (com.chess.iphone)
|
||||
* Window visibility verification (isOnScreen)
|
||||
* Size validation (width > 100 && height > 100)
|
||||
|
||||
3. Enhanced Capture System:
|
||||
- Continuous capture implementation:
|
||||
* Single persistent capture stream
|
||||
* Smooth frame processing (no flickering)
|
||||
* Efficient resource usage
|
||||
- Proper frame dimensions from window
|
||||
- Clean task cancellation handling
|
||||
- Main thread safety for UI updates
|
||||
|
||||
4. UI Simplification:
|
||||
- Removed debug overlay functionality
|
||||
- Cleaner, focused interface
|
||||
- Essential controls only:
|
||||
* Start/Stop capture
|
||||
* Full capture view
|
||||
* Board preview
|
||||
|
||||
5. Error Handling:
|
||||
- Improved error resilience:
|
||||
* Continues capturing even if board detection fails
|
||||
* Only stops on critical errors (e.g., window not found)
|
||||
* Shows error state without interrupting capture
|
||||
- Clear error states
|
||||
- Proper async/await usage
|
||||
- Task cancellation management
|
||||
- Thread-safe state updates
|
||||
|
||||
## Current Focus
|
||||
1. Board Detection:
|
||||
- Implement chess board recognition
|
||||
- Handle different board themes
|
||||
- Process captured frames efficiently
|
||||
|
||||
## Next Steps
|
||||
1. Further refine board detection:
|
||||
- Test different aspect ratios
|
||||
- Validate coordinate transformations
|
||||
- Review view constraints
|
||||
2. Add board position analysis
|
||||
3. Implement move detection
|
||||
4. Create visual overlay system
|
||||
1. Implement board detection:
|
||||
- Pattern recognition for chess pieces
|
||||
- Board coordinate mapping
|
||||
- Position validation
|
||||
2. Add position analysis
|
||||
3. Create move detection system
|
||||
4. Implement visual overlay
|
||||
5. Integrate Stockfish engine
|
||||
|
||||
## Known Issues
|
||||
- Board detection showing only bottom portion
|
||||
- Need to validate pattern detection accuracy
|
||||
- Need to handle multiple display configurations
|
||||
- Requires testing with different screen resolutions
|
||||
- Need to add proper error recovery
|
||||
- Requires additional permission handling for production
|
||||
- Board detection not yet implemented
|
||||
- Need to handle different chess.com themes
|
||||
- Need to implement piece recognition
|
||||
- Position analysis pending implementation
|
||||
|
|
|
|||
191
cline_docs/error
191
cline_docs/error
|
|
@ -1,53 +1,138 @@
|
|||
Starting screen capture...
|
||||
Available content: 1 displays, 32 windows
|
||||
AddInstanceForFactory: No factory registered for id <CFUUID 0x6000011a9580> F8BB1C28-BAE8-11D6-9C31-00039315CD46
|
||||
Screen capture started successfully
|
||||
*** Terminating app due to uncaught exception 'NSUnknownKeyException', reason: '[<NSNotificationCenter 0x6000013bc060> valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.'
|
||||
*** First throw call stack:
|
||||
(
|
||||
0 CoreFoundation 0x0000000189d0ee80 __exceptionPreprocess + 176
|
||||
1 libobjc.A.dylib 0x00000001897f6cd8 objc_exception_throw + 88
|
||||
2 CoreFoundation 0x0000000189d33534 _CFBundleGetValueForInfoKey + 0
|
||||
3 Foundation 0x000000018b6dbd84 -[NSObject(NSKeyValueCoding) valueForUndefinedKey:] + 196
|
||||
4 Foundation 0x000000018ae402fc -[NSObject(NSKeyValueCoding) valueForKey:] + 280
|
||||
5 ChessPrism.debug.dylib 0x0000000100b8f834 $s10ChessPrism13ScreenCaptureC20handleBoardDetection33_6963F6DFCAB5A0A312B3D7446D299968LL7requestySo9VNRequestC_tF + 740
|
||||
6 ChessPrism.debug.dylib 0x0000000100b8e644 $s10ChessPrism13ScreenCaptureC12processFrame33_6963F6DFCAB5A0A312B3D7446D299968LL12sampleBufferySo08CMSampleO3Refa_tFySo9VNRequestC_s5Error_pSgtcfU0_ + 200
|
||||
7 ChessPrism.debug.dylib 0x0000000100b93350 $sSo9VNRequestCs5Error_pSgIeggg_ABSo7NSErrorCSgIeyByy_TR + 140
|
||||
8 Vision 0x000000019f40aab4 -[VNRequest performInContext:error:] + 1244
|
||||
9 Vision 0x000000019f40af08 __73-[VNRequest performInContextAsync:asyncDispatchQueue:asyncDispatchGroup:]_block_invoke + 128
|
||||
10 libdispatch.dylib 0x00000001001d2ac8 _dispatch_block_async_invoke2 + 148
|
||||
11 libdispatch.dylib 0x00000001001be824 _dispatch_client_callout + 20
|
||||
12 libdispatch.dylib 0x00000001001c2350 _dispatch_continuation_pop + 1408
|
||||
13 libdispatch.dylib 0x00000001001c1028 _dispatch_async_redirect_invoke + 616
|
||||
14 libdispatch.dylib 0x00000001001d6b70 _dispatch_root_queue_drain + 404
|
||||
15 libdispatch.dylib 0x00000001001d777c _dispatch_worker_thread2 + 188
|
||||
16 libsystem_pthread.dylib 0x0000000100b090c4 _pthread_wqthread + 228
|
||||
17 libsystem_pthread.dylib 0x0000000100b10cf0 start_wqthread + 8
|
||||
)
|
||||
An uncaught exception was raised
|
||||
[<NSNotificationCenter 0x6000013bc060> valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.
|
||||
(
|
||||
0 CoreFoundation 0x0000000189d0ee80 __exceptionPreprocess + 176
|
||||
1 libobjc.A.dylib 0x00000001897f6cd8 objc_exception_throw + 88
|
||||
2 CoreFoundation 0x0000000189d33534 _CFBundleGetValueForInfoKey + 0
|
||||
3 Foundation 0x000000018b6dbd84 -[NSObject(NSKeyValueCoding) valueForUndefinedKey:] + 196
|
||||
4 Foundation 0x000000018ae402fc -[NSObject(NSKeyValueCoding) valueForKey:] + 280
|
||||
5 ChessPrism.debug.dylib 0x0000000100b8f834 $s10ChessPrism13ScreenCaptureC20handleBoardDetection33_6963F6DFCAB5A0A312B3D7446D299968LL7requestySo9VNRequestC_tF + 740
|
||||
6 ChessPrism.debug.dylib 0x0000000100b8e644 $s10ChessPrism13ScreenCaptureC12processFrame33_6963F6DFCAB5A0A312B3D7446D299968LL12sampleBufferySo08CMSampleO3Refa_tFySo9VNRequestC_s5Error_pSgtcfU0_ + 200
|
||||
7 ChessPrism.debug.dylib 0x0000000100b93350 $sSo9VNRequestCs5Error_pSgIeggg_ABSo7NSErrorCSgIeyByy_TR + 140
|
||||
8 Vision 0x000000019f40aab4 -[VNRequest performInContext:error:] + 1244
|
||||
9 Vision 0x000000019f40af08 __73-[VNRequest performInContextAsync:asyncDispatchQueue:asyncDispatchGroup:]_block_invoke + 128
|
||||
10 libdispatch.dylib 0x00000001001d2ac8 _dispatch_block_async_invoke2 + 148
|
||||
11 libdispatch.dylib 0x00000001001be824 _dispatch_client_callout + 20
|
||||
12 libdispatch.dylib 0x00000001001c2350 _dispatch_continuation_pop + 1408
|
||||
13 libdispatch.dylib 0x00000001001c1028 _dispatch_async_redirect_invoke + 616
|
||||
14 libdispatch.dylib 0x00000001001d6b70 _dispatch_root_queue_drain + 404
|
||||
15 libdispatch.dylib 0x00000001001d777c _dispatch_worker_thread2 + 188
|
||||
16 libsystem_pthread.dylib 0x0000000100b090c4 _pthread_wqthread + 228
|
||||
17 libsystem_pthread.dylib 0x0000000100b10cf0 start_wqthread + 8
|
||||
)
|
||||
FAULT: NSUnknownKeyException: [<NSNotificationCenter 0x6000013bc060> valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.; {
|
||||
NSTargetObjectUserInfoKey = "<CFNotificationCenter 0x6000011a4900 [0x1f4722240]>";
|
||||
NSUnknownUserInfoKey = lastDetectedCoordinates;
|
||||
}
|
||||
libc++abi: terminating due to uncaught exception of type NSException
|
||||
# Error Resolution Log
|
||||
|
||||
## Fixed Issues (2023)
|
||||
|
||||
### ScreenCaptureKit API Updates
|
||||
1. SCContentFilter Initialization
|
||||
- Fixed by using correct initializer and parameters:
|
||||
```swift
|
||||
SCContentFilter(display: display, excludingWindows: [])
|
||||
```
|
||||
- Using SCDisplay object directly (not displayID)
|
||||
- Correct parameter name: excludingWindows
|
||||
- Removed incorrect parameters (includingWindows/exceptingWindows)
|
||||
|
||||
2. Stream Output Type
|
||||
- Fixed type inference issue by explicit declaration:
|
||||
```swift
|
||||
let outputType: SCStreamOutputType = .screen
|
||||
try stream.addStreamOutput(self, type: outputType, ...)
|
||||
```
|
||||
- Ensures proper type resolution for .screen member
|
||||
|
||||
### Required Imports
|
||||
- Added necessary framework imports:
|
||||
* CoreMedia
|
||||
* AVFoundation
|
||||
* ScreenCaptureKit
|
||||
* CoreGraphics
|
||||
* AppKit
|
||||
* Foundation
|
||||
|
||||
### Window Capture Strategy
|
||||
1. Window Detection
|
||||
- Precise window identification:
|
||||
```swift
|
||||
let bundleID = window.owningApplication?.bundleIdentifier ?? ""
|
||||
let isChessApp = bundleID == "com.chess.iphone"
|
||||
let hasValidSize = window.frame.width > 100 && window.frame.height > 100
|
||||
return isChessApp && window.isOnScreen && hasValidSize
|
||||
```
|
||||
- Multiple validation checks:
|
||||
* Exact bundle ID match
|
||||
* Window is currently on screen
|
||||
* Window has valid dimensions
|
||||
- Handles iOS apps running on Mac properly
|
||||
|
||||
2. Capture Method
|
||||
- Implemented continuous capture:
|
||||
```swift
|
||||
// Start once
|
||||
try await screenCapture.startCapture()
|
||||
|
||||
// Process frames continuously
|
||||
while !Task.isCancelled {
|
||||
if let image = screenCapture.getCurrentImage() {
|
||||
try await processImage(image)
|
||||
}
|
||||
try await Task.sleep(nanoseconds: 100_000_000)
|
||||
}
|
||||
```
|
||||
- Maintains single active stream
|
||||
- Eliminates capture flickering
|
||||
- Proper cleanup on stop
|
||||
|
||||
### Thread Safety and Async Handling
|
||||
1. Main Actor Isolation
|
||||
- Added @MainActor to ViewModel class:
|
||||
```swift
|
||||
@MainActor
|
||||
class ScreenCaptureViewModel: ObservableObject
|
||||
```
|
||||
- Ensures all @Published property updates happen on main thread
|
||||
- Proper thread safety for SwiftUI bindings
|
||||
|
||||
2. Async Operation Handling
|
||||
- Optimized async/await usage:
|
||||
```swift
|
||||
// Only use await for truly async operations
|
||||
try await screenCapture.startCapture()
|
||||
try await Task.sleep(nanoseconds: 100_000_000)
|
||||
```
|
||||
- Removed unnecessary await keywords:
|
||||
* Non-async error handling
|
||||
* UI state updates
|
||||
* Image processing
|
||||
- Proper @MainActor usage for thread safety
|
||||
|
||||
3. Task and Error Handling
|
||||
- Improved task cancellation:
|
||||
```swift
|
||||
captureLoop: while !Task.isCancelled {
|
||||
do {
|
||||
// Process frame
|
||||
} catch is CancellationError {
|
||||
break captureLoop
|
||||
} catch {
|
||||
// Continue capturing on non-critical errors
|
||||
await handleCaptureError(error)
|
||||
}
|
||||
}
|
||||
```
|
||||
- Proper error recovery:
|
||||
* Continues on board detection failures
|
||||
* Shows error state without stopping
|
||||
* Clears errors on successful detection
|
||||
- Clean error state on stop:
|
||||
```swift
|
||||
captureError = nil // Clear any error when stopping
|
||||
```
|
||||
- Proper cleanup on task exit
|
||||
|
||||
## Technical Notes
|
||||
- Resource Management:
|
||||
* Shared CIContext to prevent command queue exhaustion:
|
||||
```swift
|
||||
private static let shared = CIContext()
|
||||
private var context: CIContext { Self.shared }
|
||||
```
|
||||
* Prevents "Command queue creation failed" errors
|
||||
* Reduces Metal resource usage
|
||||
* Proper cleanup on task completion
|
||||
|
||||
- Using correct SCContentFilter API with proper parameter names
|
||||
- Proper type safety throughout the implementation:
|
||||
* Proper Vision framework result handling:
|
||||
```swift
|
||||
guard let observations = request.results,
|
||||
!observations.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let bestObservation = observations[0]
|
||||
```
|
||||
* No unnecessary type casting
|
||||
* Safe array access
|
||||
* Proper optional handling
|
||||
- Appropriate framework dependencies
|
||||
- Follows current ScreenCaptureKit best practices
|
||||
|
|
|
|||
|
|
@ -1,12 +1,94 @@
|
|||
# System Patterns
|
||||
|
||||
## Window Capture Architecture
|
||||
|
||||
### Window Detection Pattern
|
||||
1. SCShareableContent Access
|
||||
- Async/await pattern for content access
|
||||
- Proper error propagation
|
||||
- Permission handling
|
||||
|
||||
2. Window Identification
|
||||
- Multiple validation criteria:
|
||||
```swift
|
||||
let bundleID = window.owningApplication?.bundleIdentifier ?? ""
|
||||
let isChessApp = bundleID == "com.chess.iphone"
|
||||
let hasValidSize = window.frame.width > 100 && window.frame.height > 100
|
||||
return isChessApp && window.isOnScreen && hasValidSize
|
||||
```
|
||||
- Fail-fast approach with guard statements
|
||||
- Clear error states
|
||||
|
||||
### Capture System Pattern
|
||||
1. Stream Configuration
|
||||
- Window-specific capture setup
|
||||
- Frame dimension matching
|
||||
- Proper delegate handling
|
||||
|
||||
2. Frame Processing
|
||||
- Main thread safety for UI updates
|
||||
- Efficient image conversion pipeline
|
||||
- Resource cleanup
|
||||
|
||||
### Error Handling Pattern
|
||||
1. Task Management
|
||||
- Proper cancellation points
|
||||
- Clean state management
|
||||
- Resource cleanup
|
||||
|
||||
2. Error States
|
||||
- Clear error types
|
||||
- User-friendly messages
|
||||
- State recovery
|
||||
|
||||
## UI Architecture
|
||||
|
||||
### MVVM Implementation
|
||||
1. ViewModel
|
||||
- @MainActor for thread safety
|
||||
- Published properties for state
|
||||
- Clear separation of concerns
|
||||
|
||||
2. View Layer
|
||||
- SwiftUI declarative UI
|
||||
- State-driven updates
|
||||
- Error presentation
|
||||
|
||||
### Async Operations
|
||||
1. Task Management
|
||||
- Structured concurrency
|
||||
- Proper cancellation
|
||||
- State synchronization
|
||||
|
||||
2. State Updates
|
||||
- Main thread safety
|
||||
- Clear state transitions
|
||||
- Error recovery
|
||||
|
||||
## Core Architecture
|
||||
|
||||
### Resource Management Patterns
|
||||
1. Shared CIContext Pattern
|
||||
- Static shared instance:
|
||||
```swift
|
||||
private static let shared = CIContext()
|
||||
private var context: CIContext { Self.shared }
|
||||
```
|
||||
- Benefits:
|
||||
* Prevents Metal command queue exhaustion
|
||||
* Reduces resource overhead
|
||||
* Enables long-running captures
|
||||
- Implementation:
|
||||
* Used in BoardDetector and ViewModel
|
||||
* Proper cleanup on task completion
|
||||
* Thread-safe access
|
||||
|
||||
### Screen Capture System
|
||||
- Uses ScreenCaptureKit for efficient screen capture
|
||||
- Implements SCStreamOutput protocol for frame processing
|
||||
- Handles capture session lifecycle and cleanup
|
||||
- Manages permissions and error handling
|
||||
- Optimized resource usage
|
||||
|
||||
### Board Detection System
|
||||
Two implemented approaches:
|
||||
|
|
|
|||
|
|
@ -8,21 +8,38 @@
|
|||
|
||||
## Core Technologies
|
||||
|
||||
### Metal Resource Management
|
||||
- Shared CIContext pattern:
|
||||
* Static shared instance to prevent command queue exhaustion
|
||||
* Used across BoardDetector and ViewModel
|
||||
* Proper cleanup and resource management
|
||||
- Performance considerations:
|
||||
* Reduced Metal command queue creation
|
||||
* Efficient resource utilization
|
||||
* Support for long-running captures
|
||||
|
||||
### ScreenCaptureKit
|
||||
- System framework for screen capture
|
||||
- Requires user permissions
|
||||
- Supports window/display filtering
|
||||
- Real-time frame capture capabilities
|
||||
|
||||
### Vision Framework
|
||||
- Used for board and coordinate detection
|
||||
- Implemented features:
|
||||
* Window detection using SCShareableContent
|
||||
* iOS app window capture support
|
||||
* Real-time frame capture
|
||||
* Proper error handling
|
||||
- Key components:
|
||||
* SCShareableContent: Window and display access
|
||||
* SCContentFilter: Window-specific capture
|
||||
* SCStream: Frame capture management
|
||||
* SCStreamOutput: Frame processing
|
||||
|
||||
### Vision Framework (Planned)
|
||||
- Will be used for board and coordinate detection
|
||||
- Key components to implement:
|
||||
* VNRecognizeTextRequest: Chess coordinate detection
|
||||
* VNDetectRectanglesRequest: Board boundary detection
|
||||
- Configuration:
|
||||
- Planned configuration:
|
||||
* Text recognition level: accurate
|
||||
* Language correction: disabled
|
||||
* Rectangle aspect ratio: 0.3-0.5 (for taller rectangles)
|
||||
* Rectangle aspect ratio: 0.3-0.5
|
||||
* Minimum size: 0.4
|
||||
* Maximum observations: 1
|
||||
|
||||
|
|
@ -58,36 +75,40 @@
|
|||
|
||||
## Technical Constraints
|
||||
|
||||
### Board Detection
|
||||
1. Pattern Recognition
|
||||
- Detect taller rectangles (0.3-0.5 aspect ratio)
|
||||
- Extract square board from upper portion
|
||||
- Use width as reference measurement
|
||||
- Handle coordinate system transformations
|
||||
### Window Capture System
|
||||
1. Window Detection
|
||||
- Using SCShareableContent for window access
|
||||
- Multiple validation criteria:
|
||||
* Bundle ID verification
|
||||
* Window visibility check
|
||||
* Size validation
|
||||
- Error handling for missing windows
|
||||
|
||||
2. Coordinate Recognition
|
||||
- Must detect a-h and 1-8 coordinates
|
||||
- Handles both light and dark themes
|
||||
- Requires clear coordinate visibility
|
||||
- Minimum text size requirements
|
||||
2. Frame Capture
|
||||
- Window-specific capture configuration
|
||||
- Frame dimension matching
|
||||
- Proper delegate handling
|
||||
- Resource cleanup
|
||||
|
||||
3. Board Boundaries
|
||||
- Square aspect ratio (1:1)
|
||||
- Tolerance: 0.3 for dimensions
|
||||
- Extract from detected area
|
||||
- Proper coordinate transformations
|
||||
3. Performance
|
||||
- Main thread safety for UI updates
|
||||
- Efficient image conversion
|
||||
- Proper task cancellation
|
||||
- Memory management
|
||||
- Shared CIContext for Metal efficiency
|
||||
|
||||
4. Performance
|
||||
- Frame processing on dedicated queue
|
||||
- Asynchronous Vision requests
|
||||
- Memory management for capture session
|
||||
- Resource cleanup requirements
|
||||
4. Error Handling
|
||||
- Clear error types
|
||||
- User-friendly messages
|
||||
- State recovery
|
||||
- Resource cleanup
|
||||
|
||||
### System Requirements
|
||||
- macOS 12.0 or later
|
||||
- Screen Capture permissions
|
||||
- Sufficient CPU for real-time processing
|
||||
- Adequate memory for frame buffering
|
||||
- Metal-capable GPU for image processing
|
||||
|
||||
## Dependencies
|
||||
|
||||
|
|
@ -102,6 +123,7 @@
|
|||
- Vision.framework
|
||||
- SwiftUI.framework
|
||||
- CoreImage.framework
|
||||
- Metal.framework (via CIContext)
|
||||
|
||||
## Development Guidelines
|
||||
|
||||
|
|
@ -110,8 +132,10 @@
|
|||
- Protocol-oriented design
|
||||
- Clear separation of concerns
|
||||
- Comprehensive error handling
|
||||
- Resource sharing patterns
|
||||
|
||||
### Performance Optimization
|
||||
- Shared CIContext for Metal efficiency
|
||||
- Efficient frame processing
|
||||
- Memory management
|
||||
- Resource cleanup
|
||||
|
|
@ -130,16 +154,19 @@
|
|||
- Coordinate transformations
|
||||
- Error handling
|
||||
- State management
|
||||
- Resource management
|
||||
|
||||
### Integration Tests
|
||||
- End-to-end workflows
|
||||
- Component interaction
|
||||
- Event propagation
|
||||
- Resource sharing
|
||||
|
||||
### UI Tests
|
||||
- User interaction flows
|
||||
- Error state handling
|
||||
- Visual feedback
|
||||
- Performance monitoring
|
||||
|
||||
## Documentation Requirements
|
||||
|
||||
|
|
@ -148,27 +175,29 @@
|
|||
- Parameter descriptions
|
||||
- Return value documentation
|
||||
- Error documentation
|
||||
- Resource usage documentation
|
||||
|
||||
### Architecture Documentation
|
||||
- System overview
|
||||
- Component interaction
|
||||
- Data flow diagrams
|
||||
- State management
|
||||
- Resource management patterns
|
||||
|
||||
## Current Challenges
|
||||
|
||||
### Coordinate Systems
|
||||
1. Understanding
|
||||
- Different origin points
|
||||
- Axis directions
|
||||
- Coordinate spaces
|
||||
- Transformation requirements
|
||||
### Resource Management
|
||||
1. Metal Efficiency
|
||||
- Command queue management
|
||||
- Shared context patterns
|
||||
- Resource cleanup
|
||||
- Performance monitoring
|
||||
|
||||
2. Implementation
|
||||
- Proper transformations
|
||||
- Consistent handling
|
||||
- Validation methods
|
||||
- Error checking
|
||||
2. Memory Usage
|
||||
- Frame buffer management
|
||||
- Image processing optimization
|
||||
- Resource pooling
|
||||
- Cleanup strategies
|
||||
|
||||
### Board Detection
|
||||
1. Full Capture
|
||||
|
|
@ -206,3 +235,4 @@
|
|||
- Optimize frame processing
|
||||
- Improve error recovery
|
||||
- Enhanced permission handling
|
||||
- Resource usage monitoring
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue