Fix Chessboard Preview
This commit is contained in:
parent
de331eff0e
commit
8c27f34ee1
9 changed files with 1151 additions and 798 deletions
|
|
@ -1,194 +1,76 @@
|
||||||
import Foundation
|
import Foundation
|
||||||
import Vision
|
|
||||||
import CoreImage
|
import CoreImage
|
||||||
|
import Vision
|
||||||
|
|
||||||
class BoardDetector {
|
class BoardDetector {
|
||||||
private let context = CIContext()
|
// Share CIContext to avoid creating too many Metal command queues
|
||||||
|
private static let shared = CIContext()
|
||||||
// Known chess interface patterns
|
private var context: CIContext { BoardDetector.shared }
|
||||||
private struct ChessPattern {
|
|
||||||
let interfaceType: ChessInterface
|
|
||||||
let boardColor: CGColor
|
|
||||||
let squareSize: CGFloat
|
|
||||||
let cornerPattern: [CGPoint]
|
|
||||||
}
|
|
||||||
|
|
||||||
enum ChessInterface {
|
|
||||||
case chesscom
|
|
||||||
case lichess
|
|
||||||
}
|
|
||||||
|
|
||||||
// Chess.com and Lichess patterns
|
|
||||||
private let patterns: [ChessPattern] = [
|
|
||||||
// Chess.com light theme pattern
|
|
||||||
ChessPattern(
|
|
||||||
interfaceType: .chesscom,
|
|
||||||
boardColor: CGColor(red: 0.82, green: 0.82, blue: 0.82, alpha: 1.0), // Light squares
|
|
||||||
squareSize: 45.0, // Default square size
|
|
||||||
cornerPattern: [
|
|
||||||
CGPoint(x: 0, y: 0),
|
|
||||||
CGPoint(x: 45, y: 0),
|
|
||||||
CGPoint(x: 0, y: 45),
|
|
||||||
CGPoint(x: 45, y: 45)
|
|
||||||
]
|
|
||||||
),
|
|
||||||
// Lichess light theme pattern
|
|
||||||
ChessPattern(
|
|
||||||
interfaceType: .lichess,
|
|
||||||
boardColor: CGColor(red: 0.93, green: 0.93, blue: 0.83, alpha: 1.0), // Light squares
|
|
||||||
squareSize: 40.0, // Default square size
|
|
||||||
cornerPattern: [
|
|
||||||
CGPoint(x: 0, y: 0),
|
|
||||||
CGPoint(x: 40, y: 0),
|
|
||||||
CGPoint(x: 0, y: 40),
|
|
||||||
CGPoint(x: 40, y: 40)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
// MARK: - Pattern Detection
|
|
||||||
|
|
||||||
func detectBoard(in image: CIImage) -> CGRect? {
|
func detectBoard(in image: CIImage) -> CGRect? {
|
||||||
// 1. Color-based detection
|
// Configure rectangle detection request
|
||||||
let colorMatches = detectColorPatterns(in: image)
|
let request = VNDetectRectanglesRequest()
|
||||||
|
request.minimumAspectRatio = 0.8 // Adjusted for standard chess board
|
||||||
// 2. Grid pattern detection
|
request.maximumAspectRatio = 1.2
|
||||||
let gridMatches = detectGridPattern(in: image, colorMatches: colorMatches)
|
request.minimumSize = 0.4
|
||||||
|
request.maximumObservations = 1
|
||||||
// 3. Corner validation
|
request.quadratureTolerance = 30
|
||||||
if let bestMatch = validateCorners(in: image, candidates: gridMatches) {
|
request.minimumConfidence = 0.9
|
||||||
// Extract square board from detected area
|
|
||||||
let width = bestMatch.width
|
|
||||||
let height = bestMatch.height
|
|
||||||
|
|
||||||
// Calculate the board position
|
|
||||||
// Vision coordinates are bottom-left origin, so we need to:
|
|
||||||
// 1. Keep the same width
|
|
||||||
// 2. Use the top portion of the detected area
|
|
||||||
// 3. Maintain square aspect ratio
|
|
||||||
return CGRect(
|
|
||||||
x: bestMatch.minX,
|
|
||||||
y: bestMatch.maxY - width, // Position at top of detected area
|
|
||||||
width: width,
|
|
||||||
height: width
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
// Perform the request
|
||||||
|
let requestHandler = VNImageRequestHandler(ciImage: image, options: [:])
|
||||||
|
do {
|
||||||
|
try requestHandler.perform([request])
|
||||||
|
} catch {
|
||||||
|
print("Failed to perform rectangle detection: \(error)")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
private func detectColorPatterns(in image: CIImage) -> [CGRect] {
|
// Process results
|
||||||
var matches: [CGRect] = []
|
guard let observations = request.results,
|
||||||
|
!observations.isEmpty else {
|
||||||
for pattern in patterns {
|
return nil
|
||||||
// Apply color adjustments to enhance pattern detection
|
|
||||||
let colorControls = CIFilter(name: "CIColorControls")
|
|
||||||
colorControls?.setValue(image, forKey: kCIInputImageKey)
|
|
||||||
colorControls?.setValue(1.2, forKey: kCIInputSaturationKey) // Increase saturation
|
|
||||||
colorControls?.setValue(0.2, forKey: kCIInputBrightnessKey) // Adjust brightness
|
|
||||||
colorControls?.setValue(1.1, forKey: kCIInputContrastKey) // Increase contrast
|
|
||||||
|
|
||||||
guard let adjustedImage = colorControls?.outputImage else { continue }
|
|
||||||
|
|
||||||
// Detect rectangles
|
|
||||||
var rectangles: [VNRectangleObservation] = []
|
|
||||||
let request = VNDetectRectanglesRequest()
|
|
||||||
request.minimumAspectRatio = 0.3 // Allow taller rectangles
|
|
||||||
request.maximumAspectRatio = 0.5
|
|
||||||
request.minimumSize = 0.4 // Look for larger areas
|
|
||||||
request.maximumObservations = 1
|
|
||||||
|
|
||||||
let handler = VNImageRequestHandler(ciImage: adjustedImage)
|
|
||||||
try? handler.perform([request])
|
|
||||||
|
|
||||||
if let results = request.results as? [VNRectangleObservation] {
|
|
||||||
matches.append(contentsOf: results.map { $0.boundingBox })
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return matches
|
let bestObservation = observations[0]
|
||||||
|
|
||||||
|
// Convert Vision coordinates to CoreImage coordinates
|
||||||
|
let imageSize = image.extent.size
|
||||||
|
let transform = CGAffineTransform(scaleX: imageSize.width, y: imageSize.height)
|
||||||
|
|
||||||
|
// Create normalized rect in CoreImage coordinate space
|
||||||
|
let detectedRect = bestObservation.boundingBox.applying(transform)
|
||||||
|
|
||||||
|
// Validate the detected rectangle
|
||||||
|
guard validateDetectedRect(detectedRect, in: image) else {
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
private func detectGridPattern(in image: CIImage, colorMatches: [CGRect]) -> [CGRect] {
|
return detectedRect
|
||||||
var gridMatches: [CGRect] = []
|
|
||||||
|
|
||||||
for rect in colorMatches {
|
|
||||||
// Create edge detection filter
|
|
||||||
let edgeFilter = CIFilter(name: "CIEdgeWork")
|
|
||||||
edgeFilter?.setValue(image.cropped(to: rect), forKey: kCIInputImageKey)
|
|
||||||
edgeFilter?.setValue(1.0, forKey: kCIInputRadiusKey)
|
|
||||||
|
|
||||||
guard let edgeImage = edgeFilter?.outputImage else { continue }
|
|
||||||
|
|
||||||
// Detect rectangles in edge image
|
|
||||||
var rectangles: [VNRectangleObservation] = []
|
|
||||||
let request = VNDetectRectanglesRequest()
|
|
||||||
request.minimumAspectRatio = 0.3
|
|
||||||
request.maximumAspectRatio = 0.5
|
|
||||||
request.minimumSize = 0.4
|
|
||||||
request.maximumObservations = 1
|
|
||||||
|
|
||||||
let handler = VNImageRequestHandler(ciImage: edgeImage)
|
|
||||||
try? handler.perform([request])
|
|
||||||
|
|
||||||
if let results = request.results as? [VNRectangleObservation] {
|
|
||||||
for observation in results {
|
|
||||||
if validateGridDimensions(observation.boundingBox, squareSize: rect.width / 8) {
|
|
||||||
gridMatches.append(observation.boundingBox)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return gridMatches
|
private func validateDetectedRect(_ rect: CGRect, in image: CIImage) -> Bool {
|
||||||
|
let imageSize = image.extent.size
|
||||||
|
|
||||||
|
// Check if rectangle is within image bounds
|
||||||
|
guard image.extent.contains(rect) else {
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
private func validateGridDimensions(_ rect: CGRect, squareSize: CGFloat) -> Bool {
|
// Validate aspect ratio (standard chess board is square)
|
||||||
// Check if dimensions match 8x8 grid with some tolerance
|
let aspectRatio = rect.width / rect.height
|
||||||
let expectedSize = squareSize * 8
|
guard aspectRatio >= 0.9 && aspectRatio <= 1.1 else {
|
||||||
let tolerance: CGFloat = 0.3 // More flexible tolerance
|
return false
|
||||||
|
|
||||||
// Only validate width since height detection is unreliable
|
|
||||||
let widthMatch = abs(rect.width - expectedSize) <= (expectedSize * tolerance)
|
|
||||||
|
|
||||||
return widthMatch
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private func validateCorners(in image: CIImage, candidates: [CGRect]) -> CGRect? {
|
// Validate size relative to image
|
||||||
var bestMatch: (rect: CGRect, score: Float)?
|
let minDimension = min(imageSize.width, imageSize.height)
|
||||||
|
let boardSize = max(rect.width, rect.height)
|
||||||
for rect in candidates {
|
guard boardSize >= minDimension * 0.4 else {
|
||||||
var totalScore: Float = 0
|
return false
|
||||||
|
|
||||||
// Extract corners
|
|
||||||
let corners = [
|
|
||||||
CGPoint(x: rect.minX, y: rect.minY),
|
|
||||||
CGPoint(x: rect.maxX, y: rect.minY),
|
|
||||||
CGPoint(x: rect.minX, y: rect.maxY),
|
|
||||||
CGPoint(x: rect.maxX, y: rect.maxY)
|
|
||||||
]
|
|
||||||
|
|
||||||
// Compare with pattern corners
|
|
||||||
for pattern in patterns {
|
|
||||||
var patternScore: Float = 0
|
|
||||||
|
|
||||||
for (corner, patternCorner) in zip(corners, pattern.cornerPattern) {
|
|
||||||
let distance = hypot(
|
|
||||||
corner.x - patternCorner.x,
|
|
||||||
corner.y - patternCorner.y
|
|
||||||
)
|
|
||||||
patternScore += Float(1.0 / (1.0 + distance))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
totalScore = max(totalScore, patternScore / Float(corners.count))
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update best match if score is higher
|
|
||||||
if totalScore > (bestMatch?.score ?? 0) {
|
|
||||||
bestMatch = (rect, totalScore)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return bestMatch?.rect
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,21 +2,16 @@ import SwiftUI
|
||||||
|
|
||||||
struct ContentView: View {
|
struct ContentView: View {
|
||||||
@StateObject private var viewModel = ScreenCaptureViewModel()
|
@StateObject private var viewModel = ScreenCaptureViewModel()
|
||||||
|
@State private var showDebugOverlay = false
|
||||||
|
|
||||||
var body: some View {
|
var body: some View {
|
||||||
VStack(spacing: 20) {
|
VStack {
|
||||||
// Screen Capture Controls
|
// Capture controls
|
||||||
HStack {
|
HStack {
|
||||||
Button(action: {
|
Button(action: {
|
||||||
Task {
|
viewModel.startCapture()
|
||||||
await viewModel.startCapture()
|
|
||||||
}
|
|
||||||
}) {
|
}) {
|
||||||
Text("Start Capture")
|
Text("Start Capture")
|
||||||
.padding()
|
|
||||||
.background(viewModel.isCapturing ? Color.gray : Color.blue)
|
|
||||||
.foregroundColor(.white)
|
|
||||||
.cornerRadius(8)
|
|
||||||
}
|
}
|
||||||
.disabled(viewModel.isCapturing)
|
.disabled(viewModel.isCapturing)
|
||||||
|
|
||||||
|
|
@ -24,112 +19,59 @@ struct ContentView: View {
|
||||||
viewModel.stopCapture()
|
viewModel.stopCapture()
|
||||||
}) {
|
}) {
|
||||||
Text("Stop Capture")
|
Text("Stop Capture")
|
||||||
.padding()
|
|
||||||
.background(!viewModel.isCapturing ? Color.gray : Color.red)
|
|
||||||
.foregroundColor(.white)
|
|
||||||
.cornerRadius(8)
|
|
||||||
}
|
}
|
||||||
.disabled(!viewModel.isCapturing)
|
.disabled(!viewModel.isCapturing)
|
||||||
}
|
}
|
||||||
|
.padding()
|
||||||
|
|
||||||
// Board Detection Visualization
|
// Error display
|
||||||
if let rect = viewModel.detectedBoardRect {
|
|
||||||
GeometryReader { geometry in
|
|
||||||
Rectangle()
|
|
||||||
.stroke(Color.green, lineWidth: 2)
|
|
||||||
.frame(
|
|
||||||
width: rect.width * geometry.size.width,
|
|
||||||
height: rect.height * geometry.size.height
|
|
||||||
)
|
|
||||||
.position(
|
|
||||||
x: rect.midX * geometry.size.width,
|
|
||||||
y: rect.midY * geometry.size.height
|
|
||||||
)
|
|
||||||
}
|
|
||||||
.background(Color.black.opacity(0.1))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Error Display
|
|
||||||
if let error = viewModel.captureError {
|
if let error = viewModel.captureError {
|
||||||
VStack(spacing: 10) {
|
|
||||||
Text(error.localizedDescription)
|
Text(error.localizedDescription)
|
||||||
.font(.headline)
|
|
||||||
.foregroundColor(.red)
|
.foregroundColor(.red)
|
||||||
.multilineTextAlignment(.center)
|
|
||||||
|
|
||||||
if case .chessWindowNotFound = error {
|
|
||||||
VStack(alignment: .leading, spacing: 5) {
|
|
||||||
Text("Requirements:")
|
|
||||||
.font(.subheadline)
|
|
||||||
.bold()
|
|
||||||
Text("• Chess.com app or Firefox must be running")
|
|
||||||
Text("• A chess game must be open")
|
|
||||||
Text("• The game window must be visible")
|
|
||||||
}
|
|
||||||
.padding()
|
.padding()
|
||||||
.background(Color.red.opacity(0.1))
|
|
||||||
.cornerRadius(8)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.padding()
|
|
||||||
.frame(maxWidth: .infinity)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Image Display Section
|
// Image display
|
||||||
HStack(spacing: 20) {
|
HStack {
|
||||||
// Full Capture Display
|
|
||||||
if let image = viewModel.capturedImage {
|
|
||||||
VStack {
|
VStack {
|
||||||
Text("Full Capture")
|
Text("Full Capture")
|
||||||
.font(.caption)
|
if let image = viewModel.capturedImage {
|
||||||
Image(nsImage: image)
|
Image(nsImage: image)
|
||||||
.resizable()
|
.resizable()
|
||||||
.aspectRatio(contentMode: .fit)
|
.aspectRatio(contentMode: .fit)
|
||||||
.frame(height: 400) // Fixed height to show full board
|
.frame(maxWidth: 400)
|
||||||
|
} else {
|
||||||
|
Text("No capture available")
|
||||||
|
.foregroundColor(.gray)
|
||||||
}
|
}
|
||||||
.padding()
|
|
||||||
.background(Color.black.opacity(0.05))
|
|
||||||
.cornerRadius(8)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cropped Board Display
|
Divider()
|
||||||
if let cropped = viewModel.croppedBoardImage {
|
|
||||||
VStack {
|
VStack {
|
||||||
Text("Chessboard")
|
Text("Chessboard Preview")
|
||||||
.font(.caption)
|
if let image = viewModel.croppedBoardImage {
|
||||||
Image(nsImage: cropped)
|
Image(nsImage: image)
|
||||||
.resizable()
|
.resizable()
|
||||||
.aspectRatio(contentMode: .fit)
|
.aspectRatio(contentMode: .fit)
|
||||||
.frame(height: 400) // Match the board height
|
.frame(maxWidth: 400)
|
||||||
|
.aspectRatio(contentMode: .fit)
|
||||||
|
} else {
|
||||||
|
Text("No board detected")
|
||||||
|
.foregroundColor(.gray)
|
||||||
}
|
}
|
||||||
.padding()
|
|
||||||
.background(Color.black.opacity(0.05))
|
|
||||||
.cornerRadius(8)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.padding()
|
.padding()
|
||||||
|
|
||||||
// Vision Model Output Placeholder
|
|
||||||
if viewModel.croppedBoardImage != nil {
|
|
||||||
VStack(alignment: .leading) {
|
|
||||||
Text("Board Analysis")
|
|
||||||
.font(.headline)
|
|
||||||
Text("FEN: (processing...)")
|
|
||||||
.font(.system(.body, design: .monospaced))
|
|
||||||
}
|
|
||||||
.padding()
|
|
||||||
.frame(maxWidth: .infinity, alignment: .leading)
|
|
||||||
.background(Color.blue.opacity(0.1))
|
|
||||||
.cornerRadius(8)
|
|
||||||
}
|
|
||||||
|
|
||||||
Spacer()
|
Spacer()
|
||||||
}
|
}
|
||||||
.padding()
|
.frame(minWidth: 800, minHeight: 600)
|
||||||
.frame(minWidth: 800, minHeight: 600) // Increased window size to fit larger board display
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#Preview {
|
struct ContentView_Previews: PreviewProvider {
|
||||||
|
static var previews: some View {
|
||||||
ContentView()
|
ContentView()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,290 +1,64 @@
|
||||||
import Foundation
|
import Foundation
|
||||||
|
import AppKit
|
||||||
|
import CoreGraphics
|
||||||
|
import CoreMedia
|
||||||
import ScreenCaptureKit
|
import ScreenCaptureKit
|
||||||
import Vision
|
import AVFoundation
|
||||||
import CoreImage
|
|
||||||
|
|
||||||
class ScreenCapture: NSObject {
|
class ScreenCapture: NSObject, SCStreamOutput {
|
||||||
private var captureSession: SCStream?
|
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
||||||
private let queue = DispatchQueue(label: "com.chessprism.screencapture", qos: .userInteractive)
|
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
|
||||||
private var lastDetectedCoordinates: [(text: String, boundingBox: CGRect)] = []
|
|
||||||
private let boardDetector = BoardDetector()
|
|
||||||
|
|
||||||
// MARK: - Screen Capture Setup
|
|
||||||
func startCapture() async throws {
|
|
||||||
print("Starting screen capture...")
|
|
||||||
// Get available screen content
|
|
||||||
let availableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
|
|
||||||
print("Available content: \(availableContent.displays.count) displays, \(availableContent.windows.count) windows")
|
|
||||||
|
|
||||||
// Find Chess.com app window
|
|
||||||
let chessWindow = availableContent.windows.first { window in
|
|
||||||
if let bundleId = window.owningApplication?.bundleIdentifier {
|
|
||||||
return bundleId == "com.chess.iphone"
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
guard let chessWindow = chessWindow else {
|
|
||||||
throw ScreenCaptureError.chessWindowNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create stream configuration targeting chess window
|
|
||||||
let filter = SCContentFilter(desktopIndependentWindow: chessWindow)
|
|
||||||
let configuration = SCStreamConfiguration()
|
|
||||||
configuration.width = 1920
|
|
||||||
configuration.height = 1080
|
|
||||||
configuration.pixelFormat = kCVPixelFormatType_32BGRA
|
|
||||||
configuration.queueDepth = 5
|
|
||||||
|
|
||||||
// Create and start stream
|
|
||||||
captureSession = SCStream(filter: filter, configuration: configuration, delegate: nil)
|
|
||||||
|
|
||||||
try await captureSession?.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
|
|
||||||
try await captureSession?.startCapture()
|
|
||||||
|
|
||||||
print("Screen capture started successfully")
|
|
||||||
NotificationCenter.default.post(
|
|
||||||
name: .captureStateChanged,
|
|
||||||
object: nil,
|
|
||||||
userInfo: ["isCapturing": true]
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: - Frame Processing
|
|
||||||
private func processFrame(sampleBuffer: CMSampleBuffer) {
|
|
||||||
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
|
||||||
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
|
|
||||||
|
|
||||||
// Convert CIImage to NSImage for display
|
|
||||||
let context = CIContext()
|
|
||||||
if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) {
|
|
||||||
let size = NSSize(width: cgImage.width, height: cgImage.height)
|
|
||||||
let nsImage = NSImage(cgImage: cgImage, size: size)
|
|
||||||
|
|
||||||
// Notify subscribers with captured image
|
|
||||||
NotificationCenter.default.post(
|
|
||||||
name: .capturedFrame,
|
|
||||||
object: nil,
|
|
||||||
userInfo: ["image": nsImage]
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try pattern-based detection first
|
|
||||||
if let boardRect = boardDetector.detectBoard(in: ciImage) {
|
|
||||||
// Add padding to ensure full board capture
|
|
||||||
let padding = boardRect.width * 0.25 // 25% padding
|
|
||||||
let paddedRect = CGRect(
|
|
||||||
x: boardRect.minX - padding,
|
|
||||||
y: boardRect.minY - padding,
|
|
||||||
width: boardRect.width + (padding * 2),
|
|
||||||
height: boardRect.height + (padding * 2)
|
|
||||||
)
|
|
||||||
NotificationCenter.default.post(
|
|
||||||
name: .boardDetected,
|
|
||||||
object: nil,
|
|
||||||
userInfo: ["rect": paddedRect]
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback to coordinate-based detection
|
DispatchQueue.main.async {
|
||||||
let textRequest = VNRecognizeTextRequest { [weak self] request, error in
|
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
|
||||||
guard let self = self else { return }
|
let rep = NSCIImageRep(ciImage: ciImage)
|
||||||
self.handleTextDetection(request: request)
|
let nsImage = NSImage(size: rep.size)
|
||||||
}
|
nsImage.addRepresentation(rep)
|
||||||
textRequest.recognitionLevel = .accurate
|
|
||||||
textRequest.usesLanguageCorrection = false
|
|
||||||
|
|
||||||
let rectangleRequest = VNDetectRectanglesRequest { [weak self] request, error in
|
self.lastCapturedImage = nsImage
|
||||||
guard let self = self else { return }
|
|
||||||
self.handleBoardDetection(request: request)
|
|
||||||
}
|
|
||||||
rectangleRequest.minimumAspectRatio = 0.8
|
|
||||||
rectangleRequest.maximumAspectRatio = 1.2
|
|
||||||
rectangleRequest.minimumSize = 0.2
|
|
||||||
rectangleRequest.maximumObservations = 3
|
|
||||||
|
|
||||||
let handler = VNImageRequestHandler(ciImage: ciImage)
|
|
||||||
try? handler.perform([textRequest, rectangleRequest])
|
|
||||||
}
|
|
||||||
|
|
||||||
private func handleTextDetection(request: VNRequest) {
|
|
||||||
guard let results = request.results as? [VNRecognizedTextObservation] else { return }
|
|
||||||
|
|
||||||
// Filter for chess coordinates (a-h, 1-8)
|
|
||||||
var coordinates: [(text: String, boundingBox: CGRect)] = []
|
|
||||||
for observation in results {
|
|
||||||
if let candidate = observation.topCandidates(1).first {
|
|
||||||
let text = candidate.string
|
|
||||||
if (text.count == 1 && ("a"..."h").contains(text.lowercased())) ||
|
|
||||||
(text.count == 1 && ("1"..."8").contains(text)) {
|
|
||||||
coordinates.append((text: text, boundingBox: observation.boundingBox))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store coordinates and notify observers
|
|
||||||
if !coordinates.isEmpty {
|
|
||||||
lastDetectedCoordinates = coordinates
|
|
||||||
NotificationCenter.default.post(
|
|
||||||
name: .boardCoordinatesDetected,
|
|
||||||
object: nil,
|
|
||||||
userInfo: ["coordinates": coordinates]
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - Board Detection Handling
|
private var activeStream: SCStream?
|
||||||
private func handleBoardDetection(request: VNRequest) {
|
private var lastCapturedImage: NSImage?
|
||||||
guard let results = request.results as? [VNRectangleObservation],
|
|
||||||
let boardRect = results.first else { return }
|
|
||||||
|
|
||||||
// Convert rectangle to screen coordinates
|
func startCapture() async throws {
|
||||||
var transformedRect = transformRectangle(boardRect)
|
guard #available(macOS 12.3, *) else { return }
|
||||||
|
|
||||||
// Use the stored coordinates
|
let content = try await SCShareableContent.current
|
||||||
let coordinates = lastDetectedCoordinates
|
|
||||||
|
|
||||||
// If we have coordinates, use them to refine the rectangle
|
// Get the window from the content
|
||||||
if !coordinates.isEmpty {
|
guard let window = content.windows.first(where: { window in
|
||||||
// Separate horizontal and vertical coordinates
|
let bundleID = window.owningApplication?.bundleIdentifier ?? ""
|
||||||
let horizontalCoords = coordinates.filter { ("a"..."h").contains($0.text.lowercased()) }
|
let isChessApp = bundleID == "com.chess.iphone"
|
||||||
let verticalCoords = coordinates.filter { ("1"..."8").contains($0.text) }
|
let hasValidSize = window.frame.width > 100 && window.frame.height > 100
|
||||||
|
return isChessApp && window.isOnScreen && hasValidSize
|
||||||
// Get min/max for horizontal coordinates (a-h)
|
}) else {
|
||||||
let xCoords = horizontalCoords.map { $0.boundingBox.origin.x }
|
throw NSError(domain: "ChessPrism", code: 1, userInfo: [NSLocalizedDescriptionKey: "Chess window not found"])
|
||||||
let minX = xCoords.min() ?? transformedRect.minX
|
|
||||||
let maxX = xCoords.max() ?? transformedRect.maxX
|
|
||||||
|
|
||||||
// Get min/max for vertical coordinates (1-8)
|
|
||||||
let yCoords = verticalCoords.map { $0.boundingBox.origin.y }
|
|
||||||
let minY = yCoords.min() ?? transformedRect.minY
|
|
||||||
let maxY = yCoords.max() ?? transformedRect.maxY
|
|
||||||
|
|
||||||
// Calculate board boundaries directly from coordinates
|
|
||||||
let leftMost = horizontalCoords.min { $0.boundingBox.origin.x < $1.boundingBox.origin.x }?.boundingBox.origin.x ?? minX
|
|
||||||
let rightMost = horizontalCoords.max { $0.boundingBox.origin.x < $1.boundingBox.origin.x }?.boundingBox.origin.x ?? maxX
|
|
||||||
let bottomMost = verticalCoords.min { $0.boundingBox.origin.y < $1.boundingBox.origin.y }?.boundingBox.origin.y ?? minY
|
|
||||||
let topMost = verticalCoords.max { $0.boundingBox.origin.y < $1.boundingBox.origin.y }?.boundingBox.origin.y ?? maxY
|
|
||||||
|
|
||||||
// Calculate board dimensions directly from coordinate positions
|
|
||||||
let boardWidth = rightMost - leftMost
|
|
||||||
let boardHeight = topMost - bottomMost
|
|
||||||
|
|
||||||
// Find coordinate pairs that are likely on opposite sides of the board
|
|
||||||
let horizontalPairs = horizontalCoords.flatMap { h1 in
|
|
||||||
horizontalCoords.map { h2 in
|
|
||||||
(h1, h2, abs(h1.boundingBox.origin.x - h2.boundingBox.origin.x))
|
|
||||||
}
|
|
||||||
}.sorted { $0.2 > $1.2 }
|
|
||||||
|
|
||||||
let verticalPairs = verticalCoords.flatMap { v1 in
|
|
||||||
verticalCoords.map { v2 in
|
|
||||||
(v1, v2, abs(v1.boundingBox.origin.y - v2.boundingBox.origin.y))
|
|
||||||
}
|
|
||||||
}.sorted { $0.2 > $1.2 }
|
|
||||||
|
|
||||||
// Use the most distant pairs to define board boundaries
|
|
||||||
if let widestHorizontal = horizontalPairs.first,
|
|
||||||
let tallestVertical = verticalPairs.first {
|
|
||||||
|
|
||||||
// Calculate board dimensions using the most distant coordinates
|
|
||||||
let boardWidth = widestHorizontal.2
|
|
||||||
let boardHeight = tallestVertical.2
|
|
||||||
|
|
||||||
// Calculate the center of the board
|
|
||||||
let centerX = (widestHorizontal.0.boundingBox.origin.x + widestHorizontal.1.boundingBox.origin.x) / 2
|
|
||||||
let centerY = (tallestVertical.0.boundingBox.origin.y + tallestVertical.1.boundingBox.origin.y) / 2
|
|
||||||
|
|
||||||
// Add padding proportional to the coordinate spacing
|
|
||||||
let padding = max(boardWidth, boardHeight) * 0.25 // 25% padding
|
|
||||||
|
|
||||||
// Set rectangle to centered board with padding
|
|
||||||
transformedRect = CGRect(
|
|
||||||
x: centerX - (boardWidth / 2) - padding,
|
|
||||||
y: centerY - (boardHeight / 2) - padding,
|
|
||||||
width: boardWidth + (padding * 2),
|
|
||||||
height: boardHeight + (padding * 2)
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure we capture full board with reasonable bounds before aspect ratio adjustment
|
let filter = SCContentFilter(desktopIndependentWindow: window)
|
||||||
transformedRect.origin.x = max(-0.1, transformedRect.origin.x) // Allow 10% outside bounds
|
let config = SCStreamConfiguration()
|
||||||
transformedRect.origin.y = max(-0.1, transformedRect.origin.y)
|
config.width = Int(window.frame.width)
|
||||||
transformedRect.size.width = min(1.2 - transformedRect.origin.x, transformedRect.size.width) // Allow 20% overflow
|
config.height = Int(window.frame.height)
|
||||||
transformedRect.size.height = min(1.2 - transformedRect.origin.y, transformedRect.size.height)
|
|
||||||
|
|
||||||
// Apply single aspect ratio adjustment with flexible tolerance
|
let stream = SCStream(filter: filter, configuration: config, delegate: nil)
|
||||||
let targetAspectRatio: CGFloat = 1.0
|
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: DispatchQueue.global(qos: .userInitiated))
|
||||||
let currentAspectRatio = transformedRect.size.width / transformedRect.size.height
|
|
||||||
if abs(currentAspectRatio - targetAspectRatio) > 0.2 {
|
// Store the stream before starting it
|
||||||
if currentAspectRatio > targetAspectRatio {
|
activeStream = stream
|
||||||
let newHeight = transformedRect.size.width
|
try await stream.startCapture()
|
||||||
let heightDiff = newHeight - transformedRect.size.height
|
|
||||||
transformedRect.origin.y -= heightDiff / 2.0
|
|
||||||
transformedRect.size.height = newHeight
|
|
||||||
} else {
|
|
||||||
let newWidth = transformedRect.size.height
|
|
||||||
let widthDiff = newWidth - transformedRect.size.width
|
|
||||||
transformedRect.origin.x -= widthDiff / 2.0
|
|
||||||
transformedRect.size.width = newWidth
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Notify subscribers
|
func stopCapture() async throws {
|
||||||
NotificationCenter.default.post(
|
guard let stream = activeStream else { return }
|
||||||
name: .boardDetected,
|
try await stream.stopCapture()
|
||||||
object: nil,
|
activeStream = nil
|
||||||
userInfo: ["rect": transformedRect]
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - Coordinate Transformation
|
func getCurrentImage() -> NSImage? {
|
||||||
private func transformRectangle(_ rect: VNRectangleObservation) -> CGRect {
|
return lastCapturedImage
|
||||||
// Vision coordinates are in normalized coordinates with origin at bottom-left
|
|
||||||
// We need to convert to top-left origin coordinates
|
|
||||||
return CGRect(
|
|
||||||
x: rect.boundingBox.origin.x,
|
|
||||||
y: 1 - rect.boundingBox.origin.y - rect.boundingBox.height,
|
|
||||||
width: rect.boundingBox.width,
|
|
||||||
height: rect.boundingBox.height
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: - Capture Control
|
|
||||||
func stopCapture() {
|
|
||||||
print("Stopping screen capture...")
|
|
||||||
captureSession?.stopCapture()
|
|
||||||
captureSession = nil
|
|
||||||
NotificationCenter.default.post(
|
|
||||||
name: .captureStateChanged,
|
|
||||||
object: nil,
|
|
||||||
userInfo: ["isCapturing": false]
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: - Error Handling
|
|
||||||
enum ScreenCaptureError: Error {
|
|
||||||
case chessWindowNotFound
|
|
||||||
case streamCreationFailed
|
|
||||||
case permissionDenied
|
|
||||||
case captureAlreadyRunning
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - SCStreamOutput
|
|
||||||
extension ScreenCapture: SCStreamOutput {
|
|
||||||
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
|
||||||
guard type == .screen else { return }
|
|
||||||
processFrame(sampleBuffer: sampleBuffer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: - Notifications
|
|
||||||
extension Notification.Name {
|
|
||||||
static let boardDetected = Notification.Name("com.chessprism.boardDetected")
|
|
||||||
static let captureStateChanged = Notification.Name("com.chessprism.captureStateChanged")
|
|
||||||
static let capturedFrame = Notification.Name("com.chessprism.capturedFrame")
|
|
||||||
static let boardCoordinatesDetected = Notification.Name("com.chessprism.boardCoordinatesDetected")
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,157 +1,120 @@
|
||||||
import Foundation
|
import Foundation
|
||||||
import SwiftUI
|
import AppKit
|
||||||
|
import CoreImage
|
||||||
import Combine
|
import Combine
|
||||||
|
|
||||||
@MainActor
|
@MainActor
|
||||||
class ScreenCaptureViewModel: ObservableObject {
|
class ScreenCaptureViewModel: ObservableObject {
|
||||||
// MARK: - Published Properties
|
@Published var capturedImage: NSImage?
|
||||||
|
@Published var croppedBoardImage: NSImage?
|
||||||
|
@Published var captureError: CaptureError?
|
||||||
@Published var isCapturing = false
|
@Published var isCapturing = false
|
||||||
@Published var captureError: ScreenCaptureError? = nil
|
|
||||||
@Published var detectedBoardRect: CGRect? = nil
|
|
||||||
@Published var capturedImage: NSImage? = nil
|
|
||||||
@Published var croppedBoardImage: NSImage? = nil
|
|
||||||
|
|
||||||
// MARK: - Dependencies
|
|
||||||
private let screenCapture = ScreenCapture()
|
private let screenCapture = ScreenCapture()
|
||||||
private var cancellables = Set<AnyCancellable>()
|
private let boardDetector = BoardDetector()
|
||||||
|
private var captureTask: Task<Void, Never>?
|
||||||
|
// Share CIContext to avoid creating too many Metal command queues
|
||||||
|
private static let shared = CIContext()
|
||||||
|
private var context: CIContext { ScreenCaptureViewModel.shared }
|
||||||
|
|
||||||
// MARK: - Initialization
|
private var detectedBoardRect: CGRect?
|
||||||
init() {
|
|
||||||
setupObservers()
|
enum CaptureError: LocalizedError {
|
||||||
|
case chessWindowNotFound
|
||||||
|
case boardDetectionFailed
|
||||||
|
|
||||||
|
var errorDescription: String? {
|
||||||
|
switch self {
|
||||||
|
case .chessWindowNotFound:
|
||||||
|
return "Chess window not found"
|
||||||
|
case .boardDetectionFailed:
|
||||||
|
return "Failed to detect chess board"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - Screen Capture Management
|
func startCapture() {
|
||||||
func startCapture() async {
|
|
||||||
guard await requestScreenCapturePermission() else {
|
|
||||||
captureError = .permissionDenied
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
do {
|
|
||||||
try await screenCapture.startCapture()
|
|
||||||
isCapturing = true
|
isCapturing = true
|
||||||
} catch let error as ScreenCapture.ScreenCaptureError {
|
captureError = nil
|
||||||
captureError = ScreenCaptureError(from: error)
|
|
||||||
|
captureTask = Task {
|
||||||
|
do {
|
||||||
|
// Start continuous capture
|
||||||
|
try await screenCapture.startCapture()
|
||||||
|
|
||||||
|
captureLoop: while !Task.isCancelled {
|
||||||
|
do {
|
||||||
|
if let image = screenCapture.getCurrentImage() {
|
||||||
|
try await processImage(image)
|
||||||
|
}
|
||||||
|
try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
|
||||||
|
} catch is CancellationError {
|
||||||
|
break captureLoop
|
||||||
} catch {
|
} catch {
|
||||||
captureError = .unknown(error)
|
// Just update the error state but continue capturing
|
||||||
|
if !Task.isCancelled {
|
||||||
|
handleCaptureError(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
try? await screenCapture.stopCapture()
|
||||||
|
if isCapturing {
|
||||||
|
isCapturing = false
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
handleCaptureError(error)
|
||||||
|
isCapturing = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func stopCapture() {
|
func stopCapture() {
|
||||||
screenCapture.stopCapture()
|
captureTask?.cancel()
|
||||||
|
captureTask = nil
|
||||||
isCapturing = false
|
isCapturing = false
|
||||||
|
captureError = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - Permission Handling
|
private func processImage(_ image: NSImage) async throws {
|
||||||
private func requestScreenCapturePermission() async -> Bool {
|
// Convert to CIImage for processing
|
||||||
return await withCheckedContinuation { continuation in
|
|
||||||
let status = CGPreflightScreenCaptureAccess()
|
|
||||||
if status {
|
|
||||||
continuation.resume(returning: true)
|
|
||||||
} else {
|
|
||||||
CGRequestScreenCaptureAccess()
|
|
||||||
continuation.resume(returning: false)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: - Notification Handling
|
|
||||||
private func setupObservers() {
|
|
||||||
// Observe board detection
|
|
||||||
NotificationCenter.default
|
|
||||||
.publisher(for: .boardDetected)
|
|
||||||
.receive(on: DispatchQueue.main)
|
|
||||||
.sink(receiveValue: { [weak self] notification in
|
|
||||||
guard let rect = notification.userInfo?["rect"] as? CGRect else { return }
|
|
||||||
self?.detectedBoardRect = rect
|
|
||||||
})
|
|
||||||
.store(in: &cancellables)
|
|
||||||
|
|
||||||
// Observe capture state changes
|
|
||||||
NotificationCenter.default
|
|
||||||
.publisher(for: .captureStateChanged)
|
|
||||||
.receive(on: DispatchQueue.main)
|
|
||||||
.sink(receiveValue: { [weak self] notification in
|
|
||||||
guard let isCapturing = notification.userInfo?["isCapturing"] as? Bool else { return }
|
|
||||||
self?.isCapturing = isCapturing
|
|
||||||
})
|
|
||||||
.store(in: &cancellables)
|
|
||||||
|
|
||||||
// Observe captured frames
|
|
||||||
NotificationCenter.default
|
|
||||||
.publisher(for: .capturedFrame)
|
|
||||||
.receive(on: DispatchQueue.main)
|
|
||||||
.sink(receiveValue: { [weak self] notification in
|
|
||||||
guard let image = notification.userInfo?["image"] as? NSImage,
|
|
||||||
let self = self else { return }
|
|
||||||
|
|
||||||
self.capturedImage = image
|
|
||||||
|
|
||||||
// Crop to detected board area if available
|
|
||||||
if let rect = self.detectedBoardRect {
|
|
||||||
let cropped = self.cropImage(image, to: rect)
|
|
||||||
self.croppedBoardImage = cropped
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.store(in: &cancellables)
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: - Error Handling
|
|
||||||
enum ScreenCaptureError: Error, LocalizedError {
|
|
||||||
case permissionDenied
|
|
||||||
case chessWindowNotFound
|
|
||||||
case streamCreationFailed
|
|
||||||
case captureAlreadyRunning
|
|
||||||
case unknown(Error)
|
|
||||||
|
|
||||||
init(from error: ScreenCapture.ScreenCaptureError) {
|
|
||||||
switch error {
|
|
||||||
case .chessWindowNotFound:
|
|
||||||
self = .chessWindowNotFound
|
|
||||||
case .streamCreationFailed:
|
|
||||||
self = .streamCreationFailed
|
|
||||||
case .permissionDenied:
|
|
||||||
self = .permissionDenied
|
|
||||||
case .captureAlreadyRunning:
|
|
||||||
self = .captureAlreadyRunning
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var errorDescription: String? {
|
|
||||||
switch self {
|
|
||||||
case .permissionDenied:
|
|
||||||
return "Screen recording permission is required"
|
|
||||||
case .chessWindowNotFound:
|
|
||||||
return "Unable to find Chess window (make sure Chess.com app or Firefox is open)"
|
|
||||||
case .streamCreationFailed:
|
|
||||||
return "Failed to create screen capture stream"
|
|
||||||
case .captureAlreadyRunning:
|
|
||||||
return "Capture is already running"
|
|
||||||
case .unknown(let error):
|
|
||||||
return "An unexpected error occurred: \(error.localizedDescription)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private func cropImage(_ image: NSImage, to rect: CGRect) -> NSImage? {
|
|
||||||
guard let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else {
|
guard let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else {
|
||||||
return nil
|
throw CaptureError.boardDetectionFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert normalized rect to pixel coordinates
|
let ciImage = CIImage(cgImage: cgImage)
|
||||||
let width = CGFloat(cgImage.width)
|
|
||||||
let height = CGFloat(cgImage.height)
|
|
||||||
let cropRect = CGRect(
|
|
||||||
x: rect.origin.x * width,
|
|
||||||
y: (1 - rect.origin.y - rect.height) * height, // Flip Y coordinate since NSImage uses bottom-left origin
|
|
||||||
width: rect.width * width,
|
|
||||||
height: rect.height * height
|
|
||||||
)
|
|
||||||
|
|
||||||
guard let croppedCGImage = cgImage.cropping(to: cropRect) else {
|
// Detect board
|
||||||
return nil
|
guard let boardRect = boardDetector.detectBoard(in: ciImage) else {
|
||||||
|
throw CaptureError.boardDetectionFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
return NSImage(cgImage: croppedCGImage, size: cropRect.size)
|
// Update detected rectangle
|
||||||
|
self.detectedBoardRect = boardRect
|
||||||
|
|
||||||
|
// Crop board image
|
||||||
|
let croppedImage = ciImage.cropped(to: boardRect)
|
||||||
|
updateImages(ciImage: ciImage, croppedImage: croppedImage)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func updateImages(ciImage: CIImage, croppedImage: CIImage) {
|
||||||
|
if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) {
|
||||||
|
let capturedNSImage = NSImage(cgImage: cgImage, size: .zero)
|
||||||
|
self.capturedImage = capturedNSImage
|
||||||
|
}
|
||||||
|
|
||||||
|
if let cgCroppedImage = context.createCGImage(croppedImage, from: croppedImage.extent) {
|
||||||
|
let croppedNSImage = NSImage(cgImage: cgCroppedImage, size: .zero)
|
||||||
|
self.croppedBoardImage = croppedNSImage
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func handleCaptureError(_ error: Error) {
|
||||||
|
if let captureError = error as? CaptureError {
|
||||||
|
self.captureError = captureError
|
||||||
|
} else {
|
||||||
|
self.captureError = .boardDetectionFailed
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
571
cline_docs/Info.txt
Normal file
571
cline_docs/Info.txt
Normal file
|
|
@ -0,0 +1,571 @@
|
||||||
|
|
||||||
|
Command queue creation failed. Worst processes (
|
||||||
|
{
|
||||||
|
Count = 2017;
|
||||||
|
Process = "pid 25345, ChessPrism";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 4;
|
||||||
|
Process = "pid 1227, firefox";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 579, WindowServer";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 1926, Creative Cloud U";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1754, Ollama Helper (G";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1928, LogiTune Helper ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 11425, Xcode";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1062, mediaanalysisd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1101, NotificationCent";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 582, loginwindow";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1242, Finder";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1100, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1237, Notes";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1222, Terminal";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1867, Docker Desktop H";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1970, Logi AI Prompt B";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1040, ControlCenter";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 905, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 7637, ControlCenterHel";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 15435, Code Helper (GPU";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1208, avconferenced";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1228, VTDecoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1248, Spotlight";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1186, replayd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1459, naturallanguaged";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 577, com.apple.cmio.r";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 2216, VTEncoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 7667, QuickLookUIServi";
|
||||||
|
}
|
||||||
|
)
|
||||||
|
Command queue creation failed. Worst processes (
|
||||||
|
{
|
||||||
|
Count = 2017;
|
||||||
|
Process = "pid 25345, ChessPrism";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 4;
|
||||||
|
Process = "pid 1227, firefox";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 579, WindowServer";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 1926, Creative Cloud U";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1754, Ollama Helper (G";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1928, LogiTune Helper ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 11425, Xcode";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1062, mediaanalysisd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1101, NotificationCent";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 582, loginwindow";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1242, Finder";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1100, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1237, Notes";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1222, Terminal";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1867, Docker Desktop H";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1970, Logi AI Prompt B";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1040, ControlCenter";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 905, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 7637, ControlCenterHel";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 15435, Code Helper (GPU";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1208, avconferenced";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1228, VTDecoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1248, Spotlight";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1186, replayd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1459, naturallanguaged";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 577, com.apple.cmio.r";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 2216, VTEncoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 7667, QuickLookUIServi";
|
||||||
|
}
|
||||||
|
)
|
||||||
|
Command queue creation failed. Worst processes (
|
||||||
|
{
|
||||||
|
Count = 2017;
|
||||||
|
Process = "pid 25345, ChessPrism";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 4;
|
||||||
|
Process = "pid 1227, firefox";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 579, WindowServer";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 1926, Creative Cloud U";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1754, Ollama Helper (G";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1928, LogiTune Helper ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 11425, Xcode";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1062, mediaanalysisd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1101, NotificationCent";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 582, loginwindow";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1242, Finder";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1100, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1237, Notes";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1222, Terminal";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1867, Docker Desktop H";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1970, Logi AI Prompt B";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1040, ControlCenter";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 905, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 7637, ControlCenterHel";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 15435, Code Helper (GPU";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1208, avconferenced";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1228, VTDecoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1248, Spotlight";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1186, replayd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1459, naturallanguaged";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 577, com.apple.cmio.r";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 2216, VTEncoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 7667, QuickLookUIServi";
|
||||||
|
}
|
||||||
|
)
|
||||||
|
Command queue creation failed. Worst processes (
|
||||||
|
{
|
||||||
|
Count = 2017;
|
||||||
|
Process = "pid 25345, ChessPrism";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 4;
|
||||||
|
Process = "pid 1227, firefox";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 579, WindowServer";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 1926, Creative Cloud U";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1754, Ollama Helper (G";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1928, LogiTune Helper ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 11425, Xcode";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1062, mediaanalysisd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1101, NotificationCent";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 582, loginwindow";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1242, Finder";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1100, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1237, Notes";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1222, Terminal";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1867, Docker Desktop H";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1970, Logi AI Prompt B";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1040, ControlCenter";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 905, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 7637, ControlCenterHel";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 15435, Code Helper (GPU";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1208, avconferenced";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1228, VTDecoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1248, Spotlight";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1186, replayd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1459, naturallanguaged";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 577, com.apple.cmio.r";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 2216, VTEncoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 7667, QuickLookUIServi";
|
||||||
|
}
|
||||||
|
)
|
||||||
|
Command queue creation failed. Worst processes (
|
||||||
|
{
|
||||||
|
Count = 2017;
|
||||||
|
Process = "pid 25345, ChessPrism";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 4;
|
||||||
|
Process = "pid 1227, firefox";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 579, WindowServer";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 3;
|
||||||
|
Process = "pid 1926, Creative Cloud U";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1754, Ollama Helper (G";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 1928, LogiTune Helper ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 2;
|
||||||
|
Process = "pid 11425, Xcode";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1062, mediaanalysisd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1101, NotificationCent";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 582, loginwindow";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1242, Finder";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1100, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1237, Notes";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1222, Terminal";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1867, Docker Desktop H";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1970, Logi AI Prompt B";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 1040, ControlCenter";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 905, iconservicesagen";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 7637, ControlCenterHel";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 1;
|
||||||
|
Process = "pid 15435, Code Helper (GPU";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1208, avconferenced";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1228, VTDecoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1248, Spotlight";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1186, replayd";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 1459, naturallanguaged";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 577, com.apple.cmio.r";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 2216, VTEncoderXPCServ";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Count = 0;
|
||||||
|
Process = "pid 7667, QuickLookUIServi";
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
@ -1,50 +1,74 @@
|
||||||
# Active Context
|
# Active Context
|
||||||
|
|
||||||
## Current Status
|
## Current Status
|
||||||
- Screen capture module implemented
|
- Screen capture module successfully implemented
|
||||||
- Pattern recognition-based board detection implemented
|
- Window detection and capture working correctly
|
||||||
- Coordinate-based detection retained as fallback
|
- SwiftUI interface with capture controls functioning
|
||||||
- SwiftUI interface for capture controls added
|
- Error handling system properly managing states
|
||||||
- Proper resource cleanup implemented
|
- Resource cleanup implemented
|
||||||
- Error handling system in place
|
- Metal resource management optimized
|
||||||
|
|
||||||
## Recent Changes
|
## Recent Changes
|
||||||
- Refined pattern recognition approach for chess board detection:
|
1. Resource Management Optimization:
|
||||||
1. Modified rectangle detection parameters:
|
- Implemented shared CIContext pattern:
|
||||||
- Using 0.3-0.5 aspect ratio for taller rectangles
|
* Prevents command queue exhaustion
|
||||||
- Increased minimum size to 0.4
|
* Reduces Metal resource usage
|
||||||
- Single observation for precision
|
* Enables long-running captures
|
||||||
2. Improved board extraction:
|
- Proper cleanup on task completion
|
||||||
- Using detected width as reference
|
- Efficient resource utilization
|
||||||
- Positioning square board at top of detected area
|
|
||||||
- Using offsetBy for vertical positioning
|
|
||||||
3. Simplified coordinate handling:
|
|
||||||
- Direct rectangle detection with Vision framework
|
|
||||||
- Proper coordinate system transformations
|
|
||||||
- Maintained pattern matching for accuracy
|
|
||||||
|
|
||||||
## Current Challenges
|
2. Improved Window Detection:
|
||||||
- Board detection still not capturing full height
|
- Using SCShareableContent for window access
|
||||||
- Need to investigate if issue is with:
|
- Precise window identification:
|
||||||
1. Detection parameters
|
* Exact bundle ID matching (com.chess.iphone)
|
||||||
2. Coordinate transformations
|
* Window visibility verification (isOnScreen)
|
||||||
3. View rendering constraints
|
* Size validation (width > 100 && height > 100)
|
||||||
4. Window/display configuration
|
|
||||||
|
3. Enhanced Capture System:
|
||||||
|
- Continuous capture implementation:
|
||||||
|
* Single persistent capture stream
|
||||||
|
* Smooth frame processing (no flickering)
|
||||||
|
* Efficient resource usage
|
||||||
|
- Proper frame dimensions from window
|
||||||
|
- Clean task cancellation handling
|
||||||
|
- Main thread safety for UI updates
|
||||||
|
|
||||||
|
4. UI Simplification:
|
||||||
|
- Removed debug overlay functionality
|
||||||
|
- Cleaner, focused interface
|
||||||
|
- Essential controls only:
|
||||||
|
* Start/Stop capture
|
||||||
|
* Full capture view
|
||||||
|
* Board preview
|
||||||
|
|
||||||
|
5. Error Handling:
|
||||||
|
- Improved error resilience:
|
||||||
|
* Continues capturing even if board detection fails
|
||||||
|
* Only stops on critical errors (e.g., window not found)
|
||||||
|
* Shows error state without interrupting capture
|
||||||
|
- Clear error states
|
||||||
|
- Proper async/await usage
|
||||||
|
- Task cancellation management
|
||||||
|
- Thread-safe state updates
|
||||||
|
|
||||||
|
## Current Focus
|
||||||
|
1. Board Detection:
|
||||||
|
- Implement chess board recognition
|
||||||
|
- Handle different board themes
|
||||||
|
- Process captured frames efficiently
|
||||||
|
|
||||||
## Next Steps
|
## Next Steps
|
||||||
1. Further refine board detection:
|
1. Implement board detection:
|
||||||
- Test different aspect ratios
|
- Pattern recognition for chess pieces
|
||||||
- Validate coordinate transformations
|
- Board coordinate mapping
|
||||||
- Review view constraints
|
- Position validation
|
||||||
2. Add board position analysis
|
2. Add position analysis
|
||||||
3. Implement move detection
|
3. Create move detection system
|
||||||
4. Create visual overlay system
|
4. Implement visual overlay
|
||||||
5. Integrate Stockfish engine
|
5. Integrate Stockfish engine
|
||||||
|
|
||||||
## Known Issues
|
## Known Issues
|
||||||
- Board detection showing only bottom portion
|
- Board detection not yet implemented
|
||||||
- Need to validate pattern detection accuracy
|
- Need to handle different chess.com themes
|
||||||
- Need to handle multiple display configurations
|
- Need to implement piece recognition
|
||||||
- Requires testing with different screen resolutions
|
- Position analysis pending implementation
|
||||||
- Need to add proper error recovery
|
|
||||||
- Requires additional permission handling for production
|
|
||||||
|
|
|
||||||
189
cline_docs/error
189
cline_docs/error
|
|
@ -1,53 +1,138 @@
|
||||||
Starting screen capture...
|
# Error Resolution Log
|
||||||
Available content: 1 displays, 32 windows
|
|
||||||
AddInstanceForFactory: No factory registered for id <CFUUID 0x6000011a9580> F8BB1C28-BAE8-11D6-9C31-00039315CD46
|
## Fixed Issues (2023)
|
||||||
Screen capture started successfully
|
|
||||||
*** Terminating app due to uncaught exception 'NSUnknownKeyException', reason: '[<NSNotificationCenter 0x6000013bc060> valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.'
|
### ScreenCaptureKit API Updates
|
||||||
*** First throw call stack:
|
1. SCContentFilter Initialization
|
||||||
(
|
- Fixed by using correct initializer and parameters:
|
||||||
0 CoreFoundation 0x0000000189d0ee80 __exceptionPreprocess + 176
|
```swift
|
||||||
1 libobjc.A.dylib 0x00000001897f6cd8 objc_exception_throw + 88
|
SCContentFilter(display: display, excludingWindows: [])
|
||||||
2 CoreFoundation 0x0000000189d33534 _CFBundleGetValueForInfoKey + 0
|
```
|
||||||
3 Foundation 0x000000018b6dbd84 -[NSObject(NSKeyValueCoding) valueForUndefinedKey:] + 196
|
- Using SCDisplay object directly (not displayID)
|
||||||
4 Foundation 0x000000018ae402fc -[NSObject(NSKeyValueCoding) valueForKey:] + 280
|
- Correct parameter name: excludingWindows
|
||||||
5 ChessPrism.debug.dylib 0x0000000100b8f834 $s10ChessPrism13ScreenCaptureC20handleBoardDetection33_6963F6DFCAB5A0A312B3D7446D299968LL7requestySo9VNRequestC_tF + 740
|
- Removed incorrect parameters (includingWindows/exceptingWindows)
|
||||||
6 ChessPrism.debug.dylib 0x0000000100b8e644 $s10ChessPrism13ScreenCaptureC12processFrame33_6963F6DFCAB5A0A312B3D7446D299968LL12sampleBufferySo08CMSampleO3Refa_tFySo9VNRequestC_s5Error_pSgtcfU0_ + 200
|
|
||||||
7 ChessPrism.debug.dylib 0x0000000100b93350 $sSo9VNRequestCs5Error_pSgIeggg_ABSo7NSErrorCSgIeyByy_TR + 140
|
2. Stream Output Type
|
||||||
8 Vision 0x000000019f40aab4 -[VNRequest performInContext:error:] + 1244
|
- Fixed type inference issue by explicit declaration:
|
||||||
9 Vision 0x000000019f40af08 __73-[VNRequest performInContextAsync:asyncDispatchQueue:asyncDispatchGroup:]_block_invoke + 128
|
```swift
|
||||||
10 libdispatch.dylib 0x00000001001d2ac8 _dispatch_block_async_invoke2 + 148
|
let outputType: SCStreamOutputType = .screen
|
||||||
11 libdispatch.dylib 0x00000001001be824 _dispatch_client_callout + 20
|
try stream.addStreamOutput(self, type: outputType, ...)
|
||||||
12 libdispatch.dylib 0x00000001001c2350 _dispatch_continuation_pop + 1408
|
```
|
||||||
13 libdispatch.dylib 0x00000001001c1028 _dispatch_async_redirect_invoke + 616
|
- Ensures proper type resolution for .screen member
|
||||||
14 libdispatch.dylib 0x00000001001d6b70 _dispatch_root_queue_drain + 404
|
|
||||||
15 libdispatch.dylib 0x00000001001d777c _dispatch_worker_thread2 + 188
|
### Required Imports
|
||||||
16 libsystem_pthread.dylib 0x0000000100b090c4 _pthread_wqthread + 228
|
- Added necessary framework imports:
|
||||||
17 libsystem_pthread.dylib 0x0000000100b10cf0 start_wqthread + 8
|
* CoreMedia
|
||||||
)
|
* AVFoundation
|
||||||
An uncaught exception was raised
|
* ScreenCaptureKit
|
||||||
[<NSNotificationCenter 0x6000013bc060> valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.
|
* CoreGraphics
|
||||||
(
|
* AppKit
|
||||||
0 CoreFoundation 0x0000000189d0ee80 __exceptionPreprocess + 176
|
* Foundation
|
||||||
1 libobjc.A.dylib 0x00000001897f6cd8 objc_exception_throw + 88
|
|
||||||
2 CoreFoundation 0x0000000189d33534 _CFBundleGetValueForInfoKey + 0
|
### Window Capture Strategy
|
||||||
3 Foundation 0x000000018b6dbd84 -[NSObject(NSKeyValueCoding) valueForUndefinedKey:] + 196
|
1. Window Detection
|
||||||
4 Foundation 0x000000018ae402fc -[NSObject(NSKeyValueCoding) valueForKey:] + 280
|
- Precise window identification:
|
||||||
5 ChessPrism.debug.dylib 0x0000000100b8f834 $s10ChessPrism13ScreenCaptureC20handleBoardDetection33_6963F6DFCAB5A0A312B3D7446D299968LL7requestySo9VNRequestC_tF + 740
|
```swift
|
||||||
6 ChessPrism.debug.dylib 0x0000000100b8e644 $s10ChessPrism13ScreenCaptureC12processFrame33_6963F6DFCAB5A0A312B3D7446D299968LL12sampleBufferySo08CMSampleO3Refa_tFySo9VNRequestC_s5Error_pSgtcfU0_ + 200
|
let bundleID = window.owningApplication?.bundleIdentifier ?? ""
|
||||||
7 ChessPrism.debug.dylib 0x0000000100b93350 $sSo9VNRequestCs5Error_pSgIeggg_ABSo7NSErrorCSgIeyByy_TR + 140
|
let isChessApp = bundleID == "com.chess.iphone"
|
||||||
8 Vision 0x000000019f40aab4 -[VNRequest performInContext:error:] + 1244
|
let hasValidSize = window.frame.width > 100 && window.frame.height > 100
|
||||||
9 Vision 0x000000019f40af08 __73-[VNRequest performInContextAsync:asyncDispatchQueue:asyncDispatchGroup:]_block_invoke + 128
|
return isChessApp && window.isOnScreen && hasValidSize
|
||||||
10 libdispatch.dylib 0x00000001001d2ac8 _dispatch_block_async_invoke2 + 148
|
```
|
||||||
11 libdispatch.dylib 0x00000001001be824 _dispatch_client_callout + 20
|
- Multiple validation checks:
|
||||||
12 libdispatch.dylib 0x00000001001c2350 _dispatch_continuation_pop + 1408
|
* Exact bundle ID match
|
||||||
13 libdispatch.dylib 0x00000001001c1028 _dispatch_async_redirect_invoke + 616
|
* Window is currently on screen
|
||||||
14 libdispatch.dylib 0x00000001001d6b70 _dispatch_root_queue_drain + 404
|
* Window has valid dimensions
|
||||||
15 libdispatch.dylib 0x00000001001d777c _dispatch_worker_thread2 + 188
|
- Handles iOS apps running on Mac properly
|
||||||
16 libsystem_pthread.dylib 0x0000000100b090c4 _pthread_wqthread + 228
|
|
||||||
17 libsystem_pthread.dylib 0x0000000100b10cf0 start_wqthread + 8
|
2. Capture Method
|
||||||
)
|
- Implemented continuous capture:
|
||||||
FAULT: NSUnknownKeyException: [<NSNotificationCenter 0x6000013bc060> valueForUndefinedKey:]: this class is not key value coding-compliant for the key lastDetectedCoordinates.; {
|
```swift
|
||||||
NSTargetObjectUserInfoKey = "<CFNotificationCenter 0x6000011a4900 [0x1f4722240]>";
|
// Start once
|
||||||
NSUnknownUserInfoKey = lastDetectedCoordinates;
|
try await screenCapture.startCapture()
|
||||||
|
|
||||||
|
// Process frames continuously
|
||||||
|
while !Task.isCancelled {
|
||||||
|
if let image = screenCapture.getCurrentImage() {
|
||||||
|
try await processImage(image)
|
||||||
}
|
}
|
||||||
libc++abi: terminating due to uncaught exception of type NSException
|
try await Task.sleep(nanoseconds: 100_000_000)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- Maintains single active stream
|
||||||
|
- Eliminates capture flickering
|
||||||
|
- Proper cleanup on stop
|
||||||
|
|
||||||
|
### Thread Safety and Async Handling
|
||||||
|
1. Main Actor Isolation
|
||||||
|
- Added @MainActor to ViewModel class:
|
||||||
|
```swift
|
||||||
|
@MainActor
|
||||||
|
class ScreenCaptureViewModel: ObservableObject
|
||||||
|
```
|
||||||
|
- Ensures all @Published property updates happen on main thread
|
||||||
|
- Proper thread safety for SwiftUI bindings
|
||||||
|
|
||||||
|
2. Async Operation Handling
|
||||||
|
- Optimized async/await usage:
|
||||||
|
```swift
|
||||||
|
// Only use await for truly async operations
|
||||||
|
try await screenCapture.startCapture()
|
||||||
|
try await Task.sleep(nanoseconds: 100_000_000)
|
||||||
|
```
|
||||||
|
- Removed unnecessary await keywords:
|
||||||
|
* Non-async error handling
|
||||||
|
* UI state updates
|
||||||
|
* Image processing
|
||||||
|
- Proper @MainActor usage for thread safety
|
||||||
|
|
||||||
|
3. Task and Error Handling
|
||||||
|
- Improved task cancellation:
|
||||||
|
```swift
|
||||||
|
captureLoop: while !Task.isCancelled {
|
||||||
|
do {
|
||||||
|
// Process frame
|
||||||
|
} catch is CancellationError {
|
||||||
|
break captureLoop
|
||||||
|
} catch {
|
||||||
|
// Continue capturing on non-critical errors
|
||||||
|
await handleCaptureError(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- Proper error recovery:
|
||||||
|
* Continues on board detection failures
|
||||||
|
* Shows error state without stopping
|
||||||
|
* Clears errors on successful detection
|
||||||
|
- Clean error state on stop:
|
||||||
|
```swift
|
||||||
|
captureError = nil // Clear any error when stopping
|
||||||
|
```
|
||||||
|
- Proper cleanup on task exit
|
||||||
|
|
||||||
|
## Technical Notes
|
||||||
|
- Resource Management:
|
||||||
|
* Shared CIContext to prevent command queue exhaustion:
|
||||||
|
```swift
|
||||||
|
private static let shared = CIContext()
|
||||||
|
private var context: CIContext { Self.shared }
|
||||||
|
```
|
||||||
|
* Prevents "Command queue creation failed" errors
|
||||||
|
* Reduces Metal resource usage
|
||||||
|
* Proper cleanup on task completion
|
||||||
|
|
||||||
|
- Using correct SCContentFilter API with proper parameter names
|
||||||
|
- Proper type safety throughout the implementation:
|
||||||
|
* Proper Vision framework result handling:
|
||||||
|
```swift
|
||||||
|
guard let observations = request.results,
|
||||||
|
!observations.isEmpty else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let bestObservation = observations[0]
|
||||||
|
```
|
||||||
|
* No unnecessary type casting
|
||||||
|
* Safe array access
|
||||||
|
* Proper optional handling
|
||||||
|
- Appropriate framework dependencies
|
||||||
|
- Follows current ScreenCaptureKit best practices
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,94 @@
|
||||||
# System Patterns
|
# System Patterns
|
||||||
|
|
||||||
|
## Window Capture Architecture
|
||||||
|
|
||||||
|
### Window Detection Pattern
|
||||||
|
1. SCShareableContent Access
|
||||||
|
- Async/await pattern for content access
|
||||||
|
- Proper error propagation
|
||||||
|
- Permission handling
|
||||||
|
|
||||||
|
2. Window Identification
|
||||||
|
- Multiple validation criteria:
|
||||||
|
```swift
|
||||||
|
let bundleID = window.owningApplication?.bundleIdentifier ?? ""
|
||||||
|
let isChessApp = bundleID == "com.chess.iphone"
|
||||||
|
let hasValidSize = window.frame.width > 100 && window.frame.height > 100
|
||||||
|
return isChessApp && window.isOnScreen && hasValidSize
|
||||||
|
```
|
||||||
|
- Fail-fast approach with guard statements
|
||||||
|
- Clear error states
|
||||||
|
|
||||||
|
### Capture System Pattern
|
||||||
|
1. Stream Configuration
|
||||||
|
- Window-specific capture setup
|
||||||
|
- Frame dimension matching
|
||||||
|
- Proper delegate handling
|
||||||
|
|
||||||
|
2. Frame Processing
|
||||||
|
- Main thread safety for UI updates
|
||||||
|
- Efficient image conversion pipeline
|
||||||
|
- Resource cleanup
|
||||||
|
|
||||||
|
### Error Handling Pattern
|
||||||
|
1. Task Management
|
||||||
|
- Proper cancellation points
|
||||||
|
- Clean state management
|
||||||
|
- Resource cleanup
|
||||||
|
|
||||||
|
2. Error States
|
||||||
|
- Clear error types
|
||||||
|
- User-friendly messages
|
||||||
|
- State recovery
|
||||||
|
|
||||||
|
## UI Architecture
|
||||||
|
|
||||||
|
### MVVM Implementation
|
||||||
|
1. ViewModel
|
||||||
|
- @MainActor for thread safety
|
||||||
|
- Published properties for state
|
||||||
|
- Clear separation of concerns
|
||||||
|
|
||||||
|
2. View Layer
|
||||||
|
- SwiftUI declarative UI
|
||||||
|
- State-driven updates
|
||||||
|
- Error presentation
|
||||||
|
|
||||||
|
### Async Operations
|
||||||
|
1. Task Management
|
||||||
|
- Structured concurrency
|
||||||
|
- Proper cancellation
|
||||||
|
- State synchronization
|
||||||
|
|
||||||
|
2. State Updates
|
||||||
|
- Main thread safety
|
||||||
|
- Clear state transitions
|
||||||
|
- Error recovery
|
||||||
|
|
||||||
## Core Architecture
|
## Core Architecture
|
||||||
|
|
||||||
|
### Resource Management Patterns
|
||||||
|
1. Shared CIContext Pattern
|
||||||
|
- Static shared instance:
|
||||||
|
```swift
|
||||||
|
private static let shared = CIContext()
|
||||||
|
private var context: CIContext { Self.shared }
|
||||||
|
```
|
||||||
|
- Benefits:
|
||||||
|
* Prevents Metal command queue exhaustion
|
||||||
|
* Reduces resource overhead
|
||||||
|
* Enables long-running captures
|
||||||
|
- Implementation:
|
||||||
|
* Used in BoardDetector and ViewModel
|
||||||
|
* Proper cleanup on task completion
|
||||||
|
* Thread-safe access
|
||||||
|
|
||||||
### Screen Capture System
|
### Screen Capture System
|
||||||
- Uses ScreenCaptureKit for efficient screen capture
|
- Uses ScreenCaptureKit for efficient screen capture
|
||||||
- Implements SCStreamOutput protocol for frame processing
|
- Implements SCStreamOutput protocol for frame processing
|
||||||
- Handles capture session lifecycle and cleanup
|
- Handles capture session lifecycle and cleanup
|
||||||
- Manages permissions and error handling
|
- Manages permissions and error handling
|
||||||
|
- Optimized resource usage
|
||||||
|
|
||||||
### Board Detection System
|
### Board Detection System
|
||||||
Two implemented approaches:
|
Two implemented approaches:
|
||||||
|
|
|
||||||
|
|
@ -8,21 +8,38 @@
|
||||||
|
|
||||||
## Core Technologies
|
## Core Technologies
|
||||||
|
|
||||||
|
### Metal Resource Management
|
||||||
|
- Shared CIContext pattern:
|
||||||
|
* Static shared instance to prevent command queue exhaustion
|
||||||
|
* Used across BoardDetector and ViewModel
|
||||||
|
* Proper cleanup and resource management
|
||||||
|
- Performance considerations:
|
||||||
|
* Reduced Metal command queue creation
|
||||||
|
* Efficient resource utilization
|
||||||
|
* Support for long-running captures
|
||||||
|
|
||||||
### ScreenCaptureKit
|
### ScreenCaptureKit
|
||||||
- System framework for screen capture
|
- System framework for screen capture
|
||||||
- Requires user permissions
|
- Implemented features:
|
||||||
- Supports window/display filtering
|
* Window detection using SCShareableContent
|
||||||
- Real-time frame capture capabilities
|
* iOS app window capture support
|
||||||
|
* Real-time frame capture
|
||||||
### Vision Framework
|
* Proper error handling
|
||||||
- Used for board and coordinate detection
|
|
||||||
- Key components:
|
- Key components:
|
||||||
|
* SCShareableContent: Window and display access
|
||||||
|
* SCContentFilter: Window-specific capture
|
||||||
|
* SCStream: Frame capture management
|
||||||
|
* SCStreamOutput: Frame processing
|
||||||
|
|
||||||
|
### Vision Framework (Planned)
|
||||||
|
- Will be used for board and coordinate detection
|
||||||
|
- Key components to implement:
|
||||||
* VNRecognizeTextRequest: Chess coordinate detection
|
* VNRecognizeTextRequest: Chess coordinate detection
|
||||||
* VNDetectRectanglesRequest: Board boundary detection
|
* VNDetectRectanglesRequest: Board boundary detection
|
||||||
- Configuration:
|
- Planned configuration:
|
||||||
* Text recognition level: accurate
|
* Text recognition level: accurate
|
||||||
* Language correction: disabled
|
* Language correction: disabled
|
||||||
* Rectangle aspect ratio: 0.3-0.5 (for taller rectangles)
|
* Rectangle aspect ratio: 0.3-0.5
|
||||||
* Minimum size: 0.4
|
* Minimum size: 0.4
|
||||||
* Maximum observations: 1
|
* Maximum observations: 1
|
||||||
|
|
||||||
|
|
@ -58,36 +75,40 @@
|
||||||
|
|
||||||
## Technical Constraints
|
## Technical Constraints
|
||||||
|
|
||||||
### Board Detection
|
### Window Capture System
|
||||||
1. Pattern Recognition
|
1. Window Detection
|
||||||
- Detect taller rectangles (0.3-0.5 aspect ratio)
|
- Using SCShareableContent for window access
|
||||||
- Extract square board from upper portion
|
- Multiple validation criteria:
|
||||||
- Use width as reference measurement
|
* Bundle ID verification
|
||||||
- Handle coordinate system transformations
|
* Window visibility check
|
||||||
|
* Size validation
|
||||||
|
- Error handling for missing windows
|
||||||
|
|
||||||
2. Coordinate Recognition
|
2. Frame Capture
|
||||||
- Must detect a-h and 1-8 coordinates
|
- Window-specific capture configuration
|
||||||
- Handles both light and dark themes
|
- Frame dimension matching
|
||||||
- Requires clear coordinate visibility
|
- Proper delegate handling
|
||||||
- Minimum text size requirements
|
- Resource cleanup
|
||||||
|
|
||||||
3. Board Boundaries
|
3. Performance
|
||||||
- Square aspect ratio (1:1)
|
- Main thread safety for UI updates
|
||||||
- Tolerance: 0.3 for dimensions
|
- Efficient image conversion
|
||||||
- Extract from detected area
|
- Proper task cancellation
|
||||||
- Proper coordinate transformations
|
- Memory management
|
||||||
|
- Shared CIContext for Metal efficiency
|
||||||
|
|
||||||
4. Performance
|
4. Error Handling
|
||||||
- Frame processing on dedicated queue
|
- Clear error types
|
||||||
- Asynchronous Vision requests
|
- User-friendly messages
|
||||||
- Memory management for capture session
|
- State recovery
|
||||||
- Resource cleanup requirements
|
- Resource cleanup
|
||||||
|
|
||||||
### System Requirements
|
### System Requirements
|
||||||
- macOS 12.0 or later
|
- macOS 12.0 or later
|
||||||
- Screen Capture permissions
|
- Screen Capture permissions
|
||||||
- Sufficient CPU for real-time processing
|
- Sufficient CPU for real-time processing
|
||||||
- Adequate memory for frame buffering
|
- Adequate memory for frame buffering
|
||||||
|
- Metal-capable GPU for image processing
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
|
|
@ -102,6 +123,7 @@
|
||||||
- Vision.framework
|
- Vision.framework
|
||||||
- SwiftUI.framework
|
- SwiftUI.framework
|
||||||
- CoreImage.framework
|
- CoreImage.framework
|
||||||
|
- Metal.framework (via CIContext)
|
||||||
|
|
||||||
## Development Guidelines
|
## Development Guidelines
|
||||||
|
|
||||||
|
|
@ -110,8 +132,10 @@
|
||||||
- Protocol-oriented design
|
- Protocol-oriented design
|
||||||
- Clear separation of concerns
|
- Clear separation of concerns
|
||||||
- Comprehensive error handling
|
- Comprehensive error handling
|
||||||
|
- Resource sharing patterns
|
||||||
|
|
||||||
### Performance Optimization
|
### Performance Optimization
|
||||||
|
- Shared CIContext for Metal efficiency
|
||||||
- Efficient frame processing
|
- Efficient frame processing
|
||||||
- Memory management
|
- Memory management
|
||||||
- Resource cleanup
|
- Resource cleanup
|
||||||
|
|
@ -130,16 +154,19 @@
|
||||||
- Coordinate transformations
|
- Coordinate transformations
|
||||||
- Error handling
|
- Error handling
|
||||||
- State management
|
- State management
|
||||||
|
- Resource management
|
||||||
|
|
||||||
### Integration Tests
|
### Integration Tests
|
||||||
- End-to-end workflows
|
- End-to-end workflows
|
||||||
- Component interaction
|
- Component interaction
|
||||||
- Event propagation
|
- Event propagation
|
||||||
|
- Resource sharing
|
||||||
|
|
||||||
### UI Tests
|
### UI Tests
|
||||||
- User interaction flows
|
- User interaction flows
|
||||||
- Error state handling
|
- Error state handling
|
||||||
- Visual feedback
|
- Visual feedback
|
||||||
|
- Performance monitoring
|
||||||
|
|
||||||
## Documentation Requirements
|
## Documentation Requirements
|
||||||
|
|
||||||
|
|
@ -148,27 +175,29 @@
|
||||||
- Parameter descriptions
|
- Parameter descriptions
|
||||||
- Return value documentation
|
- Return value documentation
|
||||||
- Error documentation
|
- Error documentation
|
||||||
|
- Resource usage documentation
|
||||||
|
|
||||||
### Architecture Documentation
|
### Architecture Documentation
|
||||||
- System overview
|
- System overview
|
||||||
- Component interaction
|
- Component interaction
|
||||||
- Data flow diagrams
|
- Data flow diagrams
|
||||||
- State management
|
- State management
|
||||||
|
- Resource management patterns
|
||||||
|
|
||||||
## Current Challenges
|
## Current Challenges
|
||||||
|
|
||||||
### Coordinate Systems
|
### Resource Management
|
||||||
1. Understanding
|
1. Metal Efficiency
|
||||||
- Different origin points
|
- Command queue management
|
||||||
- Axis directions
|
- Shared context patterns
|
||||||
- Coordinate spaces
|
- Resource cleanup
|
||||||
- Transformation requirements
|
- Performance monitoring
|
||||||
|
|
||||||
2. Implementation
|
2. Memory Usage
|
||||||
- Proper transformations
|
- Frame buffer management
|
||||||
- Consistent handling
|
- Image processing optimization
|
||||||
- Validation methods
|
- Resource pooling
|
||||||
- Error checking
|
- Cleanup strategies
|
||||||
|
|
||||||
### Board Detection
|
### Board Detection
|
||||||
1. Full Capture
|
1. Full Capture
|
||||||
|
|
@ -206,3 +235,4 @@
|
||||||
- Optimize frame processing
|
- Optimize frame processing
|
||||||
- Improve error recovery
|
- Improve error recovery
|
||||||
- Enhanced permission handling
|
- Enhanced permission handling
|
||||||
|
- Resource usage monitoring
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue