chaulmark/ChessPrism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Implement auto-capture and cursor-free snapshots

Browse source 
- Add automatic capture start/stop based on board detection
- Implement cursor-free snapshot system:
  * Add SCStreamConfiguration cursor control
  * Add temporary capture session management
  * Ensure clean snapshots without cursor artifacts
- Add visual feedback:
  * Status indicator (green/yellow/gray)
  * Snapshot preview below board
  * Clear capture state indication
- Update documentation:
  * Add snapshot system patterns
  * Document cursor control implementation
  * Update technical constraints
This commit is contained in:
TheMaddax 2025-01-07 09:44:37 -06:00
parent 02d1670b09
commit 4b8935afd1
7 changed files with 254 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
ChessPrism/ChessPrism/ContentView.swift
View file

@ -89,11 +89,26 @@ struct ContentView: View {
var body: some View {
VStack {
// Capture status indicator
CaptureStatusButton(
isCapturing: viewModel.isCapturing,
isBoardDetected: viewModel.isBoardDetected
)
// Status and Scan controls
HStack {
CaptureStatusButton(
isCapturing: viewModel.isCapturing,
isBoardDetected: viewModel.isBoardDetected
)
Button(action: {
Task {
await viewModel.takeSnapshot()
}
}) {
Text("Scan")
.foregroundColor(.white)
.padding(.horizontal, 20)
.padding(.vertical, 10)
}
.buttonStyle(.borderedProminent)
.disabled(!viewModel.isCapturing || !viewModel.isBoardDetected)
}
.onAppear {
// Start monitoring for chess boards when view appears
viewModel.startMonitoring()
@ -142,6 +157,22 @@ struct ContentView: View {
Text("No board detected")
.foregroundColor(.gray)
}
if viewModel.snapshotTaken {
Divider()
.padding(.vertical)
Text("Latest Snapshot")
if let snapshot = viewModel.latestSnapshot {
Image(nsImage: snapshot)
.resizable()
.aspectRatio(contentMode: .fit)
.frame(maxWidth: 400)
.padding()
.background(Color.black.opacity(0.1))
.cornerRadius(8)
}
}
}
}
.padding()

21
ChessPrism/ChessPrism/ScreenCapture.swift
View file

@ -23,8 +23,16 @@ class ScreenCapture: NSObject, SCStreamOutput {
private var activeStream: SCStream?
private var lastCapturedImage: NSImage?
private var excludeCursor: Bool = false
func startCapture(excludeCursor: Bool = false) async throws {
// Always stop any existing capture before starting a new one
if activeStream != nil {
try await stopCapture()
}
self.excludeCursor = excludeCursor
func startCapture() async throws {
guard #available(macOS 12.3, *) else { return }
let content = try await SCShareableContent.current
@ -43,6 +51,9 @@ class ScreenCapture: NSObject, SCStreamOutput {
let config = SCStreamConfiguration()
config.width = Int(window.frame.width)
config.height = Int(window.frame.height)
config.minimumFrameInterval = CMTime(value: 1, timescale: 30) // 30 FPS
config.queueDepth = 5 // Buffer up to 5 frames
config.showsCursor = !excludeCursor // Set cursor visibility
let stream = SCStream(filter: filter, configuration: config, delegate: nil)
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: DispatchQueue.global(qos: .userInitiated))
@ -53,9 +64,11 @@ class ScreenCapture: NSObject, SCStreamOutput {
}
func stopCapture() async throws {
guard let stream = activeStream else { return }
try await stream.stopCapture()
activeStream = nil
if let stream = activeStream {
try await stream.stopCapture()
activeStream = nil
lastCapturedImage = nil
}
}
func getCurrentImage() -> NSImage? {

149
ChessPrism/ChessPrism/ScreenCaptureViewModel.swift
View file

@ -11,8 +11,11 @@ class ScreenCaptureViewModel: ObservableObject {
@Published var isCapturing = false
@Published var isBoardDetected = false
@Published var isAutoCapturing = true // Default to auto-capture mode
@Published var snapshotTaken = false // Track if snapshot was taken
@Published var latestSnapshot: NSImage? // Make snapshot accessible to view
private let screenCapture = ScreenCapture()
private let captureManager = ScreenCapture() // For actual capture
private let monitorManager = ScreenCapture() // For monitoring
private let boardDetector = BoardDetector()
private var captureTask: Task<Void, Never>?
private var monitorTask: Task<Void, Never>?
@ -21,11 +24,13 @@ class ScreenCaptureViewModel: ObservableObject {
private var context: CIContext { ScreenCaptureViewModel.shared }
private var detectedBoardRect: CGRect?
private var isMonitoring = false
enum CaptureError: LocalizedError {
case chessWindowNotFound
case boardDetectionFailed
case noBoardDetected
case snapshotFailed
var errorDescription: String? {
switch self {
@ -35,92 +40,135 @@ class ScreenCaptureViewModel: ObservableObject {
return "Failed to detect chess board"
case .noBoardDetected:
return "No chess board detected"
case .snapshotFailed:
return "Failed to take snapshot"
}
}
}
func takeSnapshot() async {
// Stop current capture
try? await captureManager.stopCapture()
// Start new capture without cursor
do {
try await captureManager.startCapture(excludeCursor: true)
// Wait a brief moment for the capture to stabilize
try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
if let currentImage = captureManager.getCurrentImage() {
// Process the image to get the cropped board
try await processImage(currentImage)
// Store the cropped board as snapshot
if let boardImage = croppedBoardImage {
latestSnapshot = boardImage
snapshotTaken = true
} else {
captureError = .snapshotFailed
}
} else {
captureError = .snapshotFailed
}
// Restart normal capture
try await captureManager.startCapture(excludeCursor: false)
} catch {
captureError = .snapshotFailed
// Ensure we restart normal capture even if snapshot fails
try? await captureManager.startCapture(excludeCursor: false)
}
}
func startMonitoring() {
guard monitorTask == nil else { return }
guard !isMonitoring else { return }
isMonitoring = true
monitorTask = Task {
do {
// Start continuous monitoring
try await screenCapture.startCapture()
monitorLoop: while !Task.isCancelled {
do {
// Start monitoring capture
try await monitorManager.startCapture()
monitorLoop: while !Task.isCancelled {
do {
if let image = screenCapture.getCurrentImage(),
while !Task.isCancelled {
if let image = monitorManager.getCurrentImage(),
let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) {
// Check for board
let ciImage = CIImage(cgImage: cgImage)
if boardDetector.detectBoard(in: ciImage) != nil {
// Board detected, start capture if not already capturing
if !isCapturing {
startCapture()
await startCapture()
}
}
}
try await Task.sleep(nanoseconds: 500_000_000) // 0.5 seconds
} catch is CancellationError {
break monitorLoop
} catch {
// Just log the error and continue monitoring
print("Monitor error: \(error)")
}
} catch {
print("Monitor error: \(error)")
// If there's an error, wait briefly and try again
if !Task.isCancelled {
try? await Task.sleep(nanoseconds: 1_000_000_000) // 1 second
continue monitorLoop
}
}
try? await screenCapture.stopCapture()
} catch {
print("Monitor setup error: \(error)")
}
isMonitoring = false
try? await monitorManager.stopCapture()
}
}
func stopMonitoring() {
monitorTask?.cancel()
monitorTask = nil
isMonitoring = false
stopCapture()
Task {
try? await monitorManager.stopCapture()
}
}
func startCapture() {
func startCapture() async {
guard !isCapturing else { return }
isCapturing = true
captureError = nil
snapshotTaken = false
latestSnapshot = nil
do {
try await captureManager.startCapture()
} catch {
handleCaptureError(error)
isCapturing = false
return
}
captureTask = Task {
do {
// Start continuous capture
try await screenCapture.startCapture()
captureLoop: while !Task.isCancelled {
do {
if let image = screenCapture.getCurrentImage() {
try await processImage(image)
}
try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
} catch is CancellationError {
break captureLoop
} catch {
// Just update the error state but continue capturing
if !Task.isCancelled {
handleCaptureError(error)
if isAutoCapturing && error as? CaptureError == .noBoardDetected {
// In auto-capture mode, stop capture but keep monitoring
stopCapture()
break captureLoop
}
captureLoop: while !Task.isCancelled {
do {
if let image = captureManager.getCurrentImage() {
try await processImage(image)
}
try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
} catch is CancellationError {
break captureLoop
} catch {
// Just update the error state but continue capturing
if !Task.isCancelled {
handleCaptureError(error)
if isAutoCapturing && error as? CaptureError == .noBoardDetected {
// In auto-capture mode, stop capture but keep monitoring
stopCapture()
break captureLoop
}
}
}
}
// Clean up
try? await screenCapture.stopCapture()
if isCapturing {
isCapturing = false
}
} catch {
handleCaptureError(error)
if isCapturing {
isCapturing = false
}
}
@ -132,6 +180,13 @@ class ScreenCaptureViewModel: ObservableObject {
isCapturing = false
captureError = nil
isBoardDetected = false
snapshotTaken = false
latestSnapshot = nil
// Clean up capture session
Task {
try? await captureManager.stopCapture()
}
}
private func processImage(_ image: NSImage) async throws {

68
cline_docs/activeContext.md
View file

@ -10,23 +10,43 @@
- Automatic board detection and capture implemented
- Visual capture status indicator added
- Continuous board monitoring system implemented
- Auto-capture on game start/stop working successfully
- Manual snapshot system implemented with Scan button
- Snapshot preview display added below Chessboard Preview
- Cursor-free snapshot capture implemented
## Recent Changes
1. Implemented Continuous Board Monitoring:
1. Enhanced Manual Snapshot System:
- New cursor-free capture:
* Temporarily disables cursor during snapshot
* Ensures clean board capture without mouse pointer
* Automatically restores cursor after snapshot
- Improved snapshot process:
* Stops current capture
* Takes cursor-free snapshot
* Processes board detection
* Restores normal capture
* Handles errors gracefully
- Snapshot visualization:
* Added preview area below Chessboard Preview
* Shows latest snapshot with visual feedback
* Clear indication when snapshot is taken
2. Implemented Continuous Board Monitoring:
- Added separate monitoring and capture tasks:
* Monitor constantly checks for chess boards (0.5s interval)
* Capture processes frames when active (0.1s interval)
- Auto-capture behavior:
- Auto-capture behavior working as expected:
* Starts monitoring when app launches
* Automatically starts capture when board appears
* Stops capture (but keeps monitoring) when board disappears
* Resumes capture when new board is detected
- Fixed image conversion pipeline:
* Proper NSImage → CGImage → CIImage conversion
* Efficient resource management
* Clean error handling
* Stops capture but continues monitoring when board disappears
* Successfully resumes capture when new game starts
- Performance characteristics:
* ~40% CPU usage during operation
* Stable memory management
* Responsive to game state changes
2. Enhanced Status Indication:
3. Enhanced Status Indication:
- Visual status indicator shows capture state:
* Green: Actively capturing board
* Yellow: Waiting for board
@ -34,7 +54,7 @@
- Clear error messages for different states
- Automatic status updates based on board detection
3. Resource Management Optimization:
4. Resource Management:
- Implemented shared CIContext pattern:
* Prevents command queue exhaustion
* Reduces Metal resource usage
@ -42,22 +62,13 @@
- Proper cleanup on task completion
- Efficient resource utilization
4. Improved Window Detection:
5. Improved Window Detection:
- Using SCShareableContent for window access
- Precise window identification:
* Exact bundle ID matching (com.chess.iphone)
* Window visibility verification (isOnScreen)
* Size validation (width > 100 && height > 100)
5. Enhanced Capture System:
- Continuous capture implementation:
* Single persistent capture stream
* Smooth frame processing (no flickering)
* Efficient resource usage
- Proper frame dimensions from window
- Clean task cancellation handling
- Main thread safety for UI updates
6. Error Handling:
- Improved error resilience:
* Continues monitoring even if capture stops
@ -69,16 +80,17 @@
- Thread-safe state updates
## Current Focus
1. Board Detection:
- Implement chess board recognition
- Handle different board themes
- Process captured frames efficiently
1. Board Recognition:
- Process snapshot images
- Implement piece detection
- Extract board state
## Next Steps
1. Implement board detection:
- Pattern recognition for chess pieces
- Board coordinate mapping
- Position validation
1. Implement board recognition:
- Process snapshot images
- Detect chess pieces
- Map board coordinates
- Validate positions
2. Add position analysis
3. Create move detection system
4. Implement visual overlay

24
cline_docs/productContext.md
View file

@ -15,11 +15,13 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Recognizes board coordinates and boundaries
- Handles various board themes and orientations
- Maintains accuracy during game play
- Provides clean board snapshots for analysis
### Real-time Processing
- Captures and processes screen content in real-time
- Provides immediate feedback and analysis
- Maintains performance during long sessions
- Supports manual snapshot capture for detailed analysis
## User Experience Goals
@ -28,11 +30,14 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Works with Chess.com desktop app
- Minimal setup requirements
- Automatic board detection and tracking
- Clean snapshot capture without cursor interference
2. Intuitive Interface
- Clear visualization of analysis
- Easy-to-understand suggestions
- Minimal user intervention required
- Visual feedback for capture states
- Manual snapshot control
### Reliable Detection
1. Board Recognition
@ -41,6 +46,7 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
* Coordinate-based fallback for reliability
- Proper coordinate system handling
- Consistent board capture across sessions
- High-quality snapshots for analysis
2. Position Analysis
- Accurate piece recognition (planned)
@ -59,11 +65,13 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Understanding position evaluation
- Learning from mistakes
- Exploring alternative moves
- Analyzing specific positions via snapshots
2. Analysis
- Real-time position assessment
- Move validation
- Strategic planning
- Detailed position study
## Product Requirements
@ -73,6 +81,7 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Full board capture
- Support for Chess.com desktop app
- Reliable coordinate transformations
- Clean snapshot capability
2. Position Analysis (Planned)
- Real-time evaluation
@ -83,22 +92,27 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Analysis overlay
- Control panel
- Settings management
- Snapshot controls
- Visual status indicators
### Quality Standards
1. Accuracy
- Reliable board detection
- Complete board capture
- Precise coordinate handling
- Clean snapshots without artifacts
2. Performance
- Real-time processing
- Minimal resource usage
- Stable operation
- Efficient snapshot handling
3. Usability
- Intuitive controls
- Clear feedback
- Minimal setup
- Simple snapshot workflow
## Success Metrics
@ -107,12 +121,14 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Full board capture success rate
- Processing speed per frame
- Error recovery rate
- Snapshot quality assessment
### User Metrics
- Setup success rate
- Analysis accuracy
- User engagement time
- Feature utilization
- Snapshot usage patterns
## Current Challenges
@ -127,12 +143,14 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Full board capture
- Consistent positioning
- Reliable boundaries
- Clean snapshots
### Next Steps
1. Refine board detection
- Improve coordinate handling
- Ensure full board capture
- Validate transformations
- Optimize snapshot quality
2. Move to position analysis
- Piece recognition
@ -146,16 +164,19 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Deep position evaluation
- Opening recognition
- Endgame tablebases
- Position comparison from snapshots
2. Learning Tools
- Mistake analysis
- Improvement suggestions
- Progress tracking
- Position database from snapshots
3. Customization
- Analysis depth control
- Visual preference settings
- Platform-specific optimizations
- Snapshot management options
## Product Roadmap
@ -163,13 +184,16 @@ ChessPrism is an advanced chess analysis tool that enhances the online chess exp
- Core board detection system
- Coordinate system handling
- Basic user interface
- Manual snapshot system
### Next Phase
- Position analysis
- Move suggestion system
- Visual overlay implementation
- Enhanced snapshot analysis
### Future Phase
- Advanced analysis features
- Learning tools integration
- Customization options
- Snapshot database and comparison tools

24
cline_docs/systemPatterns.md
View file

@ -24,12 +24,35 @@
- Window-specific capture setup
- Frame dimension matching
- Proper delegate handling
- Cursor visibility control:
* Configurable cursor display
* Clean snapshot support
* State preservation
2. Frame Processing
- Main thread safety for UI updates
- Efficient image conversion pipeline
- Resource cleanup
### Snapshot System Pattern
1. Cursor-Free Capture
- Temporary capture session:
* Disables cursor visibility
* Takes clean snapshot
* Restores normal capture
- Error handling:
* Session cleanup
* State recovery
* Capture restoration
2. Process Flow
- Stop current capture
- Start cursor-free capture
- Wait for stabilization
- Take snapshot
- Process image
- Restore normal capture
### Error Handling Pattern
1. Task Management
- Proper cancellation points
@ -89,6 +112,7 @@
- Handles capture session lifecycle and cleanup
- Manages permissions and error handling
- Optimized resource usage
- Configurable cursor visibility
### Board Detection System
Two implemented approaches:

9
cline_docs/techContext.md
View file

@ -25,11 +25,13 @@
* iOS app window capture support
* Real-time frame capture
* Proper error handling
* Configurable cursor visibility
- Key components:
* SCShareableContent: Window and display access
* SCContentFilter: Window-specific capture
* SCStream: Frame capture management
* SCStreamOutput: Frame processing
* SCStreamConfiguration: Capture settings including cursor control
### Vision Framework (Planned)
- Will be used for board and coordinate detection
@ -89,6 +91,10 @@
- Frame dimension matching
- Proper delegate handling
- Resource cleanup
- Cursor visibility control:
* Configurable via SCStreamConfiguration
* State preservation between captures
* Clean snapshot support
3. Performance
- Main thread safety for UI updates
@ -155,18 +161,21 @@
- Error handling
- State management
- Resource management
- Cursor control functionality
### Integration Tests
- End-to-end workflows
- Component interaction
- Event propagation
- Resource sharing
- Snapshot system
### UI Tests
- User interaction flows
- Error state handling
- Visual feedback
- Performance monitoring
- Snapshot visualization
## Documentation Requirements