Merge pull request #21 from NakaokaRei/feat/locate-all-on-screen

feat: add locateAllOnScreen() function

Author: NakaokaRei

README.md

@@ -170,6 +170,27 @@ if let center = SwiftAutoGUI.locateCenterOnScreen("target.png", confidence: 0.8,
     SwiftAutoGUI.move(to: center)
     SwiftAutoGUI.leftClick()
 }
+
+// Find all occurrences of an image on screen
+let buttons = SwiftAutoGUI.locateAllOnScreen("button.png")
+print("Found \(buttons.count) buttons")
+for (index, button) in buttons.enumerated() {
+    print("Button \(index): \(button)")
+    SwiftAutoGUI.move(to: CGPoint(x: button.midX, y: button.midY))
+    SwiftAutoGUI.leftClick()
+    Thread.sleep(forTimeInterval: 0.5)
+}
+
+// locateAllOnScreen with confidence threshold for flexible matching
+let icons = SwiftAutoGUI.locateAllOnScreen("app_icon.png", confidence: 0.85)
+for icon in icons {
+    // Process each found icon
+    print("Found icon at: \(icon)")
+}
+
+// Search for multiple matches in a specific region
+let topRegion = CGRect(x: 0, y: 0, width: 1920, height: 100)
+let menuItems = SwiftAutoGUI.locateAllOnScreen("menu_item.png", region: topRegion)
 ```
 
 # Contributors

Sample/Sample/ContentView.swift

@@ -144,6 +144,10 @@ struct ContentView: View {
                         locateAndClickTestImage()
                     }
 
+                    Button("Find All Test Images") {
+                        findAllTestImages()
+                    }
+                    
                     if !imageRecognitionResult.isEmpty {
                         Text(imageRecognitionResult)
                             .font(.caption)
@@ -280,6 +284,52 @@ struct ContentView: View {
             imageRecognitionResult = "Test image not found on screen. Make sure the test image is visible in an app window."
         }
     }
+    
+    private func findAllTestImages() {
+        guard !testImagePath.isEmpty else {
+            imageRecognitionResult = "Please create a test image first"
+            return
+        }
+        
+        imageRecognitionResult = "Searching for all test images..."
+        
+        // Find all instances of the test image
+        let allMatches = SwiftAutoGUI.locateAllOnScreen(testImagePath, confidence: 0.8)
+        
+        if !allMatches.isEmpty {
+            imageRecognitionResult = "Found \(allMatches.count) instances of the test image:\n"
+            
+            // Show details of each match and highlight them
+            for (index, rect) in allMatches.enumerated() {
+                imageRecognitionResult += "\n[\(index + 1)] at x=\(Int(rect.origin.x)), y=\(Int(rect.origin.y)), size=\(Int(rect.width))x\(Int(rect.height))"
+                
+                // Draw a box around each found instance
+                let corners = [
+                    CGPoint(x: rect.origin.x, y: rect.origin.y),
+                    CGPoint(x: rect.origin.x + rect.width, y: rect.origin.y),
+                    CGPoint(x: rect.origin.x + rect.width, y: rect.origin.y + rect.height),
+                    CGPoint(x: rect.origin.x, y: rect.origin.y + rect.height),
+                    CGPoint(x: rect.origin.x, y: rect.origin.y)
+                ]
+                
+                for i in 0..<corners.count - 1 {
+                    SwiftAutoGUI.move(to: corners[i])
+                    Thread.sleep(forTimeInterval: 0.1)
+                    SwiftAutoGUI.move(to: corners[i + 1])
+                    Thread.sleep(forTimeInterval: 0.1)
+                }
+            }
+            
+            // Move to center of the first match
+            if let firstMatch = allMatches.first {
+                SwiftAutoGUI.move(to: CGPoint(x: firstMatch.midX, y: firstMatch.midY))
+            }
+            
+            imageRecognitionResult += "\n\nHighlighted all \(allMatches.count) matches!"
+        } else {
+            imageRecognitionResult = "No test images found on screen. Try opening multiple windows with the test image visible."
+        }
+    }
 }
 
 struct ContentView_Previews: PreviewProvider {

Sources/SwiftAutoGUI/ImageRecognition.swift

@@ -120,6 +120,66 @@ extension SwiftAutoGUI {
         return CGPoint(x: rect.midX, y: rect.midY)
     }
 
+    /// Locate all instances of an image on the screen and return their positions
+    ///
+    /// - Parameters:
+    ///   - imagePath: Path to the image file to search for
+    ///   - grayscale: Convert to grayscale for faster matching (currently ignored, for future implementation)
+    ///   - confidence: Matching confidence threshold (0.0-1.0). If nil, uses exact matching (0.95 by default)
+    ///   - region: Limit search to specific screen region. If nil, searches entire screen
+    /// - Returns: Array of CGRect with locations (x, y, width, height) of all found instances, empty array if none found
+    ///
+    /// This method uses OpenCV's template matching algorithm to find all instances of an image on the screen.
+    /// It applies non-maximum suppression to avoid duplicate detections of the same object.
+    ///
+    /// Example:
+    /// ```swift
+    /// // Find all buttons on screen
+    /// let buttons = SwiftAutoGUI.locateAllOnScreen("button.png")
+    /// for (index, button) in buttons.enumerated() {
+    ///     print("Button \(index) found at: \(button)")
+    ///     SwiftAutoGUI.move(to: CGPoint(x: button.midX, y: button.midY))
+    ///     SwiftAutoGUI.leftClick()
+    ///     Thread.sleep(forTimeInterval: 0.5)
+    /// }
+    ///
+    /// // Find all matches with confidence threshold
+    /// let closeButtons = SwiftAutoGUI.locateAllOnScreen("close_button.png", confidence: 0.85)
+    /// print("Found \(closeButtons.count) close buttons")
+    ///
+    /// // Search in specific region for better performance
+    /// let searchRegion = CGRect(x: 0, y: 0, width: 800, height: 600)
+    /// let icons = SwiftAutoGUI.locateAllOnScreen("icon.png", region: searchRegion)
+    /// ```
+    public static func locateAllOnScreen(
+        _ imagePath: String,
+        grayscale: Bool = false,
+        confidence: Double? = nil,
+        region: CGRect? = nil
+    ) -> [CGRect] {
+        // Load the needle image
+        guard let needleImage = NSImage(contentsOfFile: imagePath) else {
+            print("SwiftAutoGUI: Could not load image from path: \(imagePath)")
+            return []
+        }
+        
+        // Take screenshot of the region or entire screen
+        let screenshot: NSImage?
+        if let region = region {
+            screenshot = self.screenshot(region: region)
+        } else {
+            screenshot = self.screenshot()
+        }
+        
+        guard let haystackImage = screenshot else {
+            print("SwiftAutoGUI: Could not capture screenshot")
+            return []
+        }
+        
+        // Perform image matching to find all instances
+        return findAllImagesInImage(needle: needleImage, haystack: haystackImage, confidence: confidence, searchRegion: region)
+    }
+    
     // MARK: Private Helper Methods
 
     /// Find needle image within haystack image using OpenCV template matching
@@ -204,6 +264,124 @@ extension SwiftAutoGUI {
 
         return nil
     }
+    
+    /// Find all instances of needle image within haystack image using OpenCV template matching
+    private static func findAllImagesInImage(
+        needle: NSImage,
+        haystack: NSImage,
+        confidence: Double?,
+        searchRegion: CGRect?
+    ) -> [CGRect] {
+        // Convert NSImages to OpenCV Mat format
+        guard let needleMat = needle.toMat(),
+              let haystackMat = haystack.toMat() else {
+            print("SwiftAutoGUI: Could not convert images to OpenCV Mat")
+            return []
+        }
+        
+        // Apply search region if specified
+        let searchMat: Mat
+        let regionOffset: CGPoint
+        
+        if let region = searchRegion {
+            let rect = Rect2i(
+                x: Int32(region.origin.x),
+                y: Int32(region.origin.y),
+                width: Int32(region.width),
+                height: Int32(region.height)
+            )
+            searchMat = Mat(mat: haystackMat, rect: rect)
+            regionOffset = region.origin
+        } else {
+            searchMat = haystackMat
+            regionOffset = .zero
+        }
+        
+        // Perform template matching using OpenCV
+        let result = Mat()
+        Imgproc.matchTemplate(
+            image: searchMat,
+            templ: needleMat,
+            result: result,
+            method: TemplateMatchModes.TM_CCOEFF_NORMED  // Normalized correlation coefficient
+        )
+        
+        let threshold = confidence ?? 0.95
+        var matches: [CGRect] = []
+        
+        // Get screen scale factor
+        let screen = NSScreen.main ?? NSScreen.screens[0]
+        let scaleFactor = screen.backingScaleFactor
+        
+        // Find all matches above threshold
+        var resultData = [Float](repeating: 0, count: Int(result.rows() * result.cols()))
+        _ = try? result.get(row: 0, col: 0, data: &resultData)
+        
+        let templateWidth = needleMat.cols()
+        let templateHeight = needleMat.rows()
+        
+        // Create a copy of result to track which areas we've already processed
+        var processedMask = Array(repeating: false, count: resultData.count)
+        
+        while true {
+            // Find the maximum value and its location
+            var maxVal: Float = -1
+            var maxIdx = -1
+            
+            for i in 0..<resultData.count {
+                if !processedMask[i] && resultData[i] > maxVal {
+                    maxVal = resultData[i]
+                    maxIdx = i
+                }
+            }
+            
+            // If no more matches above threshold, break
+            if maxVal < Float(threshold) || maxIdx == -1 {
+                break
+            }
+            
+            // Calculate coordinates from the index
+            let y = maxIdx / Int(result.cols())
+            let x = maxIdx % Int(result.cols())
+            
+            // Convert OpenCV coordinates to CGRect
+            let pixelRect = CGRect(
+                x: CGFloat(x) + regionOffset.x,
+                y: CGFloat(y) + regionOffset.y,
+                width: CGFloat(templateWidth),
+                height: CGFloat(templateHeight)
+            )
+            
+            // Convert from pixels to points (logical coordinates)
+            let pointRect = CGRect(
+                x: pixelRect.origin.x / scaleFactor,
+                y: pixelRect.origin.y / scaleFactor,
+                width: pixelRect.width / scaleFactor,
+                height: pixelRect.height / scaleFactor
+            )
+            
+            matches.append(pointRect)
+            
+            // Apply non-maximum suppression: mark nearby pixels as processed
+            // to avoid detecting the same object multiple times
+            let suppressionRadius = min(templateWidth, templateHeight) / 2
+            
+            for dy in -suppressionRadius...suppressionRadius {
+                for dx in -suppressionRadius...suppressionRadius {
+                    let ny = y + Int(dy)
+                    let nx = x + Int(dx)
+                    
+                    if ny >= 0 && ny < Int(result.rows()) && nx >= 0 && nx < Int(result.cols()) {
+                        let idx = ny * Int(result.cols()) + nx
+                        processedMask[idx] = true
+                    }
+                }
+            }
+        }
+        
+        print("SwiftAutoGUI: Found \(matches.count) matches with confidence >= \(threshold)")
+        return matches
+    }
 }
 
 // MARK: - NSImage to OpenCV Mat conversion

Tests/SwiftAutoGUITests/ImageRecognitionTests.swift

@@ -136,6 +136,106 @@ struct ImageRecognitionTests {
         #expect(true)
     }
 
+    @Test("locateAllOnScreen with valid image path")
+    func testLocateAllOnScreenValidPath() {
+        // Create a test image
+        let testImagePath = createTestImage()
+        defer { try? FileManager.default.removeItem(atPath: testImagePath) }
+        
+        // Try to locate all instances of the image
+        let results = SwiftAutoGUI.locateAllOnScreen(testImagePath)
+        
+        // Results should be an array (possibly empty)
+        #expect(results.count >= 0)
+        
+        // If any images are found, verify they have valid dimensions
+        for rect in results {
+            #expect(rect.width > 0)
+            #expect(rect.height > 0)
+        }
+        
+        // Test passes whether or not images are found (depends on permissions)
+        #expect(true)
+    }
+    
+    @Test("locateAllOnScreen with invalid image path")
+    func testLocateAllOnScreenInvalidPath() {
+        let results = SwiftAutoGUI.locateAllOnScreen("/nonexistent/image.png")
+        
+        // Should return empty array for invalid path
+        #expect(results.isEmpty)
+    }
+    
+    @Test("locateAllOnScreen with region")
+    func testLocateAllOnScreenWithRegion() {
+        // Create a test image
+        let testImagePath = createTestImage()
+        defer { try? FileManager.default.removeItem(atPath: testImagePath) }
+        
+        // Search in a specific region
+        let region = CGRect(x: 0, y: 0, width: 400, height: 400)
+        let results = SwiftAutoGUI.locateAllOnScreen(testImagePath, region: region)
+        
+        // If found, verify all results are within the search region
+        for rect in results {
+            // The found rectangle should at least partially overlap with the search region
+            #expect(region.intersects(rect))
+        }
+        
+        // Test passes whether or not images are found
+        #expect(true)
+    }
+    
+    @Test("locateAllOnScreen with confidence parameter")
+    func testLocateAllOnScreenWithConfidence() {
+        // Create a test image
+        let testImagePath = createTestImage()
+        defer { try? FileManager.default.removeItem(atPath: testImagePath) }
+        
+        // Search with different confidence levels
+        let highConfidenceResults = SwiftAutoGUI.locateAllOnScreen(testImagePath, confidence: 0.95)
+        let lowConfidenceResults = SwiftAutoGUI.locateAllOnScreen(testImagePath, confidence: 0.5)
+        
+        // Lower confidence should find same or more matches
+        // But both could be empty if no screen access
+        #expect(highConfidenceResults.count <= lowConfidenceResults.count)
+        
+        // Test passes whether or not images are found
+        #expect(true)
+    }
+    
+    @Test("locateAllOnScreen returns non-overlapping results")
+    func testLocateAllOnScreenNonOverlapping() {
+        // Create a test image
+        let testImagePath = createTestImage()
+        defer { try? FileManager.default.removeItem(atPath: testImagePath) }
+        
+        // Find all instances
+        let results = SwiftAutoGUI.locateAllOnScreen(testImagePath, confidence: 0.8)
+        
+        // Verify that results don't significantly overlap (due to non-maximum suppression)
+        for i in 0..<results.count {
+            for j in (i+1)..<results.count {
+                let rect1 = results[i]
+                let rect2 = results[j]
+                
+                // Calculate overlap
+                let intersection = rect1.intersection(rect2)
+                let overlap = intersection.width * intersection.height
+                let minArea = min(rect1.width * rect1.height, rect2.width * rect2.height)
+                
+                // Overlap should be less than 50% of the smaller rectangle
+                if minArea > 0 {
+                    let overlapRatio = overlap / minArea
+                    #expect(overlapRatio < 0.5)
+                }
+            }
+        }
+        
+        // Test passes
+        #expect(true)
+    }
+    
     // Helper function to create a test image
     private func createTestImage() -> String {
         let size = NSSize(width: 50, height: 50)