diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3acc279ae1..8e8b8f9654 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,7 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 + submodules: recursive - name: Fetch tags run: git fetch --tags --force - name: Install Java 17 @@ -240,21 +241,22 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 + submodules: recursive - name: Install Java 17 uses: actions/setup-java@v5 with: java-version: 17 distribution: temurin - run: git fetch --tags --force - - run: ./gradlew photon-targeting:build photon-lib:build + - run: ./gradlew photon-targeting:build photon-lib:build -PphotonAppleForceStubs name: Build with Gradle - - run: ./gradlew photon-lib:publish photon-targeting:publish + - run: ./gradlew photon-lib:publish photon-targeting:publish -PphotonAppleForceStubs name: Publish env: ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }} if: github.event_name == 'push' && github.repository_owner == 'photonvision' # Copy artifacts to build/outputs/maven - - run: ./gradlew photon-lib:publish photon-targeting:publish -PcopyOfflineArtifacts + - run: ./gradlew photon-lib:publish photon-targeting:publish -PcopyOfflineArtifacts -PphotonAppleForceStubs - uses: actions/upload-artifact@v6 with: name: maven-${{ matrix.artifact-name }} @@ -417,6 +419,90 @@ jobs: steps: *build-package-steps + build-package-macos: + needs: [build-gradle, build-offline-docs] + + strategy: + fail-fast: false + matrix: + include: + - os: macos-26 + artifact-name: macOS + architecture: aarch64 + arch-override: macarm64 + + runs-on: ${{ matrix.os }} + name: "Build fat JAR - ${{ matrix.artifact-name }}" + + steps: &build-package-steps-macos + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Install Java 25 (macOS toolchain for SwiftKit) + uses: actions/setup-java@v5 + if: runner.os == 'macOS' + with: + java-version: 25 + distribution: temurin + architecture: ${{ matrix.architecture }} + - name: Install Java 24 (for Gradle on macOS) + uses: actions/setup-java@v5 + if: runner.os == 'macOS' + with: + java-version: 24 + distribution: temurin + architecture: ${{ matrix.architecture }} + - name: Install Java 17 (for Gradle on non-macOS) + uses: actions/setup-java@v5 + if: runner.os != 'macOS' + with: + java-version: 17 + distribution: temurin + architecture: ${{ matrix.architecture }} + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: photon-client/pnpm-lock.yaml + - name: Cache Swift build artifacts (macOS only) + uses: actions/cache@v4 + if: runner.os == 'macOS' + with: + path: photon-apple/.build + key: ${{ matrix.artifact-name }}-photon-apple-build-${{ hashFiles('photon-apple/swift-java/Package.swift', 'photon-apple/swift-java/gradle.properties', 'photon-apple/swift-java/settings.gradle') }} + restore-keys: | + ${{ matrix.artifact-name }}-photon-apple-build- + - name: Publish SwiftKit to Maven Local (macOS only) + if: runner.os == 'macOS' + run: | + cd photon-apple/swift-java + ./gradlew publishToMavenLocal -PskipSamples=true + - name: Install Arm64 Toolchain + run: ./gradlew installArm64Toolchain + if: ${{ (matrix.artifact-name) == 'LinuxArm64' }} + - uses: actions/download-artifact@v6 + with: + name: built-docs + path: photon-server/src/main/resources/web/docs + - run: ./gradlew photon-targeting:jar photon-server:shadowJar -PArchOverride=${{ matrix.arch-override }} + if: ${{ (matrix.arch-override != 'none') }} + - run: ./gradlew photon-server:shadowJar + if: ${{ (matrix.arch-override == 'none') }} + - uses: actions/upload-artifact@v6 + with: + name: jar-${{ matrix.artifact-name }} + path: photon-server/build/libs + - uses: actions/upload-artifact@v6 + with: + name: photon-targeting_jar-${{ matrix.artifact-name }} + path: photon-targeting/build/libs + run-smoketest-native: needs: [build-package-linux, build-package-macos, build-package-windows] @@ -429,14 +515,21 @@ jobs: extraOpts: -Djdk.lang.Process.launchMechanism=vfork - os: windows-latest artifact-name: jar-Win64 - - os: macos-latest + - os: macos-26 artifact-name: jar-macOS runs-on: ${{ matrix.os }} steps: - - name: Install Java 17 + - name: Install Java 25 (macOS) + uses: actions/setup-java@v5 + if: runner.os == 'macOS' + with: + java-version: 25 + distribution: temurin + - name: Install Java 17 (non-macOS) uses: actions/setup-java@v5 + if: runner.os != 'macOS' with: java-version: 17 distribution: temurin diff --git a/.gitignore b/.gitignore index 6a3ddc0b6f..d4cbe6ec24 100644 --- a/.gitignore +++ b/.gitignore @@ -156,3 +156,5 @@ photon-client/playwright-report/ photon-client/blob-report/ photon-client/playwright/.cache/ photon-client/playwright/.auth/ + +.build/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..f9e03bb229 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "photon-apple/swift-java"] + path = photon-apple/swift-java + url = https://github.com/swiftlang/swift-java.git diff --git a/.wpiformat b/.wpiformat index 9a0af6cac8..9c3b4c682f 100644 --- a/.wpiformat +++ b/.wpiformat @@ -6,4 +6,7 @@ modifiableFileExclude { photon-lib/py/photonlibpy/generated/ photon-targeting/src/generated/ photon-targeting/src/main/native/cpp/photon/constrained_solvepnp/generate/ + .build/ + photon-apple/.build/ + photon-apple/swift-java/ } diff --git a/README.md b/README.md index fc8400e25d..1107664b61 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ PhotonVision is the free, fast, and easy-to-use computer vision solution for the The latest release of platform-specific jars and images is found [here](https://github.com/PhotonVision/photonvision/releases). -If you are interested in contributing code or documentation to the project, please [read our getting started page for contributors](https://docs.photonvision.org/en/latest/docs/contributing/index.html) and **[join the Discord](https://discord.gg/wYxTwym) to introduce yourself!** We hope to provide a welcoming community to anyone who is interested in helping. +If you are interested in contributing code or documentation to the project, please [read our getting started page for contributors](https://docs.photonvision.org/en/latest/docs/contributing/index.html) and **[join the Discord](https://discord.gg/wYxTwym) to introduce yourself!** We hope to provide a welcoming community to anyone who is interested in helping out. ## Documentation diff --git a/build.gradle b/build.gradle index d0be427460..09ddcb4ab7 100644 --- a/build.gradle +++ b/build.gradle @@ -23,6 +23,23 @@ allprojects { } wpilibRepositories.addAllReleaseRepositories(it) wpilibRepositories.addAllDevelopmentRepositories(it) + + // Allow Java 25 dependencies (SwiftKit) to be consumed by Java 24 projects + // This is needed because photon-apple requires SwiftKit which needs Java 25, + // but photon-server runs on Java 24 + dependencies { + components { + all { ComponentMetadataDetails details -> + if (details.id.group == 'org.swift.swiftkit') { + details.allVariants { VariantMetadata variant -> + variant.attributes { + attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 24) + } + } + } + } + } + } } ext.localMavenURL = file("$project.buildDir/outputs/maven") @@ -62,7 +79,8 @@ spotless { java { target fileTree('.') { include '**/*.java' - exclude '**/build/**', '**/build-*/**', '**/src/generated/**' + exclude '**/build/**', '**/build-*/**', '**/src/generated/**', + '**/.build/**', '**/photon-apple/swift-java/**' } toggleOffOn() googleJavaFormat() @@ -75,7 +93,8 @@ spotless { groovyGradle { target fileTree('.') { include '**/*.gradle' - exclude '**/build/**', '**/build-*/**' + exclude '**/build/**', '**/build-*/**', + '**/.build/**', '**/photon-apple/swift-java/**' } greclipse() indentWithSpaces(4) @@ -85,7 +104,8 @@ spotless { format 'misc', { target fileTree('.') { include '**/*.md', '**/.gitignore' - exclude '**/build/**', '**/build-*/**', '**/node_modules/**' + exclude '**/build/**', '**/build-*/**', '**/node_modules/**', + '**/.build/**', '**/photon-apple/swift-java/**' } trimTrailingWhitespace() indentWithSpaces(2) diff --git a/docs/source/docs/contributing/building-photon.md b/docs/source/docs/contributing/building-photon.md index 90d44bc78c..1c6e476eea 100644 --- a/docs/source/docs/contributing/building-photon.md +++ b/docs/source/docs/contributing/building-photon.md @@ -8,7 +8,18 @@ This section contains the build instructions from the source code available at [ **Java Development Kit:** - This project requires Java Development Kit (JDK) 17 to be compiled. This is the same Java version that comes with WPILib for 2026+. **Windows Users must use the JDK that ships with WPILib.** For other platforms, you can follow the instructions to install JDK 17 for your platform [here](https://bell-sw.com/pages/downloads/#jdk-17-lts). + PhotonLib requires Java Development Kit (JDK) 17 to be compiled. This is the same Java version that comes with WPILib for 2026+. **Windows Users must use the JDK that ships with WPILib.** For other platforms, you can follow the instructions to install JDK 17 for your platform [here](https://bell-sw.com/pages/downloads/#jdk-17-lts). + + The PhotonVision server application requires JDK 17 to be compiled on Windows and Linux. + On macOS, it requires JDK 24 and JDK 25 (both JDK 24 and JDK 25 must be installed with JAVA_HOME pointing at JDK 24). + + **JDK Version Split** + + WPILib uses JDK 17 for the 2026 season. Photonlib must be built with JDK 17 compatibility. + + PhotonVision on macOS uses a library (swift-java) to wrap swift code in java for CoreML based object detection. + swift-java requires Java 25 to build. Photonvision and WPILib use Gradle 8. Gradle 8 does not support Java 25. Gradle 9 is required for Java 25. So we build swift-java using Gradle 9 and Java 25 then use Gradle 8 and Java 24 to build photonvision. + **Node JS:** @@ -57,6 +68,41 @@ pnpm run dev This allows you to make UI changes quickly without having to spend time rebuilding the jar. Hot reload is enabled, so changes that you make and save are reflected in the UI immediately. Running this command will give you the URL for accessing the UI, which is on a different port than normal. You must use the printed URL to use hot reload. +### One time build the UI + +To build the UI once and have it be served directly by PhotonVision, issue the following command in the root directory: + +```{eval-rst} +.. tab-set:: + + .. tab-item:: Linux + :sync: linux + + ``./gradlew buildAndCopyUI`` + + .. tab-item:: macOS + :sync: macos + + ``./gradlew buildAndCopyUI`` + + .. tab-item:: Windows (cmd) + :sync: windows + + ``gradlew buildAndCopyUI`` +``` + +### macOS only - Compile swift-java + +On macOS, before building PhotonVision, it is required to build swift-java one time. + +To build swift-java, issue the following from the root directory: + +```bash +git submodule update --init --recursive +cd photon-apple/swift-java +./gradlew publishToMavenLocal -x :Samples:build +``` + ### Build and Run PhotonVision To compile and run the project, issue the following command in the root directory: @@ -308,3 +354,8 @@ build action, which can be found [here](https://github.com/PhotonVision/photonvi In order to force the Object Detection interface to be visible, it's necessary to hardcode the platform that `Platform.java` returns. This can be done by changing the function that detects the RK3588S/QCS6490 platform to always return true, and changing the `getCurrentPlatform()` function to always return the RK3588S/QCS6490 architecture. Alternatively, it's possible to modify the frontend code by changing all instances of `useSettingsStore().general.supportedBackends.length > 0` to `true`, which will force the card to render. Make sure to revert these changes before submitting a Pull Request. + +#### Building platform specific native code in the monorepo + +Platform specific native code can be built in the monorepo by isolating the native code to a new subfolder with its own build.gradle. +On the desired platform, CI will build the native code and the java wrappers for it. On other platforms, configure the project to use stubs. See photon-apple/build.gradle. diff --git a/photon-apple/Package.resolved b/photon-apple/Package.resolved new file mode 100644 index 0000000000..774d15d05f --- /dev/null +++ b/photon-apple/Package.resolved @@ -0,0 +1,51 @@ +{ + "originHash" : "b0b889681226a6cefdaac96640317da0350d526f8e1409502aed88d2a442d87f", + "pins" : [ + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "c5d11a805e765f52ba34ec7284bd4fcd6ba68615", + "version" : "1.7.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections", + "state" : { + "revision" : "7b847a3b7008b2dc2f47ca3110d8c782fb2e5c7e", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log", + "state" : { + "revision" : "bc386b95f2a16ccd0150a8235e7c69eab2b866ca", + "version" : "1.8.0" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax", + "state" : { + "revision" : "4799286537280063c85a32f09884cfbca301b1a1", + "version" : "602.0.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system", + "state" : { + "revision" : "395a77f0aa927f0ff73941d7ac35f2b46d47c9db", + "version" : "1.6.3" + } + } + ], + "version" : 3 +} diff --git a/photon-apple/Package.swift b/photon-apple/Package.swift new file mode 100644 index 0000000000..6dfc60fb7a --- /dev/null +++ b/photon-apple/Package.swift @@ -0,0 +1,81 @@ +// swift-tools-version: 6.0 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import CompilerPluginSupport +import PackageDescription + +import class Foundation.FileManager +import class Foundation.ProcessInfo + +// Note: the JAVA_HOME environment variable must be set to point to where +// Java is installed, e.g., +// /Users/vasis/.sdkman/candidates/java/24.0.2-tem +func findJavaHome() -> String { + if let home = ProcessInfo.processInfo.environment["JAVA_HOME"] { + return home + } + + // This is a workaround for envs (some IDEs) which have trouble with + // picking up env variables during the build process + let path = "\(FileManager.default.homeDirectoryForCurrentUser.path()).java_home" + if let home = try? String(contentsOfFile: path, encoding: .utf8) { + if let lastChar = home.last, lastChar.isNewline { + return String(home.dropLast()) + } + + return home + } + + fatalError("Please set the JAVA_HOME environment variable to point to where Java is installed.") +} +let javaHome = findJavaHome() + +let javaIncludePath = "\(javaHome)/include" +#if os(Linux) + let javaPlatformIncludePath = "\(javaIncludePath)/linux" +#elseif os(macOS) + let javaPlatformIncludePath = "\(javaIncludePath)/darwin" +#else + // TODO: Handle windows as well + #error("Currently only macOS and Linux platforms are supported, this may change in the future.") +#endif + +let package = Package( + name: "PhotonApple", + platforms: [ + .macOS(.v15), + .iOS(.v18), + ], + products: [ + .library( + name: "PhotonAppleLibrary", + type: .dynamic, + targets: ["PhotonAppleLibrary"] + ), + ], + dependencies: [ + // SwiftKit libraries from vendored swift-java submodule + .package(name: "swift-java", path: "./swift-java/"), + ], + targets: [ + .target( + name: "PhotonAppleLibrary", + dependencies: [ + .product(name: "SwiftJava", package: "swift-java"), + .product(name: "CSwiftJavaJNI", package: "swift-java"), + .product(name: "SwiftRuntimeFunctions", package: "swift-java"), + ], + swiftSettings: [ + .swiftLanguageMode(.v5), + .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]) + ], + plugins: [ + .plugin(name: "JExtractSwiftPlugin", package: "swift-java"), + ] + ), + .testTarget( + name: "PhotonAppleLibraryTests", + dependencies: ["PhotonAppleLibrary"] + ), + ] +) diff --git a/photon-apple/Sources/PhotonAppleLibrary/DetectionResult.swift b/photon-apple/Sources/PhotonAppleLibrary/DetectionResult.swift new file mode 100644 index 0000000000..c48977fa66 --- /dev/null +++ b/photon-apple/Sources/PhotonAppleLibrary/DetectionResult.swift @@ -0,0 +1,20 @@ +import Foundation + +/// Represents a single object detection result +public class DetectionResult { + public let x: Double // Center x coordinate (normalized 0-1) + public let y: Double // Center y coordinate (normalized 0-1) + public let width: Double // Width (normalized 0-1) + public let height: Double // Height (normalized 0-1) + public let classId: Int32 // Class identifier + public let confidence: Double // Confidence score (0-1) + + public init(x: Double, y: Double, width: Double, height: Double, classId: Int32, confidence: Double) { + self.x = x + self.y = y + self.width = width + self.height = height + self.classId = classId + self.confidence = confidence + } +} diff --git a/photon-apple/Sources/PhotonAppleLibrary/DetectionResultArray.swift b/photon-apple/Sources/PhotonAppleLibrary/DetectionResultArray.swift new file mode 100644 index 0000000000..cd5168b2aa --- /dev/null +++ b/photon-apple/Sources/PhotonAppleLibrary/DetectionResultArray.swift @@ -0,0 +1,19 @@ +import Foundation + +/// Array wrapper for detection results that can be accessed from Java +/// SwiftJava FFM doesn't support arrays +public class DetectionResultArray { + private let results: [DetectionResult] + + public init(results: [DetectionResult]) { + self.results = results + } + + public func count() -> Int { + return results.count + } + + public func get(index: Int) -> DetectionResult { + return results[index] + } +} diff --git a/photon-apple/Sources/PhotonAppleLibrary/ObjectDetector.swift b/photon-apple/Sources/PhotonAppleLibrary/ObjectDetector.swift new file mode 100644 index 0000000000..caaa065c26 --- /dev/null +++ b/photon-apple/Sources/PhotonAppleLibrary/ObjectDetector.swift @@ -0,0 +1,219 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation +import Vision +import CoreML +import CoreImage +import CoreVideo + + +// MARK: - Debug Logging + +/// Print with source location for debugging +fileprivate func p(_ message: String, function: String = #function, line: Int = #line) { + print("[ObjectDetector.swift:\(line)](\(function)) \(message)") +} + +fileprivate func d(_ message: String, function: String = #function, line: Int = #line) { + // print("[ObjectDetector.swift:\(line)](\(function)) \(message)") +} + +// MARK: - Object Detector + +/// Object detector using Apple's Vision framework +/// Vision framework automatically handles image resizing and cropping for the model +public class ObjectDetector { + private let modelPath: String + private var vnModel: VNCoreMLModel? + private var request: VNCoreMLRequest? + + /// Initialize the ObjectDetector with a CoreML model + /// - Parameter modelPath: Absolute path to the .mlmodel or .mlmodelc file + /// Vision framework will handle all image preprocessing + public init(modelPath: String) { + self.modelPath = modelPath + p("ObjectDetector created with model path: \(modelPath)") + } + + /// Load the CoreML model (called lazily on first use) + /// Automatically compiles .mlmodel files to .mlmodelc format if needed + private func ensureModelLoaded() -> Bool { + guard vnModel == nil else { return true } // Already loaded + + p("Loading CoreML model from: \(modelPath)") + let modelURL = URL(fileURLWithPath: modelPath) + + do { + let compiledURL: URL + if modelPath.hasSuffix(".mlmodel") { + // Compile the model to a temporary location + p("Compiling .mlmodel to .mlmodelc format...") + compiledURL = try MLModel.compileModel(at: modelURL) + p("Model compiled to: \(compiledURL.path)") + } else { + // Already compiled (.mlmodelc) or compiled directory + compiledURL = modelURL + } + + let config = MLModelConfiguration() + config.computeUnits = .all + let model = try MLModel(contentsOf: compiledURL, configuration: config) + let vn = try VNCoreMLModel(for: model) + vnModel = vn + + var err: (any Error)? + // Create Vision request with completion handler + request = VNCoreMLRequest(model: vn) { request, error in + err = error + } + + if let error = err { + throw error + } + + request!.imageCropAndScaleOption = .scaleFill + + p("CoreML model loaded successfully") + return true + } catch { + p("Failed to load CoreML model: \(error)") + return false + } + } + + /// Detect objects in an image using Vision framework + /// Vision framework automatically handles image resizing and cropping + /// - Parameters: + /// - imageData: Pointer to raw BGRA image bytes in Java memory + /// - width: Image width in pixels + /// - height: Image height in pixels + /// - pixelFormat: Format of the pixel data (must be 2=BGRA) + /// - boxThreshold: Minimum confidence threshold for detections (0.0 - 1.0) + /// - nmsThreshold: Non-maximum suppression IoU threshold (0.0 - 1.0) + /// - Returns: Array of detection results + public func detect( + imageData: UnsafeMutableRawPointer, + width: Int, + height: Int, + boxThreshold: Double, + ) -> DetectionResultArray { + // Ensure model is loaded + guard ensureModelLoaded(), vnModel != nil, let request = request else { + p("Model not loaded, returning empty results") + return DetectionResultArray(results: []) + } + + // Create CVPixelBuffer from BGRA data + // Vision framework will handle resizing this to the model's input size + // guard let pixelBuffer = createBGRAPixelBuffer( + // from: imageData, + // width: width, + // height: height + // ) else { + // p("Failed to create CVPixelBuffer") + // return DetectionResultArray(results: []) + // } + // p("copy pixel buffer") + + var pixelBuffer: CVPixelBuffer? + + let result = CVPixelBufferCreateWithBytes( + kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + imageData, + width * 4, // Bytes per row + nil, // Called when CVPixelBuffer is released + nil, // Release callback context + nil, + &pixelBuffer + ) + + guard result == kCVReturnSuccess, let pixelBuffer = pixelBuffer else { + p("Failed to create CVPixelBuffer: \(result)") + return DetectionResultArray(results: []) + } + + // Perform detection using Vision framework + // Vision automatically resizes and crops the image to fit the model + let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]) + do { + try handler.perform([request]) + } catch { + p("Failed to perform detection: \(error)") + return DetectionResultArray(results: []) + } + + // Process results + guard let observations = request.results as? [VNRecognizedObjectObservation] else { + p("No results or unexpected result type") + return DetectionResultArray(results: []) + } + + // Filter by confidence threshold + // Note: Vision framework already performs NMS internally, so we don't need to do it manually + let filteredObservations = observations.filter { $0.confidence >= Float(boxThreshold) } + d("Detected \(observations.count) objects, \(filteredObservations.count) above threshold \(boxThreshold)") + + // Convert to DetectionResult array + let results = filteredObservations.map { observation -> DetectionResult in + let boundingBox = observation.boundingBox + + // Get the top classification + let topLabel = observation.labels.first + let classId = topLabel?.identifier.split(separator: " ").first.flatMap { Int32($0) } ?? -1 + + return DetectionResult( + x: Double(boundingBox.midX), + y: Double(1.0 - boundingBox.midY), + width: Double(boundingBox.width), + height: Double(boundingBox.height), + classId: classId, + confidence: Double(observation.confidence) + ) + } + + return DetectionResultArray(results: results) + } + + /// Fake detection method for testing - returns synthetic detection results + /// This method is useful for testing the Swift→Java data passing without requiring a CoreML model + /// - Parameters: + /// - imageData: Pointer to raw BGRA image bytes (not actually used) + /// - width: Image width in pixels + /// - height: Image height in pixels + /// - pixelFormat: Format of the pixel data (ignored) + /// - boxThreshold: Minimum confidence threshold (ignored) + /// - nmsThreshold: NMS threshold (ignored) + /// - Returns: Array of synthetic detection results for testing + public func detectFake( + imageData: UnsafeRawPointer, + width: Int, + height: Int, + pixelFormat: Int32, + boxThreshold: Double, + nmsThreshold: Double + ) -> DetectionResultArray { + // Return 3 fake detection results for testing + let fakeResults = [ + DetectionResult(x: 0.1, y: 0.2, width: 0.3, height: 0.4, classId: 1, confidence: 0.95), + DetectionResult(x: 0.5, y: 0.5, width: 0.2, height: 0.2, classId: 2, confidence: 0.87), + DetectionResult(x: 0.7, y: 0.1, width: 0.15, height: 0.25, classId: 3, confidence: 0.72) + ] + + return DetectionResultArray(results: fakeResults) + } +} diff --git a/photon-apple/Sources/PhotonAppleLibrary/swift-java.config b/photon-apple/Sources/PhotonAppleLibrary/swift-java.config new file mode 100644 index 0000000000..67f7c48598 --- /dev/null +++ b/photon-apple/Sources/PhotonAppleLibrary/swift-java.config @@ -0,0 +1,3 @@ +{ + "javaPackage": "org.photonvision.apple" +} diff --git a/photon-apple/Tests/PhotonAppleLibraryTests/ObjectDetectorTests.swift b/photon-apple/Tests/PhotonAppleLibraryTests/ObjectDetectorTests.swift new file mode 100644 index 0000000000..80cb58b34d --- /dev/null +++ b/photon-apple/Tests/PhotonAppleLibraryTests/ObjectDetectorTests.swift @@ -0,0 +1,69 @@ +import XCTest +@testable import PhotonAppleLibrary + +final class ObjectDetectorTests: XCTestCase { + func testDetectFake() { + // Test the fake detection method + let detector = ObjectDetector(modelPath: "/fake/path/model.mlmodel") + + // Create a dummy image buffer + let width = 640 + let height = 480 + let buffer = UnsafeMutableRawPointer.allocate(byteCount: width * height * 4, alignment: 1) + defer { buffer.deallocate() } + + // Call detectFake + let results = detector.detectFake( + imageData: buffer, + width: width, + height: height, + pixelFormat: 2, // BGRA + boxThreshold: 0.5, + nmsThreshold: 0.4 + ) + + // Verify we get 3 fake results + XCTAssertEqual(results.count(), 3, "Should return 3 fake detections") + + // Verify first result + let det0 = results.get(index: 0) + XCTAssertEqual(det0.x, 0.1, accuracy: 0.001) + XCTAssertEqual(det0.y, 0.2, accuracy: 0.001) + XCTAssertEqual(det0.width, 0.3, accuracy: 0.001) + XCTAssertEqual(det0.height, 0.4, accuracy: 0.001) + XCTAssertEqual(det0.classId, 1) + XCTAssertEqual(det0.confidence, 0.95, accuracy: 0.001) + } + + func testDetectionResultCreation() { + let result = DetectionResult( + x: 0.5, + y: 0.5, + width: 0.2, + height: 0.3, + classId: 1, + confidence: 0.9 + ) + + XCTAssertEqual(result.x, 0.5) + XCTAssertEqual(result.y, 0.5) + XCTAssertEqual(result.width, 0.2) + XCTAssertEqual(result.height, 0.3) + XCTAssertEqual(result.classId, 1) + XCTAssertEqual(result.confidence, 0.9) + } + + func testDetectionResultArrayCreation() { + let results = [ + DetectionResult(x: 0.1, y: 0.2, width: 0.3, height: 0.4, classId: 1, confidence: 0.9), + DetectionResult(x: 0.5, y: 0.6, width: 0.2, height: 0.2, classId: 2, confidence: 0.8), + ] + + let array = DetectionResultArray(results: results) + XCTAssertEqual(array.count(), 2) + + let first = array.get(index: 0) + XCTAssertEqual(first.x, 0.1, accuracy: 0.001) + XCTAssertEqual(first.classId, 1) + } +} diff --git a/photon-apple/build.gradle b/photon-apple/build.gradle new file mode 100644 index 0000000000..47bd277181 --- /dev/null +++ b/photon-apple/build.gradle @@ -0,0 +1,126 @@ +plugins { + id 'java-library' +} + +def isMacOS = System.getProperty("os.name").toLowerCase().contains("mac") +def forceStubs = project.hasProperty('photonAppleForceStubs') + +// Configure Java version based on build mode +if (isMacOS && !forceStubs) { + // macOS real impl: Declare as Java 24 compatible for dependency resolution + // (even though SwiftKit deps require Java 25 at runtime) + java { + sourceCompatibility = JavaVersion.VERSION_24 + targetCompatibility = JavaVersion.VERSION_24 + } + + // Override dependency resolution attributes to declare Java 24 compatibility + // This allows photon-server (Java 24) to depend on photon-apple, even though + // SwiftKit dependencies require Java 25 at runtime + configurations.configureEach { + attributes { + attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 24) + } + resolutionStrategy { + // Disable strict Java version checking for dependency resolution + capabilitiesResolution.all { it.selectHighestVersion() } + } + } +} else { + // PhotonLib stubs: Use Java 17 for FRC compatibility + java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } +} + + +if (isMacOS && !forceStubs) { + // macOS with real implementation: Build Swift library and use real implementations + sourceSets { + main { + java { + srcDirs = [ + 'src/main/java', + '.build/plugins/outputs/photon-apple/PhotonAppleLibrary/destination/JExtractSwiftPlugin/src/generated/java' + ] + } + } + } + + dependencies { + testImplementation 'org.junit.jupiter:junit-jupiter:5.11.4' + + // SwiftKit dependencies (only on macOS) + // Note: These require Java 25 at runtime but we allow them during Java 24 dependency resolution + api 'org.swift.swiftkit:swiftkit-core:1.0-SNAPSHOT' + api 'org.swift.swiftkit:swiftkit-ffm:1.0-SNAPSHOT' + } + + // Task to build Swift library and generate Java bindings + task buildSwift(type: Exec) { + commandLine 'swift', 'build', '-c', 'release' + workingDir projectDir + } + + // Package Swift dylibs into JAR resources for portable deployment + processResources { + dependsOn buildSwift + + // Package all Swift dylibs from local build + def buildDir = "${projectDir}/.build/release" + from(buildDir) { + include 'libPhotonAppleLibrary.dylib' + include 'libSwiftJava.dylib' + include 'libSwiftRuntimeFunctions.dylib' + // Package at root of JAR for SwiftLibraries.loadResourceLibrary() to find + } + } + + // Ensure Swift library is built before compiling Java + tasks.compileJava.dependsOn(buildSwift) + + // Use Java 25 for compilation (SwiftKit requires Java 25) + tasks.withType(JavaCompile).configureEach { + javaCompiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(25) + } + } + + test { + useJUnitPlatform() + // Skip tests by default (requires macOS and CoreML model) + enabled = project.hasProperty('runAppleTests') + } +} else { + // Linux/Windows: Use stub implementations, skip Swift build + sourceSets { + main { + java { + srcDirs = [ + 'src/stub/java'] // Use stubs instead + } + } + // Don't compile tests on non-macOS (they require Java 25 APIs on macOS) + test { + java { + srcDirs = [] + } + } + } + + dependencies { + testImplementation 'org.junit.jupiter:junit-jupiter:5.11.4' + // No SwiftKit dependencies on non-macOS + } + + test { + useJUnitPlatform() + enabled = false // Can't run tests without real implementation + } +} + +repositories { + mavenCentral() + mavenLocal() +} diff --git a/photon-apple/src/main/java/org/photonvision/apple/MemorySegmentCompat.java b/photon-apple/src/main/java/org/photonvision/apple/MemorySegmentCompat.java new file mode 100644 index 0000000000..4ab4c63178 --- /dev/null +++ b/photon-apple/src/main/java/org/photonvision/apple/MemorySegmentCompat.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +import java.lang.foreign.MemorySegment; + +/** + * Compatibility wrapper for MemorySegment to handle API differences between Java versions. + * + *

Java 24+ uses java.lang.foreign.MemorySegment (finalized API). On non-macOS platforms, this + * class is stubbed since foreign memory access is only needed for Swift interop. + */ +public class MemorySegmentCompat { + private MemorySegmentCompat() {} + + /** + * Create a MemorySegment from a native address. + * + * @param address The native memory address + * @return A MemorySegment wrapping the given address + */ + public static MemorySegment ofAddress(long address) { + return MemorySegment.ofAddress(address); + } +} diff --git a/photon-apple/src/main/java/org/photonvision/apple/SwiftArena.java b/photon-apple/src/main/java/org/photonvision/apple/SwiftArena.java new file mode 100644 index 0000000000..8bd90e7749 --- /dev/null +++ b/photon-apple/src/main/java/org/photonvision/apple/SwiftArena.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +import org.swift.swiftkit.ffm.AllocatingSwiftArena; + +/** + * Wrapper around SwiftKit's AllocatingSwiftArena to avoid direct SwiftKit imports in photon-core. + * + *

This allows photon-core to use Swift arenas without depending on SwiftKit directly. On + * non-macOS platforms, this class is stubbed and throws UnsupportedOperationException. + */ +public class SwiftArena implements AutoCloseable { + private final AllocatingSwiftArena arena; + + private SwiftArena(AllocatingSwiftArena arena) { + this.arena = arena; + } + + /** + * Create an automatic arena that is cleaned up by the garbage collector. + * + *

Use this for long-lived arenas like detector-scoped allocations. + */ + public static SwiftArena ofAuto() { + return new SwiftArena(AllocatingSwiftArena.ofAuto()); + } + + /** + * Create a confined arena that must be explicitly closed. + * + *

Use this for short-lived arenas like frame-scoped allocations in try-with-resources blocks. + */ + public static SwiftArena ofConfined() { + return new SwiftArena(AllocatingSwiftArena.ofConfined()); + } + + /** + * Get the underlying AllocatingSwiftArena for use with Swift bindings. + * + *

This is used internally when passing the arena to Swift-generated code. + */ + public AllocatingSwiftArena unwrap() { + return arena; + } + + /** + * Close this arena (only for confined arenas). + * + *

For ofAuto() arenas, this is a no-op. For ofConfined() arenas, this would close the arena if + * AllocatingSwiftArena supported it. Currently, confined arenas are also automatically cleaned + * up. + */ + @Override + public void close() { + // AllocatingSwiftArena doesn't have a close method + // Both ofAuto() and ofConfined() are automatically cleaned up + } +} diff --git a/photon-apple/src/stub/java/org/photonvision/apple/DetectionResult.java b/photon-apple/src/stub/java/org/photonvision/apple/DetectionResult.java new file mode 100644 index 0000000000..d10b31d31f --- /dev/null +++ b/photon-apple/src/stub/java/org/photonvision/apple/DetectionResult.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +/** + * Stub implementation of DetectionResult for non-macOS platforms. This class is generated from + * Swift code on macOS. + */ +public final class DetectionResult { + private DetectionResult() {} + + public static DetectionResult init( + double x, + double y, + double width, + double height, + int classId, + double confidence, + Object swiftArena) { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public double getX() { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public double getY() { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public double getWidth() { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public double getHeight() { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public int getClassId() { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public double getConfidence() { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } +} diff --git a/photon-apple/src/stub/java/org/photonvision/apple/DetectionResultArray.java b/photon-apple/src/stub/java/org/photonvision/apple/DetectionResultArray.java new file mode 100644 index 0000000000..b41f07ddc6 --- /dev/null +++ b/photon-apple/src/stub/java/org/photonvision/apple/DetectionResultArray.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +/** + * Stub implementation of DetectionResultArray for non-macOS platforms. This class is generated from + * Swift code on macOS. + */ +public final class DetectionResultArray { + private DetectionResultArray() {} + + public long count() { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public DetectionResult get(long index, Object swiftArena) { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } +} diff --git a/photon-apple/src/stub/java/org/photonvision/apple/MemorySegmentCompat.java b/photon-apple/src/stub/java/org/photonvision/apple/MemorySegmentCompat.java new file mode 100644 index 0000000000..4fc07ceb8d --- /dev/null +++ b/photon-apple/src/stub/java/org/photonvision/apple/MemorySegmentCompat.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +/** + * Stub implementation of MemorySegmentCompat for non-macOS platforms. + * + *

This class is never called on non-macOS platforms since Apple object detection is not + * supported. + */ +public class MemorySegmentCompat { + private MemorySegmentCompat() {} + + /** + * Stub method that throws UnsupportedOperationException. + * + * @param address The native memory address (unused) + * @return Never returns + * @throws UnsupportedOperationException Always thrown on non-macOS platforms + */ + public static Object ofAddress(long address) { + throw new UnsupportedOperationException( + "MemorySegment is only supported on macOS for Swift interop"); + } +} diff --git a/photon-apple/src/stub/java/org/photonvision/apple/ObjectDetector.java b/photon-apple/src/stub/java/org/photonvision/apple/ObjectDetector.java new file mode 100644 index 0000000000..dbb8536e77 --- /dev/null +++ b/photon-apple/src/stub/java/org/photonvision/apple/ObjectDetector.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +/** + * Stub implementation of ObjectDetector for non-macOS platforms. This class is generated from Swift + * code on macOS. + */ +public final class ObjectDetector { + private ObjectDetector() {} + + public static ObjectDetector init(String modelPath, Object swiftArena) { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public DetectionResultArray detect( + Object imageData, long width, long height, double boxThreshold, Object swiftArena) { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } + + public DetectionResultArray detectFake( + Object imageData, + long width, + long height, + int pixelFormat, + double boxThreshold, + double nmsThreshold, + Object swiftArena) { + throw new UnsupportedOperationException("Apple object detection is only supported on macOS"); + } +} diff --git a/photon-apple/src/stub/java/org/photonvision/apple/PhotonAppleLibrary.java b/photon-apple/src/stub/java/org/photonvision/apple/PhotonAppleLibrary.java new file mode 100644 index 0000000000..daaa793211 --- /dev/null +++ b/photon-apple/src/stub/java/org/photonvision/apple/PhotonAppleLibrary.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +/** + * Stub implementation of PhotonAppleLibrary for non-macOS platforms. This class is generated from + * Swift code on macOS. + */ +public final class PhotonAppleLibrary { + private PhotonAppleLibrary() {} +} diff --git a/photon-apple/src/stub/java/org/photonvision/apple/SwiftArena.java b/photon-apple/src/stub/java/org/photonvision/apple/SwiftArena.java new file mode 100644 index 0000000000..5c43e6b6dd --- /dev/null +++ b/photon-apple/src/stub/java/org/photonvision/apple/SwiftArena.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +/** + * Stub implementation of SwiftArena for non-macOS platforms. + * + *

This class is never instantiated on non-macOS platforms since Apple object detection is not + * supported. + */ +public class SwiftArena implements AutoCloseable { + private SwiftArena() {} + + public static SwiftArena ofAuto() { + throw new UnsupportedOperationException("SwiftArena is only supported on macOS"); + } + + public static SwiftArena ofConfined() { + throw new UnsupportedOperationException("SwiftArena is only supported on macOS"); + } + + public Object unwrap() { + throw new UnsupportedOperationException("SwiftArena is only supported on macOS"); + } + + @Override + public void close() { + // No-op stub + } +} diff --git a/photon-apple/src/test/java/org/photonvision/apple/ObjectDetectorTest.java b/photon-apple/src/test/java/org/photonvision/apple/ObjectDetectorTest.java new file mode 100644 index 0000000000..44b11780a0 --- /dev/null +++ b/photon-apple/src/test/java/org/photonvision/apple/ObjectDetectorTest.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.apple; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.foreign.MemorySegment; +import org.junit.jupiter.api.Test; +import org.swift.swiftkit.ffm.AllocatingSwiftArena; + +/** Tests for ObjectDetector functionality */ +class ObjectDetectorTest { + @Test + void test_ObjectDetector_detectFake_returnsSyntheticResults() { + // Test that we can successfully receive DetectionResult data from Swift + // This validates the Swift→Java data passing without requiring a CoreML model + + try (var arena = AllocatingSwiftArena.ofConfined()) { + // Create detector (doesn't need a real model for detectFake) + ObjectDetector detector = ObjectDetector.init("/fake/path/model.mlmodel", arena); + assertNotNull(detector); + + // Create a dummy BGRA image + try (var frameArena = AllocatingSwiftArena.ofConfined()) { + int width = 640; + int height = 480; + int totalBytes = height * width * 4; + MemorySegment imageData = frameArena.allocate(totalBytes, 1); + + // Call detectFake which returns synthetic test data + DetectionResultArray results = + detector.detectFake( + imageData, + 640L, + 480L, + 2, // BGRA format + 0.5, + 0.4, + frameArena); + + // Validate we got the expected synthetic results + assertNotNull(results); + assertEquals(3, results.count(), "Should return 3 fake detection results"); + + // Validate first detection result + DetectionResult det0 = results.get(0, frameArena); + assertNotNull(det0); + + assertEquals(0.1, det0.getX(), 0.001, "First detection X coordinate"); + assertEquals(0.2, det0.getY(), 0.001, "First detection Y coordinate"); + assertEquals(0.3, det0.getWidth(), 0.001, "First detection width"); + assertEquals(0.4, det0.getHeight(), 0.001, "First detection height"); + assertEquals(1, det0.getClassId(), "First detection class ID"); + assertEquals(0.95, det0.getConfidence(), 0.001, "First detection confidence"); + } + } + } +} diff --git a/photon-apple/swift-java b/photon-apple/swift-java new file mode 160000 index 0000000000..3ddd7f090f --- /dev/null +++ b/photon-apple/swift-java @@ -0,0 +1 @@ +Subproject commit 3ddd7f090f46f873570d29bb8d9025c343d225ca diff --git a/photon-core/build.gradle b/photon-core/build.gradle index a8a3c972f4..c87f2eea76 100644 --- a/photon-core/build.gradle +++ b/photon-core/build.gradle @@ -63,6 +63,8 @@ dependencies { transitive = false } + implementation project(':photon-apple') + testImplementation group: 'org.junit-pioneer' , name: 'junit-pioneer', version: '2.2.0' } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java index b1df50fa44..57a0b8188b 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java @@ -38,6 +38,7 @@ import org.photonvision.common.hardware.Platform; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; +import org.photonvision.vision.objects.CoremlModel; import org.photonvision.vision.objects.Model; import org.photonvision.vision.objects.RknnModel; import org.photonvision.vision.objects.RubikModel; @@ -201,6 +202,7 @@ private NeuralNetworkModelManager() { switch (Platform.getCurrentPlatform()) { case LINUX_QCS6490 -> supportedBackends.add(Family.RUBIK); case LINUX_RK3588_64 -> supportedBackends.add(Family.RKNN); + case MACOS -> supportedBackends.add(Family.COREML); default -> { logger.warn( "No supported neural network backends found for this platform: " @@ -239,7 +241,8 @@ public static NeuralNetworkModelManager getInstance(boolean reset) { public enum Family { RKNN(".rknn"), - RUBIK(".tflite"); + RUBIK(".tflite"), + COREML(".mlmodel"); private final String fileExtension; @@ -308,7 +311,12 @@ public Optional getDefaultModel() { return Optional.empty(); } - return models.get(supportedBackends.get(0)).stream().findFirst(); + ArrayList backendModels = models.get(supportedBackends.get(0)); + if (backendModels == null) { + return Optional.empty(); + } + + return backendModels.stream().findFirst(); } // Do checking later on, when we create the model object @@ -362,6 +370,9 @@ private void loadModel(Path path) { case RUBIK -> { models.get(properties.family()).add(new RubikModel(properties)); } + case COREML -> { + models.get(properties.family()).add(new CoremlModel(properties)); + } } logger.info( "Loaded model " @@ -395,9 +406,19 @@ public void discoverModels() { files .filter(Files::isRegularFile) .filter( - path -> - supportedBackends.stream() - .anyMatch(family -> path.toString().endsWith(family.extension()))) + path -> { + String pathStr = path.toString(); + return supportedBackends.stream() + .anyMatch( + family -> { + // For APPLE, also check for .mlmodelc (compiled models) + if (family == Family.COREML) { + return pathStr.endsWith(family.extension()) + || pathStr.endsWith(".mlmodelc"); + } + return pathStr.endsWith(family.extension()); + }); + }) .forEach(this::loadModel); } catch (IOException e) { logger.error("Failed to discover models at " + modelsDirectory.getAbsolutePath(), e); diff --git a/photon-core/src/main/java/org/photonvision/vision/objects/CoremlModel.java b/photon-core/src/main/java/org/photonvision/vision/objects/CoremlModel.java new file mode 100644 index 0000000000..032ecc961c --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/objects/CoremlModel.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.objects; + +import java.io.File; +import org.opencv.core.Size; +import org.photonvision.common.configuration.NeuralNetworkModelManager.Family; +import org.photonvision.common.configuration.NeuralNetworkPropertyManager.ModelProperties; + +/** Manages an Apple CoreML model for object detection using the Apple Vision framework. */ +public class CoremlModel implements Model { + public final File modelFile; + public final ModelProperties properties; + + /** + * Creates a new AppleModel from the given properties. + * + * @param properties The properties to create the model from + * @throws IllegalArgumentException If the properties are invalid + */ + public CoremlModel(ModelProperties properties) throws IllegalArgumentException { + // Validate: file must exist + if (!new File(properties.modelPath().toString()).exists()) { + throw new IllegalArgumentException("Model file does not exist: " + properties.modelPath()); + } + + // Validate: must have labels + if (properties.labels() == null || properties.labels().isEmpty()) { + throw new IllegalArgumentException( + "CoremlModel properties must have labels: " + properties.modelPath()); + } + + // Validate: must be APPLE family + if (properties.family() != Family.COREML) { + throw new IllegalArgumentException( + "CoremlModel properties must have family COREML: " + properties.family()); + } + + this.modelFile = new File(properties.modelPath().toString()); + this.properties = properties; + } + + @Override + public ObjectDetector load() { + return new CoremlObjectDetector( + this, new Size(properties.resolutionWidth(), properties.resolutionHeight())); + } + + public ModelProperties getModel() { + return properties; + } + + @Override + public String getUID() { + return properties.modelPath().toString(); + } + + @Override + public String getNickname() { + return properties.nickname(); + } + + @Override + public Family getFamily() { + return properties.family(); + } + + @Override + public ModelProperties getProperties() { + return properties; + } + + @Override + public String toString() { + return "CoremlModel{" + + "modelFile=" + + modelFile + + ", nickname='" + + properties.nickname() + + '\'' + + ", version=" + + properties.version() + + '}'; + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/objects/CoremlObjectDetector.java b/photon-core/src/main/java/org/photonvision/vision/objects/CoremlObjectDetector.java new file mode 100644 index 0000000000..25cadd9551 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/objects/CoremlObjectDetector.java @@ -0,0 +1,159 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.objects; + +import java.util.ArrayList; +import java.util.List; +import org.opencv.core.Mat; +import org.opencv.core.Rect2d; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.photonvision.apple.MemorySegmentCompat; +import org.photonvision.apple.SwiftArena; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult; + +/** + * Manages an object detector using Apple's CoreML and Vision framework backend. + * + *

This detector uses Apple's Vision framework which provides automatic image preprocessing + * (resizing, cropping) and built-in NMS (Non-Maximum Suppression). + */ +public class CoremlObjectDetector implements ObjectDetector { + private static final Logger logger = new Logger(CoremlObjectDetector.class, LogGroup.General); + + private final SwiftArena arena; + + /** Swift ObjectDetector instance */ + private final org.photonvision.apple.ObjectDetector swiftDetector; + + private final CoremlModel model; + + /** Returns the model in use by this detector. */ + @Override + public CoremlModel getModel() { + return model; + } + + /** + * Creates a new AppleObjectDetector from the given model. + * + * @param model The model to create the detector from. + * @param inputSize unused - The required image dimensions for the model + */ + public CoremlObjectDetector(CoremlModel model, Size inputSize) { + this.model = model; + + // ofAuto() instead of ofConfined() to allow access from VisionRunner thread + this.arena = SwiftArena.ofAuto(); + + // Initialize Swift ObjectDetector + this.swiftDetector = + org.photonvision.apple.ObjectDetector.init( + model.modelFile.getAbsolutePath(), arena.unwrap()); + + logger.info("Created AppleObjectDetector for model: " + model.getNickname()); + } + + @Override + public List getClasses() { + return model.properties.labels(); + } + + /** + * Detects objects in the given input image. + * + *

Note: Vision framework performs its own image preprocessing, so we only need to convert to + * BGRA format. + * + * @param in The input image to perform object detection on. + * @param nmsThresh The threshold value for NMS (unused - Vision framework handles NMS + * internally). + * @param boxThresh The threshold value for bounding box confidence. + * @return A list of NeuralNetworkPipeResult objects + */ + @Override + public List detect(Mat in, double nmsThresh, double boxThreshold) { + try (var frameArena = SwiftArena.ofConfined()) { + // Convert to BGRA format (required by Swift side) + Mat bgra = new Mat(); + var matFilled = false; + + if (in.channels() == 1) { + Imgproc.cvtColor(in, bgra, Imgproc.COLOR_GRAY2BGRA); + matFilled = true; + } else if (in.channels() == 3) { + Imgproc.cvtColor(in, bgra, Imgproc.COLOR_BGR2BGRA); + matFilled = true; + } else if (in.channels() == 4) { + bgra = in; + } else { + logger.error( + "AppleObjectDetector: Input image has unsupported number of channels: " + + in.channels()); + return new ArrayList<>(); + } + + var mem = MemorySegmentCompat.ofAddress(bgra.dataAddr()); + var width = in.width(); + var height = in.height(); + + var results = swiftDetector.detect(mem, width, height, boxThreshold, frameArena.unwrap()); + + List detections = new ArrayList<>(); + long count = results.count(); + + for (int i = 0; i < count; i++) { + var detection = results.get(i, frameArena.unwrap()); + + double w = detection.getWidth() * width; + double h = detection.getHeight() * height; + double x = detection.getX() * width - w / 2.0; + double y = detection.getY() * height - h / 2.0; + + Rect2d bbox = new Rect2d(x, y, w, h); + + detections.add( + new NeuralNetworkPipeResult( + bbox, (int) detection.getClassId(), detection.getConfidence())); + } + + if (matFilled) { + bgra.release(); + } + + return detections; + + } catch (Exception e) { + logger.error("Error during Apple object detection", e); + return new ArrayList<>(); + } + } + + /** + * Releases the detector and cleans up native resources. + * + *

This method is thread-safe and can be called multiple times. + */ + @Override + public void release() { + // ofConfined arenas are already closed via try-with-resources + // ofAuto is cleaned up by the GC + } +} diff --git a/photon-server/build.gradle b/photon-server/build.gradle index 6269bb8efd..ebab36a094 100644 --- a/photon-server/build.gradle +++ b/photon-server/build.gradle @@ -6,6 +6,7 @@ apply from: "${rootDir}/shared/common.gradle" dependencies { implementation project(':photon-core') + implementation project(':photon-apple') // Zip implementation 'org.zeroturnaround:zt-zip:1.14' @@ -64,6 +65,7 @@ processResources { run { environment "PATH_PREFIX", "../" + if (project.hasProperty("profile")) { jvmArgs=[ "-Dcom.sun.management.jmxremote=true", diff --git a/photon-server/src/main/java/org/photonvision/server/RequestHandler.java b/photon-server/src/main/java/org/photonvision/server/RequestHandler.java index c6ed9b8d24..db92bd5e0a 100644 --- a/photon-server/src/main/java/org/photonvision/server/RequestHandler.java +++ b/photon-server/src/main/java/org/photonvision/server/RequestHandler.java @@ -58,6 +58,7 @@ import org.photonvision.vision.calibration.CameraCalibrationCoefficients; import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.camera.PVCameraInfo; +import org.photonvision.vision.objects.CoremlModel; import org.photonvision.vision.objects.ObjectDetector; import org.photonvision.vision.objects.RknnModel; import org.photonvision.vision.objects.RubikModel; @@ -624,6 +625,9 @@ public static void onImportObjectDetectionModelRequest(Context ctx) { case LINUX_RK3588_64: family = NeuralNetworkModelManager.Family.RKNN; break; + case MACOS: + family = NeuralNetworkModelManager.Family.COREML; + break; default: ctx.status(400); ctx.result("The current platform does not support object detection models"); @@ -680,6 +684,7 @@ public static void onImportObjectDetectionModelRequest(Context ctx) { switch (family) { case RUBIK -> new RubikModel(modelProperties).load(); case RKNN -> new RknnModel(modelProperties).load(); + case COREML -> new CoremlModel(modelProperties).load(); }; } catch (RuntimeException e) { ctx.status(400); diff --git a/photon-targeting/src/main/java/org/photonvision/common/hardware/Platform.java b/photon-targeting/src/main/java/org/photonvision/common/hardware/Platform.java index 65c1970ba9..fd18ed5b84 100644 --- a/photon-targeting/src/main/java/org/photonvision/common/hardware/Platform.java +++ b/photon-targeting/src/main/java/org/photonvision/common/hardware/Platform.java @@ -65,9 +65,11 @@ public enum Platform { LINUX_ARM64( "Linux ARM64", Platform::getLinuxDeviceTreeModel, false, OSType.LINUX, true), // ODROID C2, N2 + // Bleeding Edge - Partial support for development, no official support for installed use + MACOS("Mac OS", Platform::getUnknownModel, false, OSType.MACOS, true), + // Completely unsupported WINDOWS_32("Windows x86", Platform::getUnknownModel, false, OSType.WINDOWS, false), - MACOS("Mac OS", Platform::getUnknownModel, false, OSType.MACOS, false), LINUX_ARM32( "Linux ARM32", Platform::getUnknownModel, false, OSType.LINUX, false), // ODROID XU4, C1+ UNKNOWN("Unsupported Platform", Platform::getUnknownModel, false, OSType.UNKNOWN, false); diff --git a/settings.gradle b/settings.gradle index 30f98b2fb4..9ceb4084d5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,3 +15,6 @@ include 'photon-core' include 'photon-server' include 'photon-lib' include 'photon-docs' + +// Include photon-apple on all platforms (uses stubs on non-macOS) +include 'photon-apple' diff --git a/shared/common.gradle b/shared/common.gradle index a1dd8e3bf7..62f4afd83c 100644 --- a/shared/common.gradle +++ b/shared/common.gradle @@ -3,8 +3,9 @@ apply plugin: "java" apply plugin: "jacoco" java { - sourceCompatibility = JavaVersion.VERSION_17 - targetCompatibility = JavaVersion.VERSION_17 + // Use whatever Java version is running Gradle (Java 17 on most CI, Java 24 on macOS CI) + sourceCompatibility = JavaVersion.current() + targetCompatibility = JavaVersion.current() } wpilibTools.deps.wpilibVersion = wpilibVersion @@ -64,7 +65,7 @@ test { } jacoco { - toolVersion = "0.8.10" + toolVersion = "0.8.13" reportsDirectory = layout.buildDirectory.dir('customJacocoReportDir') } diff --git a/shared/javacommon.gradle b/shared/javacommon.gradle index 7ccbaaa974..030f221d26 100644 --- a/shared/javacommon.gradle +++ b/shared/javacommon.gradle @@ -152,7 +152,7 @@ dependencies { } jacoco { - toolVersion = "0.8.10" + toolVersion = "0.8.13" } jacocoTestReport {