diff --git a/RealityMixer/Calibration/Builders/CalibrationBuilder.swift b/RealityMixer/Calibration/Builders/CalibrationBuilder.swift index 3c2d94a..c1d3d07 100644 --- a/RealityMixer/Calibration/Builders/CalibrationBuilder.swift +++ b/RealityMixer/Calibration/Builders/CalibrationBuilder.swift @@ -10,12 +10,10 @@ import ARKit struct CalibrationBuilder { - static func fov(from frame: ARFrame) -> (Float, Float) { - let projection = frame.camera.projectionMatrix + static func fov(projectionMatrix projection: simd_float4x4, imageResolution: CGSize) -> (Float, Float) { let yScale = projection[1,1] let yFov = 2 * atan(1/yScale) - let imageResolution = frame.camera.imageResolution let xFov = yFov * Float(imageResolution.width / imageResolution.height) return (xFov, yFov) } @@ -61,10 +59,10 @@ struct CalibrationBuilder { rightControllerPosition: Vector3, rightControllerScreenCoordinates: CGPoint, centerPose: Pose, - frame: ARFrame + imageResolution: CGSize, + projectionMatrix: simd_float4x4 ) -> (SCNMatrix4, CalibrationResult) { - let imageResolution = frame.camera.imageResolution - let (xFov, yFov) = fov(from: frame) + let (xFov, yFov) = fov(projectionMatrix: projectionMatrix, imageResolution: imageResolution) let anglePerVerticalPixel = yFov/Float(imageResolution.height) let anglePerHorizontalPixel = xFov/Float(imageResolution.width) diff --git a/RealityMixer/Calibration/ViewControllers/CalibrationViewController.swift b/RealityMixer/Calibration/ViewControllers/CalibrationViewController.swift index ea39e74..a344981 100644 --- a/RealityMixer/Calibration/ViewControllers/CalibrationViewController.swift +++ b/RealityMixer/Calibration/ViewControllers/CalibrationViewController.swift @@ -165,7 +165,21 @@ final class CalibrationViewController: UIViewController { saveButtonContainer.isHidden = true updateInfo("Step 3 of 4") + let viewPortSize = sceneView.bounds.size + let interfaceOrientation = UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? UIInterfaceOrientation.unknown + let transform = frame.displayTransform(for: interfaceOrientation, viewportSize: viewPortSize).inverted() + let ciImage = CIImage(cvImageBuffer: frame.capturedImage).transformed(by: transform) + let projectionMatrix = frame.camera.projectionMatrix(for: interfaceOrientation, viewportSize: viewPortSize, zNear: 0.001, zFar: 1000.0) + + let context = CIContext(options: nil) + guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else { + return + } + let uiImage = UIImage(cgImage: cgImage) + let viewController = ProjectionViewController( + uiImage: uiImage, + projectionMatrix: projectionMatrix, scaleFactor: scaleFactor, cameraOrigin: cameraOrigin, rightControllerPosition: rightControllerPosition, diff --git a/RealityMixer/Calibration/ViewControllers/ProjectionPickerViewController.swift b/RealityMixer/Calibration/ViewControllers/ProjectionPickerViewController.swift index d7ad2cb..72f2871 100644 --- a/RealityMixer/Calibration/ViewControllers/ProjectionPickerViewController.swift +++ b/RealityMixer/Calibration/ViewControllers/ProjectionPickerViewController.swift @@ -20,16 +20,17 @@ protocol ProjectionPickerViewControllerDelegate: AnyObject { final class ProjectionPickerViewController: UIViewController { weak var delegate: ProjectionPickerViewControllerDelegate? + + private let image: UIImage + private let projectionMatrix: simd_float4x4 + private let scaleFactor: Double private let cameraOrigin: Vector3 private let rightControllerPosition: Vector3 private let frame: ARFrame private let lastPoseUpdate: PoseUpdate - private var image: UIImage { - UIImage(ciImage: CIImage(cvImageBuffer: frame.capturedImage)) - } - + @IBOutlet private weak var imageView: UIImageView! @IBOutlet private weak var sceneOverlay: SCNView! @IBOutlet private weak var blueView: UIView! @@ -55,12 +56,16 @@ final class ProjectionPickerViewController: UIViewController { private var first = true init( + uiImage: UIImage, + projectionMatrix: simd_float4x4, scaleFactor: Double, cameraOrigin: Vector3, rightControllerPosition: Vector3, frame: ARFrame, lastPoseUpdate: PoseUpdate ) { + self.image = uiImage + self.projectionMatrix = projectionMatrix self.scaleFactor = scaleFactor self.cameraOrigin = cameraOrigin self.rightControllerPosition = rightControllerPosition @@ -100,20 +105,20 @@ final class ProjectionPickerViewController: UIViewController { let camera = SCNCamera() camera.zNear = 0.1 camera.zFar = 100.0 - let (xFov, yFov) = CalibrationBuilder.fov(from: frame) + let (xFov, yFov) = CalibrationBuilder.fov(projectionMatrix: projectionMatrix, imageResolution: image.size) let imageViewRatio = imageView.frame.size.width/imageView.frame.size.height - let imageRatio = frame.camera.imageResolution.width/frame.camera.imageResolution.height + let imageRatio = image.size.width/image.size.height if imageViewRatio > imageRatio { - let imageHeightInImageViewCoordinates = frame.camera.imageResolution.height * (imageView.frame.size.width/frame.camera.imageResolution.width) + let imageHeightInImageViewCoordinates = image.size.height * (imageView.frame.size.width/image.size.width) let distanceInImageViewCoordinates = (imageHeightInImageViewCoordinates * 0.5)/CGFloat(tan(yFov/2.0)) let adjustedYFov = CGFloat(2.0 * atan2((imageView.frame.size.height * 0.5), distanceInImageViewCoordinates)) camera.projectionDirection = .vertical camera.fieldOfView = (adjustedYFov * (180.0/CGFloat.pi)) } else { - let imageWidthInImageViewCoordinates = frame.camera.imageResolution.width * (imageView.frame.size.height/frame.camera.imageResolution.height) + let imageWidthInImageViewCoordinates = image.size.width * (imageView.frame.size.height/image.size.height) let distanceInImageViewCoordinates = (imageWidthInImageViewCoordinates * 0.5)/CGFloat(tan(xFov/2.0)) let adjustedXFov = CGFloat(2.0 * atan2((imageView.frame.size.width * 0.5), distanceInImageViewCoordinates)) @@ -189,7 +194,8 @@ final class ProjectionPickerViewController: UIViewController { rightControllerPosition: rightControllerPosition, rightControllerScreenCoordinates: pixelCoordinate(from: blueViewCenter), centerPose: lastPoseUpdate.trackingTransformRaw, - frame: frame + imageResolution: image.size, + projectionMatrix: projectionMatrix ) mainNode?.transform = calibration.0 diff --git a/RealityMixer/Calibration/ViewControllers/ProjectionViewController.swift b/RealityMixer/Calibration/ViewControllers/ProjectionViewController.swift index 018d248..169e94e 100644 --- a/RealityMixer/Calibration/ViewControllers/ProjectionViewController.swift +++ b/RealityMixer/Calibration/ViewControllers/ProjectionViewController.swift @@ -26,6 +26,8 @@ final class ProjectionViewController: UIViewController { private var projectionPickerViewController: ProjectionPickerViewController init( + uiImage: UIImage, + projectionMatrix: simd_float4x4, scaleFactor: Double, cameraOrigin: Vector3, rightControllerPosition: Vector3, @@ -34,6 +36,8 @@ final class ProjectionViewController: UIViewController { delegate: ProjectionViewControllerDelegate ) { self.projectionPickerViewController = ProjectionPickerViewController( + uiImage: uiImage, + projectionMatrix: projectionMatrix, scaleFactor: scaleFactor, cameraOrigin: cameraOrigin, rightControllerPosition: rightControllerPosition, diff --git a/RealityMixer/Capture/Misc/ARKitHelpers.swift b/RealityMixer/Capture/Misc/ARKitHelpers.swift index 2ce32ff..ee53ebc 100644 --- a/RealityMixer/Capture/Misc/ARKitHelpers.swift +++ b/RealityMixer/Capture/Misc/ARKitHelpers.swift @@ -10,10 +10,8 @@ import ARKit struct ARKitHelpers { // FIXME: Check this. - static func planeSizeForDistance(_ distance: Float, frame: ARFrame) -> CGSize { - let projection = frame.camera.projectionMatrix + static func planeSizeForDistance(_ distance: Float, imageResolution: CGSize, projection: simd_float4x4) -> CGSize { let yScale = projection[1,1] - let imageResolution = frame.camera.imageResolution let width = (2.0 * distance) * tan(atan(1/yScale) * Float(imageResolution.width / imageResolution.height)) let height = width * Float(imageResolution.height / imageResolution.width) return CGSize(width: CGFloat(width), height: CGFloat(height)) @@ -30,8 +28,8 @@ struct ARKitHelpers { return planeNode } - static func makePlaneNodeForDistance(_ distance: Float, frame: ARFrame) -> SCNNode { - makePlane(size: planeSizeForDistance(distance, frame: frame), distance: distance) + static func makePlaneNodeForDistance(_ distance: Float, viewPortSize: CGSize, projectionMatrix: simd_float4x4) -> SCNNode { + makePlane(size: planeSizeForDistance(distance, imageResolution: viewPortSize, projection: projectionMatrix), distance: distance) } @discardableResult diff --git a/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift b/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift index 4c6574c..170cc30 100644 --- a/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift +++ b/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift @@ -109,9 +109,9 @@ final class ChromaKeyConfigurationViewController: UIViewController { updateValueLabels() } - private func configureBackgroundPlane(with frame: ARFrame) { + private func configureBackgroundPlane(viewPortSize: CGSize, projectionMatrix: simd_float4x4) { let distance: Float = 100 - let backgroundPlaneSize = ARKitHelpers.planeSizeForDistance(distance, frame: frame) + let backgroundPlaneSize = ARKitHelpers.planeSizeForDistance(distance, imageResolution: viewPortSize, projection: projectionMatrix) let backgroundPlaneNode = ARKitHelpers.makePlane(size: backgroundPlaneSize, distance: distance) let planeMaterial = backgroundPlaneNode.geometry?.firstMaterial @@ -129,8 +129,8 @@ final class ChromaKeyConfigurationViewController: UIViewController { self.backgroundPlaneNode = backgroundPlaneNode } - private func configurePlane(with frame: ARFrame) { - let planeNode = ARKitHelpers.makePlaneNodeForDistance(0.1, frame: frame) + private func configurePlane(viewPortSize: CGSize, projectionMatrix: simd_float4x4) { + let planeNode = ARKitHelpers.makePlaneNodeForDistance(0.1, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) planeNode.geometry?.firstMaterial?.lightingModel = .constant planeNode.geometry?.firstMaterial?.transparencyMode = .rgbZero planeNode.geometry?.firstMaterial?.shaderModifiers = [.surface: Shaders.surfaceChromaKeyConfiguration()] @@ -248,6 +248,7 @@ final class ChromaKeyConfigurationViewController: UIViewController { return } + // FIXME: Image is rotated! self.maskImage = ChromaKeyMaskBuilder.buildMask(for: currentFrame, chromaConfiguration: currentConfiguration()) } else { self.maskImage = nil @@ -284,8 +285,12 @@ extension ChromaKeyConfigurationViewController: ARSessionDelegate { func session(_ session: ARSession, didUpdate frame: ARFrame) { if first { - configureBackgroundPlane(with: frame) - configurePlane(with: frame) + let viewPortSize = sceneView.bounds.size + let interfaceOrientation = UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? UIInterfaceOrientation.unknown + let projectionMatrix = frame.camera.projectionMatrix(for: interfaceOrientation, viewportSize: viewPortSize, zNear: 0.001, zFar: 1000.0) + + configureBackgroundPlane(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) + configurePlane(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) didUpdateValues() first = false } diff --git a/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift b/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift index af789d5..d9f0f47 100644 --- a/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift +++ b/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift @@ -129,9 +129,9 @@ final class MixedRealityViewController: UIViewController { NotificationCenter.default.addObserver(self, selector: #selector(willResignActive), name: UIApplication.willResignActiveNotification, object: nil) } - private func configureBackground(with frame: ARFrame) { + private func configureBackground(viewPortSize: CGSize, projectionMatrix: simd_float4x4) { if case .hidden = configuration.backgroundLayerOptions.visibility { return } - let backgroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(100.0, frame: frame) + let backgroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(100.0, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) // Flipping image if configuration.shouldFlipOutput { @@ -162,11 +162,11 @@ final class MixedRealityViewController: UIViewController { self.backgroundNode = backgroundPlaneNode } - private func configureMiddle(with frame: ARFrame) { + private func configureMiddle(viewPortSize: CGSize, projectionMatrix: simd_float4x4) { guard case .greenScreen = configuration.captureMode, let chromaConfiguration = chromaConfiguration else { return } - let middlePlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.02, frame: frame) + let middlePlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.02, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) middlePlaneNode.geometry?.firstMaterial?.transparencyMode = .rgbZero @@ -203,9 +203,9 @@ final class MixedRealityViewController: UIViewController { self.middlePlaneNode = middlePlaneNode } - private func configureForeground(with frame: ARFrame) { + private func configureForeground(viewPortSize: CGSize, projectionMatrix: simd_float4x4) { guard case .visible(let useMagentaAsTransparency) = configuration.foregroundLayerOptions.visibility else { return } - let foregroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.01, frame: frame) + let foregroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.01, viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) // Flipping image if configuration.shouldFlipOutput { @@ -344,11 +344,16 @@ extension MixedRealityViewController: ARSessionDelegate { func session(_ session: ARSession, didUpdate frame: ARFrame) { if first { - configureBackground(with: frame) - configureMiddle(with: frame) - configureForeground(with: frame) + let viewPortSize = sceneView.bounds.size + let interfaceOrientation = UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? UIInterfaceOrientation.unknown + let projectionMatrix = frame.camera.projectionMatrix(for: interfaceOrientation, viewportSize: viewPortSize, zNear: 0.001, zFar: 1000.0) + + configureBackground(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) + configureMiddle(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) + configureForeground(viewPortSize: viewPortSize, projectionMatrix: projectionMatrix) first = false } else { + // TODO: Check this (this might need to be updated) cameraPoseSender?.didUpdate(frame: frame) } diff --git a/RealityMixer/Info.plist b/RealityMixer/Info.plist index 3db742b..b887b35 100644 --- a/RealityMixer/Info.plist +++ b/RealityMixer/Info.plist @@ -47,11 +47,8 @@ UIStatusBarStyleLightContent UISupportedInterfaceOrientations - UIInterfaceOrientationLandscapeRight - - UISupportedInterfaceOrientations~ipad - - UIInterfaceOrientationLandscapeRight + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown UIUserInterfaceStyle Light