Skip to content

M1 Mac Experiments #105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fabio914 wants to merge 4 commits into
base: main
Choose a base branch
from
Open

M1 Mac Experiments #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bf13828
M1 Experiments
fabio914 Feb 5, 2022
67bc5d6
Change behavior of the rotation
fabio914 Feb 5, 2022
799e22e
Replacing ARKit view with SceneKit view
fabio914 Feb 5, 2022
b94211d
Moving things to the same thread
fabio914 Feb 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion RealityMixer/Capture/Misc/ARConfigurationFactory.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ final class ARConfigurationFactory {
configuration.planeDetection = [.horizontal, .vertical]
configuration.environmentTexturing = .none
configuration.isLightEstimationEnabled = true
configuration.isAutoFocusEnabled = mrConfiguration.enableAutoFocus
// configuration.isAutoFocusEnabled = mrConfiguration.enableAutoFocus
return configuration
}

Expand Down
14 changes: 12 additions & 2 deletions RealityMixer/Capture/Misc/ARKitHelpers.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,15 @@ struct ARKitHelpers {

// FIXME: Check this.
static func planeSizeForDistance(_ distance: Float, frame: ARFrame) -> CGSize {
let projection = frame.camera.projectionMatrix
planeSizeForDistance(distance, projection: frame.camera.projectionMatrix, imageResolution: frame.camera.imageResolution)
}

static func planeSizeForDistance(_ distance: Float, camera: SCNCamera, imageResolution: CGSize) -> CGSize {
planeSizeForDistance(distance, projection: .init(camera.projectionTransform), imageResolution: imageResolution)
}

static func planeSizeForDistance(_ distance: Float, projection: simd_float4x4, imageResolution: CGSize) -> CGSize {
let yScale = projection[1,1]
let imageResolution = frame.camera.imageResolution
let width = (2.0 * distance) * tan(atan(1/yScale) * Float(imageResolution.width / imageResolution.height))
let height = width * Float(imageResolution.height / imageResolution.width)
return CGSize(width: CGFloat(width), height: CGFloat(height))
Expand All @@ -34,6 +40,10 @@ struct ARKitHelpers {
makePlane(size: planeSizeForDistance(distance, frame: frame), distance: distance)
}

static func makePlaneNodeForDistance(_ distance: Float, camera: SCNCamera, imageResolution: CGSize) -> SCNNode {
makePlane(size: planeSizeForDistance(distance, camera: camera, imageResolution: imageResolution), distance: distance)
}

@discardableResult
static func create(textureCache: inout CVMetalTextureCache?, for sceneView: SCNView) -> Bool {
guard let metalDevice = sceneView.device else {
Expand Down
124 changes: 62 additions & 62 deletions RealityMixer/Capture/Misc/CameraPoseSender.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,80 +148,80 @@ struct CameraRotationPayload {
final class CameraPoseSender {
private weak var client: TCPClient?

private struct InitialPose {
let position: Vector3
let inverseRotation: Quaternion
}
// private struct InitialPose {
// let position: Vector3
// let inverseRotation: Quaternion
// }

private var initialReliableCameraPose: InitialPose?
private let initialCalibrationPose: Pose
// private var initialReliableCameraPose: InitialPose?
// private let initialCalibrationPose: Pose

init?(client: TCPClient) {
guard let pose = TemporaryCalibrationStorage.shared.calibrationPose else {
return nil
}
// guard let pose = TemporaryCalibrationStorage.shared.calibrationPose else {
// return nil
// }

self.initialCalibrationPose = pose
// self.initialCalibrationPose = pose
self.client = client
}

private func sendCameraUpdate(pose: Pose/*, intrinsics: CameraIntrinsicsPayload*/) {
func sendCameraUpdate(pose: Pose/*, intrinsics: CameraIntrinsicsPayload*/) {
// _ = client?.send(data: intrinsics.data)
_ = client?.send(data: CameraPositionPayload(position: pose.position).data)
_ = client?.send(data: CameraRotationPayload(rotation: pose.rotation).data)
}

private func updateCached(pose: Pose) {
TemporaryCalibrationStorage.shared.update(pose: pose)
}

func didUpdate(frame: ARFrame) {
if let initialReliableCameraPose = initialReliableCameraPose {
switch frame.camera.trackingState {
case .normal, .limited:
let position = frame.camera.transform.columns.3

let positionVector = Vector3(
x: .init(position.x),
y: .init(position.y),
z: .init(position.z)
)

let positionDelta = positionVector - initialReliableCameraPose.position
let rotationDelta = initialReliableCameraPose.inverseRotation * Quaternion(rotationMatrix: SCNMatrix4(frame.camera.transform))

let pose = Pose(
position: positionDelta,
rotation: rotationDelta
)

let poseResult = initialCalibrationPose * pose
sendCameraUpdate(pose: poseResult /*, intrinsics: intrinsics*/)
updateCached(pose: poseResult)
default:
break
}
} else {

// Assuming there was no movement between the first ARFrame and the first reliable
// camera position and orientation

if case .normal = frame.camera.trackingState {
let position = frame.camera.transform.columns.3

initialReliableCameraPose = InitialPose(
position: .init(
x: .init(position.x),
y: .init(position.y),
z: .init(position.z)
),
inverseRotation: Quaternion(
rotationMatrix: SCNMatrix4(frame.camera.transform)
).inverse
)
}
}
}
// private func updateCached(pose: Pose) {
// TemporaryCalibrationStorage.shared.update(pose: pose)
// }
//
// func didUpdate(frame: ARFrame) {
// if let initialReliableCameraPose = initialReliableCameraPose {
// switch frame.camera.trackingState {
// case .normal, .limited:
// let position = frame.camera.transform.columns.3
//
// let positionVector = Vector3(
// x: .init(position.x),
// y: .init(position.y),
// z: .init(position.z)
// )
//
// let positionDelta = positionVector - initialReliableCameraPose.position
// let rotationDelta = initialReliableCameraPose.inverseRotation * Quaternion(rotationMatrix: SCNMatrix4(frame.camera.transform))
//
// let pose = Pose(
// position: positionDelta,
// rotation: rotationDelta
// )
//
// let poseResult = initialCalibrationPose * pose
// sendCameraUpdate(pose: poseResult /*, intrinsics: intrinsics*/)
// updateCached(pose: poseResult)
// default:
// break
// }
// } else {
//
// // Assuming there was no movement between the first ARFrame and the first reliable
// // camera position and orientation
//
// if case .normal = frame.camera.trackingState {
// let position = frame.camera.transform.columns.3
//
// initialReliableCameraPose = InitialPose(
// position: .init(
// x: .init(position.x),
// y: .init(position.y),
// z: .init(position.z)
// ),
// inverseRotation: Quaternion(
// rotationMatrix: SCNMatrix4(frame.camera.transform)
// ).inverse
// )
// }
// }
// }
}

// TODO: Improve this...
Expand Down
2 changes: 1 addition & 1 deletion RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ final class ChromaKeyConfigurationViewController: UIViewController {
configuration.planeDetection = [.horizontal, .vertical]
configuration.environmentTexturing = .none
configuration.isLightEstimationEnabled = false
configuration.isAutoFocusEnabled = false
// configuration.isAutoFocusEnabled = false
sceneView.session.run(configuration)
}

Expand Down
Loading