Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,35 @@ concurrency:
cancel-in-progress: true

jobs:
# The runner's default Xcode is the newest on the image, so this job alone
# never exercised the oldest toolchain the README supports. Three Swift 6.2
# build breaks shipped in 0.5.0 behind that gap.
build-oldest-supported-toolchain:
if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
runs-on: macos-26
steps:
- uses: actions/checkout@v4
- name: Select the oldest supported Xcode
run: |
app=$(ls -d /Applications/Xcode_26.3*.app 2>/dev/null | head -1)
if [ -z "$app" ]; then
echo "Xcode 26.3 is not on this runner image; available:" >&2
ls -d /Applications/Xcode*.app >&2
exit 1
fi
sudo xcode-select -s "$app"
- name: Report the toolchain
run: swift --version
- name: Build release products
run: swift build -c release

test:
if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
runs-on: macos-26
steps:
- uses: actions/checkout@v4
- name: Report the toolchain
run: swift --version
- name: Build release products
run: swift build -c release
- name: Run serial tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public enum AboutPanelPresentation {
// Most users build from a clone, where there is no Info.plist to read a
// version from, so this constant is what they see. Scripts/check_app_version.rb
// fails CI when it falls behind the newest published release.
public static let fallbackShortVersion = "0.7.0"
public static let fallbackShortVersion = "0.7.1"

private static let licenseURL = URL(
string: "https://github.com/drumih/turbo-fieldfare/blob/main/LICENSE")!
Expand Down
133 changes: 82 additions & 51 deletions Sources/TurboFieldfareServer/Core/ServerInference.swift
Original file line number Diff line number Diff line change
Expand Up @@ -435,44 +435,65 @@ enum ServerRequestImages {
return planned
}

/// Encodes an image from the plan its count was taken from. Only an image
/// What an image's encode reads through.
enum Source {
/// The plan the image's count was taken from, still open.
case plan(VisionImagePlan)
/// The file again, for an image the open bound released.
case reopened(URL)
}

/// An image is encoded from the plan its count was taken from. Only one
/// whose plan the open bound released is read a second time.
static func encode<Features>(
_ image: Planned,
fromPlan: (VisionImagePlan) throws -> Features,
byReopening: (URL) throws -> Features
) throws -> Features {
guard let plan = image.plan else { return try byReopening(image.url) }
return try fromPlan(plan)
///
/// Returned rather than dispatched through caller-supplied closures:
/// Swift 6.2's region isolation rejects an actor-isolated closure that
/// captures a session's `VisionRuntime` and `Model` and is handed to a
/// nonisolated callee, which broke the build on Xcode 26.0-26.3.
static func source(for image: Planned) -> Source {
guard let plan = image.plan else { return .reopened(image.url) }
return .plan(plan)
}

/// Every image a full prefill needs, each read through one open file.
/// Every image of a request planned, each still paired with the id its
/// features are filed under.
///
/// The pairing is the reason this is not two statements at the call site:
/// `[UUID: URL]` has no order, so the ids and the plans have to be taken
/// from one snapshot. Pairing them from two walks of the dictionary files
/// one image's features under another's id, and when both project to the
/// same soft-token count nothing downstream can notice.
///
/// A full prefill lays its spans out from the encoded features rather than
/// from a count, so it asked for no plan of its own and let
/// `encodeImage(at:)` make one per image internally — the third copy of the
/// pattern, and the one that also encoded its way through the images ahead
/// of one that could not be read at all.
static func encodeAll<Features>(
/// Planning every image before the caller encodes any is the other half:
/// the spans are laid out from the same open files the encodes read, and a
/// request whose last image cannot be read at all is refused before the
/// tower has run on the ones ahead of it.
static func plannedEntries(
_ imageFiles: [UUID: URL],
with preprocessor: Gemma4ImagePreprocessor,
maximumOpenPlans: Int = ServerRequestImages.maximumOpenPlans,
checkCancellation: () throws -> Void = {},
encode: (Planned) throws -> Features
) throws -> [UUID: Features] {
checkCancellation: () throws -> Void = {}
) throws -> [(id: UUID, image: Planned)] {
let entries = Array(imageFiles)
let planned = try plans(
for: entries.map { $0.value },
with: preprocessor,
maximumOpenPlans: maximumOpenPlans,
checkCancellation: checkCancellation)
var features: [UUID: Features] = [:]
features.reserveCapacity(entries.count)
for (entry, image) in zip(entries, planned) {
try checkCancellation()
features[entry.key] = try encode(image)
}
return features
return zip(entries, planned).map { (id: $0.key, image: $1) }
}

/// A file the caller supplied that cannot be read is a bad request, not a
/// server fault, and it stays one wherever the read fails. Admission reads
/// dimensions with stream verification off; the encode re-plans with it on,
/// so a truncated upload is admitted and only fails the second read.
/// Without this that second failure reached the generic handler as a 500,
/// which official clients retry and 4xx they do not.
static func requestError(forUnreadable error: any Error) -> any Error {
if error is ServerRequestError || error is CancellationError { return error }
return ServerRequestError.invalid(
message: "image could not be read: \(error)",
param: "messages", code: "invalid_image")
}
}

Expand Down Expand Up @@ -682,22 +703,20 @@ public actor ServerModelSession: ServerInferenceBackend {
private func encodeTurnImage(
_ image: ServerRequestImages.Planned, visionRuntime: VisionRuntime
) throws -> VisionFeatures {
try ServerRequestImages.encode(
image,
fromPlan: {
try visionRuntime.encodeImage(
plan: $0,
languageModel: model,
residencyPolicy: visionResidencyPolicy,
checkCancellation: { try Task.checkCancellation() })
},
byReopening: {
try visionRuntime.encodeImage(
at: $0,
languageModel: model,
residencyPolicy: visionResidencyPolicy,
checkCancellation: { try Task.checkCancellation() })
})
switch ServerRequestImages.source(for: image) {
case .plan(let plan):
return try visionRuntime.encodeImage(
plan: plan,
languageModel: model,
residencyPolicy: visionResidencyPolicy,
checkCancellation: { try Task.checkCancellation() })
case .reopened(let url):
return try visionRuntime.encodeImage(
at: url,
languageModel: model,
residencyPolicy: visionResidencyPolicy,
checkCancellation: { try Task.checkCancellation() })
}
}

/// Projected token count per image from headers alone; no pixel is decoded
Expand All @@ -717,12 +736,8 @@ public actor ServerModelSession: ServerInferenceBackend {
do {
let geometry = try preprocessor.admissionGeometry(fileURL: url)
counts.append(geometry.softTokenCount)
} catch let error as ServerRequestError {
throw error
} catch {
throw ServerRequestError.invalid(
message: "image could not be read: \(error)",
param: "messages", code: "invalid_image")
throw ServerRequestImages.requestError(forUnreadable: error)
}
}
return counts
Expand Down Expand Up @@ -805,11 +820,27 @@ public actor ServerModelSession: ServerInferenceBackend {
param: "messages", code: "vision_unavailable")
}
do {
let features = try ServerRequestImages.encodeAll(
request.imageFiles,
with: imagePreprocessor(visionRuntime),
checkCancellation: { try Task.checkCancellation() },
encode: { try encodeTurnImage($0, visionRuntime: visionRuntime) })
var features: [UUID: VisionFeatures] = [:]
// Scoped, so the plans — up to `maximumOpenPlans` live ImageIO
// descriptors — are released before the render walks the whole
// history, rather than held across it.
do {
let planned: [(id: UUID, image: ServerRequestImages.Planned)]
do {
planned = try ServerRequestImages.plannedEntries(
request.imageFiles,
with: imagePreprocessor(visionRuntime),
checkCancellation: { try Task.checkCancellation() })
} catch {
throw ServerRequestImages.requestError(forUnreadable: error)
}
features.reserveCapacity(planned.count)
for entry in planned {
try Task.checkCancellation()
features[entry.id] = try encodeTurnImage(
entry.image, visionRuntime: visionRuntime)
}
}
return try MultimodalPromptRenderer.render(
messages: messages,
featuresByID: features,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,23 @@ enum TranscriptFrameRenderer {
/// `TURBO_FIELDFARE_FRAME_DIR` is set, also writes it there so frames can
/// be reviewed without passing `--attachments-path`.
static func record(_ image: NSImage, named name: String) throws {
#if compiler(>=6.3)
Attachment.record(image, named: name, as: .png)
#else
// `NSImage` gained its `Attachable` conformance in the Testing library
// shipped with Swift 6.3, and this suite has to build on the 6.2
// toolchains the README supports. The bytes are the same; what is lost
// is the lazy serialization, so a 6.2 run pays a PNG encode per frame
// even without `--attachments-path`. Isolating a lazy wrapper is not
// available either: a struct holding an `NSImage` cannot conform
// without the conformance crossing into main-actor code.
if let rep = image.representations.first as? NSBitmapImageRep,
let data = rep.representation(using: .png, properties: [:]) {
Attachment.record(data, named: name)
} else {
Issue.record("frame \(name) has no PNG representation to attach")
}
#endif
guard let directory = ProcessInfo.processInfo
.environment["TURBO_FIELDFARE_FRAME_DIR"], !directory.isEmpty else {
return
Expand Down
Loading
Loading