Repository navigation
Expand file tree
/
Copy pathDockerfile
More file actions
136 lines (124 loc) · 7.18 KB
/
Copy pathDockerfile
File metadata and controls
136 lines (124 loc) · 7.18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
# syntax=docker/dockerfile:1.7
#
# Built natively on the arm64 runners rather than under QEMU: the kw cluster is
# arm64 throughout, and emulating a Rust release build with LTO is far slower
# than it is worth.
# The management UI's build (P4). A separate stage, not a `build.rs` that
# shells out to `npm` from the Rust build — see `src/control/ui.rs` and
# `TODO.md`'s original design note for why: that would make `cargo build`
# and `cargo test` require Node for everyone, including CI jobs that only
# want to run the Rust test suite. This stage's only job is to produce
# `web/dist/`, which the `build` stage below copies in before compiling —
# `rust-embed` reads whatever is on disk under `web/dist/` at that point,
# nothing more.
FROM node:22-bookworm-slim AS web
WORKDIR /web
COPY web/package.json web/package-lock.json* ./
RUN npm install
COPY web/ ./
# The screen and interaction checks run here rather than only in CI: this stage
# is the one place the UI is guaranteed to be built before it is embedded, and
# a screen that throws on mount — or a button that posts a body the handler
# discards — would otherwise ship as something nobody noticed until an
# operator opened it. Two RUNs, not one: which step failed is the diagnosis.
# hadolint ignore=DL3059
RUN npm test
RUN npm run build
# Trixie, not bookworm, and specifically because of ONNX Runtime. The prebuilt
# `ort` static library is compiled against a newer libstdc++ than bookworm
# ships, so linking it there fails on symbols like `_M_replace_cold` and
# `__cxa_call_terminate`. The runtime stage below must match: a binary linked
# against trixie's libstdc++ will not start on bookworm.
FROM rust:1-trixie AS build
WORKDIR /src
COPY Cargo.toml Cargo.lock ./
COPY src ./src
COPY migrations ./migrations
# `src/control/api.rs` embeds this with `include_str!`, so it is a build
# input rather than documentation that happens to live nearby. Omitting it
# fails the compile — which is how it was found, in a release build, after
# every local build had succeeded because the file was simply there.
COPY openapi.json ./openapi.json
# The root manifest declares a workspace, so cargo insists on loading every
# member before it will build anything — omitting either of these fails the
# build outright rather than merely skipping that member. `default-members`
# still keeps them out of the release build; only their manifests have to be
# readable.
#
# Adding a workspace member and not a line here is a build that passes every
# test and fails at image time, which is exactly how `operator` broke this
# once already.
COPY bench ./bench
COPY operator ./operator
# Built `web/dist/` from the `web` stage above, not the `web/` source tree —
# `rust-embed`'s derive macro (src/control/ui.rs) only ever reads this one
# directory. Without this `COPY`, the directory `rust-embed` needs to exist
# at compile time would be missing entirely inside this stage's filesystem;
# with it, the binary embeds a real, built SPA instead of degrading to the
# "UI not available" placeholder a bare `cargo build` (no prior `npm run
# build`) produces.
COPY --from=web /web/dist ./web/dist
# The fast-tier classifier model, baked in so a proxy never reaches the network
# to start serving. ~61MB of static token vectors — see `src/classifier` for why
# this tier is a lookup table rather than a transformer, and docs/classifier.md
# for the measurements that chose this model over its neighbours.
#
# The refined tier's ONNX weights are baked in too. They cost ~130MB of image,
# which is real, but the alternative is an operator discovering at configuration
# time that the tier they just enabled needs files nobody told them to mount.
# Nothing loads them unless a routing rule names a refined class, so the runtime
# cost of carrying them is zero.
ADD --chmod=0644 https://huggingface.co/minishlab/potion-code-16M/resolve/main/model.safetensors \
/usr/local/share/fastllm/classifier/model.safetensors
ADD --chmod=0644 https://huggingface.co/minishlab/potion-code-16M/resolve/main/tokenizer.json \
/usr/local/share/fastllm/classifier/tokenizer.json
ADD --chmod=0644 https://huggingface.co/minishlab/potion-code-16M/resolve/main/config.json \
/usr/local/share/fastllm/classifier/config.json
# The int8 build, not the fp32 one. Measured with `classify-bench` in this
# image on a 2-core arm64 pod: 13.4-15.3ms per prompt against fp32's 28.7ms,
# roughly 2x, and 34MB of layer instead of 133MB.
#
# Gated on accuracy, not just latency, because accuracy is the only reason this
# tier costs anything at all. `bench/minilm` over the same StackExchange data
# puts architecture precision at 93.2% quantised against 93.3% fp32, and
# centroid similarity at 0.944 against 0.943 — the geometry is unchanged, so
# `min_margin` floors tuned against the fp32 model stay valid.
ADD --chmod=0644 https://huggingface.co/Xenova/bge-small-en-v1.5/resolve/main/onnx/model_quantized.onnx \
/usr/local/share/fastllm/classifier-tier2/model.onnx
ADD --chmod=0644 https://huggingface.co/Xenova/bge-small-en-v1.5/resolve/main/tokenizer.json \
/usr/local/share/fastllm/classifier-tier2/tokenizer.json
ADD --chmod=0644 https://huggingface.co/Xenova/bge-small-en-v1.5/resolve/main/config.json \
/usr/local/share/fastllm/classifier-tier2/config.json
ADD --chmod=0644 https://huggingface.co/Xenova/bge-small-en-v1.5/resolve/main/special_tokens_map.json \
/usr/local/share/fastllm/classifier-tier2/special_tokens_map.json
ADD --chmod=0644 https://huggingface.co/Xenova/bge-small-en-v1.5/resolve/main/tokenizer_config.json \
/usr/local/share/fastllm/classifier-tier2/tokenizer_config.json
# Cache mounts are not part of the resulting layer, so the binary has to be
# copied out of the target directory inside the same step that produced it.
RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
--mount=type=cache,target=/src/target,sharing=locked \
cargo build --release --locked --features "control classifier-tier2" \
&& cp target/release/fastllm-proxy /usr/local/bin/fastllm-proxy
FROM debian:trixie-slim
# ca-certificates is what makes an `https://` api_base work: the proxy prefers
# the system root store and only falls back to its bundled copy. Unpinned on
# purpose: Debian drops old versions from the mirrors, so a pin rots into a
# build that cannot run anywhere.
# hadolint ignore=DL3008
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates \
&& rm -rf /var/lib/apt/lists/* \
&& useradd --uid 65532 --user-group --no-create-home --shell /usr/sbin/nologin fastllm
COPY --from=build /usr/local/bin/fastllm-proxy /usr/local/bin/fastllm-proxy
COPY --from=build /usr/local/share/fastllm/classifier /usr/local/share/fastllm/classifier
COPY --from=build /usr/local/share/fastllm/classifier-tier2 /usr/local/share/fastllm/classifier-tier2
USER 65532:65532
EXPOSE 4000
# Loopback is the right default for a binary someone runs on a workstation, and
# the wrong one for a container, where the kubelet has to reach it.
ENV FASTLLM_HOST=0.0.0.0 \
FASTLLM_PORT=4000 \
FASTLLM_CONFIG=/etc/fastllm/config.yaml \
FASTLLM_CLASSIFIER_MODEL=/usr/local/share/fastllm/classifier \
FASTLLM_CLASSIFIER_TIER2_MODEL=/usr/local/share/fastllm/classifier-tier2
ENTRYPOINT ["/usr/local/bin/fastllm-proxy"]