From c4a96961f885e84ba954c67344b16ef07aa2a5d6 Mon Sep 17 00:00:00 2001 From: sehkone Date: Sat, 29 Aug 2026 15:38:40 +0900 Subject: [PATCH 1/5] Renew the registrar surface's two leaves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both leaves the registrar surface runs on expire, and nothing renewed them: start-time issuance mints them once and leaves a usable pair alone, so a lapse took every enrollment in the deployment down and the registrar could not re-mint itself. The daemon now owns one renewal adapter, created only where the endpoint is enabled, running on the rendered internal agent configuration's own cadence, lead time and issuance-retry budget rather than on a second policy of its own. It initializes the per-leaf state accessor from the certificates already on disk, before the endpoint serves anything, and joins with the daemon's other tasks on shutdown. Nothing live changes until everything that can fail has succeeded. The replacement is issued off to one side — the ACME path is split so it can return the chain, the candidate and a fresh key instead of publishing them — then validated, then merged into a staged CA bundle, then turned into a whole next TLS configuration. Only then are the bundle, the certificate and the key written, each behind a snapshot of its bytes, mode and ownership; a write that fails restores what it reached, and a rollback that fails says so rather than claiming the files were put back. The endpoint's active configuration is exchanged rather than its resolver alone. The old acceptor retains the old client verifier, so replacing only the presented certificate would leave a trust-anchor rotation undecided until a restart. The accept loop loads the active configuration immediately before each handshake, so the exchange costs no restart, no signal, no socket rebind and no dropped connection. The caller's half of the same contract is the per-dial load: the pair is published by two separate renames, so a reader that lands between them re-reads rather than presenting a mismatch, five reads with the 1/2/4/8 ms waits and then a typed error. Closes #768 --- CHANGELOG.md | 18 +- docs/en/operations.md | 67 ++ docs/ko/operations.md | 63 ++ src/acme.rs | 9 +- src/acme/flow.rs | 236 +++-- src/daemon.rs | 89 +- src/fs_util.rs | 14 + src/lib.rs | 4 + src/registrar/endpoint.rs | 106 +- src/registrar/endpoint/client.rs | 139 ++- src/registrar/endpoint/client/tests.rs | 180 ++++ src/registrar/endpoint/serve.rs | 8 +- src/registrar/endpoint/tests.rs | 84 +- src/registrar/endpoint/tls.rs | 114 +- src/registrar_certs.rs | 78 +- src/registrar_certs/tests.rs | 91 ++ src/registrar_renewal.rs | 1249 ++++++++++++++++++++++ src/registrar_renewal/tests.rs | 1351 ++++++++++++++++++++++++ 18 files changed, 3776 insertions(+), 124 deletions(-) create mode 100644 src/registrar_renewal.rs create mode 100644 src/registrar_renewal/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b7bf6a..619c9399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,23 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm material is in date starts having asked nothing of `OpenBao` or of the CA, and its certificates and keys survive the restart byte-identically. An already-expired leaf is repaired at start, before the endpoint's TLS - material loads, rather than at the first renewal tick. + material loads, rather than at the first renewal tick. Both leaves are + then kept valid on the daemon's own loop, under the cadence, lead time + and retry settings of the bootroot-internal profile: a leaf is replaced + when it falls inside lead time or stops chaining to `[trust] + ca_bundle_path`, and the replacement is issued off to one side and + fully validated — its key, its name, and for the endpoint leaf an + anchor the endpoint pin file already names — before anything live is + written. A publication that fails restores every file it reached, and + the endpoint keeps serving what it was serving. A successful one + exchanges the whole active TLS configuration at once, so the next + handshake presents the renewed leaf and accepts callers under the + renewed trust anchors with no restart, no signal, no socket rebind and + no dropped connection. The endpoint pin file is never rewritten and + never gains a leaf fingerprint. A caller reloads per dial: it rereads + the pair every time and, because the two files are published by + separate renames, retries a momentarily mismatched pair up to five + reads before failing rather than presenting it. - `bootroot-agent` now rotates `OpenBao`'s file audit device on a host whose registrar endpoint is enabled, so the deployment no longer needs an external rotator against it. Every 60 seconds the daemon renames the diff --git a/docs/en/operations.md b/docs/en/operations.md index 467c7e0e..bac5432d 100644 --- a/docs/en/operations.md +++ b/docs/en/operations.md @@ -2009,6 +2009,73 @@ None of the four certificate paths is reloadable. A `SIGHUP` that changes that changed, and the running daemon is left as it is. Only the *contents* at those paths can change under a running daemon. +#### Renewal, and the reload contract on both sides + +Both leaves expire, and the daemon renews both itself. Where the endpoint is +enabled it runs one renewal pass immediately at start and then one per interval, +under the cadence, lead time and issuance-retry settings of the rendered +internal agent configuration's single profile — the same values that govern the +bootroot-internal credential's own renewal. A leaf is renewed when it falls +inside lead time, or when it no longer chains to `trust.ca_bundle_path`, which +is what a trust-anchor rotation under a running daemon leaves behind. Where the +endpoint is disabled nothing of this exists: no pass runs and nothing is asked +of OpenBao or the CA. + +A renewal is published as a transaction. The replacement is issued, validated +and turned into a complete next TLS configuration **before** any live file is +written: its key must be the leaf's, a client replacement must carry the same +instance, host and domain it already had, and a server replacement must carry +the exact endpoint SAN and chain to an anchor the endpoint pin file already +names. The pin file is never rewritten and never gains a leaf fingerprint, so a +server replacement no pinned caller would accept is discarded rather than +published. Only then are the merged CA bundle, the certificate and the key +written; if any of those writes fails, every path the publication reached is +restored from a snapshot of its bytes, mode and ownership, the endpoint keeps +serving what it was serving, and the next pass tries again. A rollback that +itself fails is logged as exactly that — both errors, and no claim that the +files were put back. + +**The server side reloads with no restart.** Once every write has landed, the +daemon exchanges the whole active TLS configuration at once: the certificate the +endpoint presents *and* the verifier every client certificate is checked +against. Replacing only the presented certificate would leave the old verifier +deciding who may connect, so a trust-anchor rotation would not take effect until +a restart. The next handshake uses the new configuration; the socket is not +rebound, its pathname and inode are unchanged, no signal is sent and no +connection or handshake already in flight is dropped. + +**The caller side reloads per dial, and must ride out the torn pair.** The +certificate and the key are published by two separate renames, so neither file +is ever half-written but the *pair* is not replaced atomically: a caller reading +in between sees the new certificate beside the old key, or the old certificate +beside the new one. Presenting that pair fails the handshake with a signature +error that says nothing about what happened, so a caller must not present it. +The contract this repository's own client implements, and the behaviour the +deployed registrar is expected to match, is: + +- **Re-read both files on every dial.** No caching, and no configuration reload + or process restart to pick up a renewal — the dial after the publication + presents the new pair. +- **Check that the key is the leaf's** before presenting anything. +- **On a mismatch, re-read.** Five reads in total, with waits of 1 ms, 2 ms, + 4 ms and 8 ms after the first four mismatches. A pair that matches on any of + the five is used. +- **After the fifth mismatch, fail with a typed error** naming both paths and + the number of reads. That is a misconfiguration — two files that are not each + other's — rather than a race, and it is reported instead of being retried + further or presented anyway. + +This is a fixed local policy of the reader's and is deliberately not the +daemon's issuance-retry backoff: it is measured against one pair of renames on +the same host, not against a CA that is down. Any other failure — an absent +file, an unreadable one, a certificate PEM the parser refuses — is returned at +once and is not re-read. + +The reference implementation is this repository's in-repo registrar endpoint +client (`src/registrar/endpoint/client.rs`). It has no production consumer here; +it exists so the caller behaviour this endpoint expects is written down as +running code. + #### Installing the units Both units are checked in under `systemd/` in this repository. Copy them diff --git a/docs/ko/operations.md b/docs/ko/operations.md index 73aa8234..dbc68164 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -1908,6 +1908,69 @@ step-ca로 나가는 ACME 경로를 통해서입니다. 프로비저닝 도구 남습니다. 실행 중인 데몬 아래에서 바뀔 수 있는 것은 그 경로들의 *내용* 뿐입니다. +#### 갱신, 그리고 양쪽의 리로드 계약 {#renewal-and-the-reload-contract-on-both-sides} + +두 리프 모두 만료되며, 데몬이 두 리프를 직접 갱신합니다. 엔드포인트가 활성화된 +호스트에서는 기동 직후 한 번 갱신 패스를 돌린 뒤 주기마다 한 번씩 돌리고, 그 +주기·리드 타임·발급 재시도 설정은 렌더링된 내부 에이전트 설정의 단 하나뿐인 +프로필에서 가져옵니다. bootroot 내부 자격 증명 자신의 갱신을 지배하는 바로 그 +값들입니다. 리프는 리드 타임 안에 들어왔을 때, 또는 `trust.ca_bundle_path`까지 +체인이 더 이상 이어지지 않을 때 갱신됩니다. 후자는 실행 중인 데몬 아래에서 신뢰 +앵커 로테이션이 일어났을 때 남는 상태입니다. 엔드포인트가 비활성화된 호스트에는 +이 중 아무것도 존재하지 않습니다. 패스는 돌지 않고, OpenBao와 CA에 아무것도 +요청하지 않습니다. + +갱신은 하나의 트랜잭션으로 발행됩니다. 대체 자재는 **살아 있는 파일을 하나라도 +쓰기 전에** 발급되고 검증되어 완전한 다음 TLS 설정으로 조립됩니다. 키는 그 +리프의 키여야 하고, 클라이언트 대체본은 원래 갖고 있던 인스턴스·호스트·도메인을 +그대로 유지해야 하며, 서버 대체본은 정확한 엔드포인트 SAN을 갖고 엔드포인트 핀 +파일이 이미 지명한 앵커까지 체인이 이어져야 합니다. 핀 파일은 다시 쓰이지 않고 +리프 지문이 추가되지도 않으므로, 핀을 건 호출자라면 아무도 받아들이지 않을 서버 +대체본은 발행되지 않고 폐기됩니다. 그 다음에야 병합된 CA 번들, 인증서, 키가 +기록됩니다. 이 중 어느 쓰기라도 실패하면, 발행이 닿은 모든 경로가 +바이트·모드·소유권 스냅숏에서 복원되고, 엔드포인트는 서비스하던 것을 그대로 +계속 서비스하며, 다음 패스가 다시 시도합니다. 롤백 자체가 실패하면 정확히 +그렇게 기록됩니다. 두 오류가 함께 남고, 파일이 되돌려졌다는 주장은 하지 +않습니다. + +**서버 쪽은 재시작 없이 리로드합니다.** 모든 쓰기가 끝난 뒤 데몬은 활성 TLS +설정 전체를 한 번에 교체합니다. 엔드포인트가 제시하는 인증서 *와* 모든 +클라이언트 인증서를 검사하는 검증기를 함께 바꿉니다. 제시하는 인증서만 바꾸면 +예전 검증기가 계속 누가 접속할 수 있는지를 결정하므로, 신뢰 앵커 로테이션이 +재시작 전까지 반영되지 않습니다. 다음 핸드셰이크부터 새 설정이 쓰이고, 소켓은 +다시 바인딩되지 않으며, 경로명과 아이노드도 그대로이고, 시그널도 보내지 않으며, +이미 진행 중인 연결이나 핸드셰이크는 하나도 끊기지 않습니다. + +**호출자 쪽은 다이얼마다 리로드하며, 찢어진 쌍을 견뎌야 합니다.** 인증서와 키는 +별개의 rename 두 번으로 발행되므로, 두 파일 어느 쪽도 절반만 쓰인 상태로 +보이지는 않지만 *쌍* 자체는 원자적으로 교체되지 않습니다. 그 사이에 읽은 +호출자는 새 인증서 옆의 옛 키, 또는 옛 인증서 옆의 새 키를 보게 됩니다. 그 쌍을 +제시하면 무슨 일이 있었는지 아무것도 말해 주지 않는 서명 오류로 핸드셰이크가 +실패하므로, 호출자는 그것을 제시해서는 안 됩니다. 이 저장소의 클라이언트가 +구현하고 있고 배포된 레지스트라가 따라야 하는 계약은 다음과 같습니다. + +- **다이얼마다 두 파일을 다시 읽습니다.** 캐시하지 않고, 갱신을 반영하기 위한 + 설정 리로드나 프로세스 재시작도 필요 없습니다. 발행 직후의 다이얼이 새 쌍을 + 제시합니다. +- 무엇이든 제시하기 전에 **키가 그 리프의 키인지 확인합니다.** +- **맞지 않으면 다시 읽습니다.** 총 다섯 번 읽고, 처음 네 번의 불일치 뒤에 각각 + 1 ms·2 ms·4 ms·8 ms를 기다립니다. 다섯 번 중 어느 한 번이라도 맞는 쌍을 + 찾으면 그것을 씁니다. +- **다섯 번째 불일치 뒤에는 타입이 있는 오류로 실패합니다.** 두 경로와 읽은 + 횟수를 밝힙니다. 이는 경합이 아니라 서로의 짝이 아닌 두 파일, 즉 잘못된 + 설정이므로, 더 재시도하거나 그대로 제시하지 않고 보고합니다. + +이는 읽는 쪽의 고정된 로컬 정책이며, 데몬의 발급 재시도 백오프와는 의도적으로 +다릅니다. 같은 호스트에서 일어나는 rename 한 쌍을 기준으로 잡은 값이지, CA가 +죽어 있는 상황을 기준으로 잡은 값이 아닙니다. 파일이 없거나 읽을 수 없거나 +인증서 PEM을 파서가 거부하는 등 그 밖의 실패는 즉시 반환되고 다시 읽지 +않습니다. + +참조 구현은 이 저장소의 인레포 레지스트라 엔드포인트 +클라이언트(`src/registrar/endpoint/client.rs`)입니다. 여기에는 프로덕션 +소비자가 없습니다. 이 엔드포인트가 기대하는 호출자 동작을 실행되는 코드로 적어 +두기 위해 존재합니다. + #### 유닛 설치 두 유닛 모두 이 저장소의 `systemd/` 아래에 포함되어 있습니다. diff --git a/src/acme.rs b/src/acme.rs index 31abcf94..b3fb1e4e 100644 --- a/src/acme.rs +++ b/src/acme.rs @@ -8,6 +8,13 @@ pub(crate) mod types; // surface issuance, which is inside this crate; only the ordinary // `issue_certificate` is reached from the binary crates. pub(crate) use flow::{ - CsrShape, IssuanceOptions, LeafPublication, issue_certificate_with_bootstrap, + CsrShape, IssuanceOptions, IssuedMaterial, LeafPublication, issue_certificate_material, + issue_certificate_with_bootstrap, }; pub use flow::{build_registrar_client_csr_params, issue_certificate}; +// The two publication helpers a renewal stages its merged CA bundle +// with, so the staged bytes are computed by the same code the ordinary +// publication merges with rather than by a second rule. Their only +// consumer is the renewal adapter, which exists on Linux alone. +#[cfg(target_os = "linux")] +pub(crate) use flow::{merge_ca_bundle, verify_chain_fingerprints}; diff --git a/src/acme/flow.rs b/src/acme/flow.rs index 637f301d..44787049 100644 --- a/src/acme/flow.rs +++ b/src/acme/flow.rs @@ -187,7 +187,7 @@ fn sha256_hex(bytes: &[u8]) -> String { output } -fn verify_chain_fingerprints(chain: &[Vec], trusted: &[String]) -> Result<()> { +pub(crate) fn verify_chain_fingerprints(chain: &[Vec], trusted: &[String]) -> Result<()> { let allowed: HashSet = trusted .iter() .map(|value| value.to_ascii_lowercase()) @@ -212,7 +212,7 @@ fn verify_chain_fingerprints(chain: &[Vec], trusted: &[String]) -> Result<() /// existing bundle and keeping every block whose fingerprint is in /// `trusted` preserves the root across issuances while still filtering /// out any junk left behind by an earlier misconfiguration. -fn merge_ca_bundle( +pub(crate) fn merge_ca_bundle( existing_bundle: Option<&[u8]>, new_chain: &[Vec], trusted: &[String], @@ -497,6 +497,68 @@ pub async fn issue_certificate_with( .await } +/// Everything one ACME issuance produced, before a single byte of it has +/// been written anywhere. +/// +/// The value [`issue_certificate_material`] returns, and the value +/// [`issue_certificate_with_bootstrap`] publishes. Splitting the two +/// apart is what lets a renewal validate a candidate — its key, its SAN, +/// its issuer chain — before it touches the pair a running endpoint and +/// a running caller are reading. +#[derive(Debug)] +pub(crate) struct IssuedMaterial { + /// The certificate file's PEM, already in the shape + /// [`IssuanceOptions::leaf_publication`] selects. + pub(crate) cert_pem: String, + /// The private key, as PEM. Every issuance generates a fresh one. + pub(crate) key_pem: String, + /// The issuer chain the CA returned, as DER, with the leaf removed. + /// + /// Empty when `trust.ca_bundle_path` is unconfigured, where the + /// whole response is published as the leaf and there is no bundle to + /// merge into. + pub(crate) chain: Vec>, +} + +/// Runs one ACME issuance and returns what it produced, writing nothing. +/// +/// The same outbound path [`issue_certificate_with_bootstrap`] runs — +/// the same account registration, the same HTTP-01 validation, the same +/// CSR shape and the same fresh key — stopped immediately before +/// publication. It does not verify the returned chain against +/// `trust.trusted_ca_sha256`, does not read or merge +/// `trust.ca_bundle_path`, and does not write `profile.paths`. +/// +/// `Ok(None)` is an order that finalized without a certificate, which is +/// the one non-error outcome that produces no material. +/// +/// # Errors +/// +/// Returns an error if the ACME protocol fails or the returned PEM +/// cannot be split into a leaf and its chain. +// The only production caller is the registrar surface's renewal +// adapter, which needs the activated endpoint and so exists on Linux +// alone. Elsewhere this is reached from tests only. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub(crate) async fn issue_certificate_material( + settings: &crate::config::Settings, + profile: &crate::config::DaemonProfileSettings, + eab_creds: Option, + insecure_mode: bool, + options: IssuanceOptions, + bootstrap_pins: Option<&[String]>, +) -> Result> { + run_issuance( + settings, + profile, + eab_creds, + insecure_mode, + options, + bootstrap_pins, + ) + .await +} + /// Issues a registrar-surface certificate with optional pin-only bootstrap /// TLS while the configured output bundle is being repaired. pub(crate) async fn issue_certificate_with_bootstrap( @@ -507,6 +569,82 @@ pub(crate) async fn issue_certificate_with_bootstrap( options: IssuanceOptions, bootstrap_pins: Option<&[String]>, ) -> Result<()> { + let Some(material) = run_issuance( + settings, + profile, + eab_creds, + insecure_mode, + options, + bootstrap_pins, + ) + .await? + else { + return Ok(()); + }; + publish_issued_material(settings, profile, &material).await +} + +/// Publishes what an issuance produced at the configured paths. +/// +/// The chain first, the leaf after. An unpinned fingerprint or a CA +/// bundle that cannot be read fails here, before anything is published, +/// so no consumer is left holding a leaf whose issuer this host does not +/// trust. With `ca_bundle_path` unconfigured there is nothing to verify +/// or merge, and with an empty chain the bundle is left alone with a +/// warning; neither is a refusal. +async fn publish_issued_material( + settings: &crate::config::Settings, + profile: &crate::config::DaemonProfileSettings, + material: &IssuedMaterial, +) -> Result<()> { + let policy = crate::cert_group::CertGroupPolicy { + gid: profile.cert_group_gid, + }; + if let Some(bundle_path) = &settings.trust.ca_bundle_path { + if material.chain.is_empty() { + warn!("Certificate chain not present; CA bundle not updated."); + } else { + verify_chain_fingerprints(&material.chain, &settings.trust.trusted_ca_sha256)?; + write_merged_ca_bundle( + bundle_path, + &material.chain, + &settings.trust.trusted_ca_sha256, + policy, + ) + .await?; + info!("CA bundle saved to: {:?}", bundle_path); + } + } + + // Each file is staged and `rename(2)`d, so neither destination + // ever holds a truncated PEM. The *pair* is not atomic: the two + // renames are separate, so a reader landing between them sees the + // new leaf with the old key, or the old leaf with the new key. + // Closing that window is the reader's side of the contract — a + // bounded retry on a mismatched pair — and is not done here. + fs_util::write_cert_and_key( + &profile.paths.cert, + &profile.paths.key, + &material.cert_pem, + &material.key_pem, + policy, + ) + .await?; + info!("Certificate saved to: {:?}", profile.paths.cert); + info!("Private key saved to: {:?}", profile.paths.key); + Ok(()) +} + +/// Everything from the ACME directory through the downloaded +/// certificate, with no write of any kind. +async fn run_issuance( + settings: &crate::config::Settings, + profile: &crate::config::DaemonProfileSettings, + eab_creds: Option, + insecure_mode: bool, + options: IssuanceOptions, + bootstrap_pins: Option<&[String]>, +) -> Result> { // `--insecure` remains its existing explicit override. Bootstrap mode // never changes that mode's ACME or responder transport behavior. let bootstrap_pins = (!insecure_mode).then_some(bootstrap_pins).flatten(); @@ -555,78 +693,38 @@ pub(crate) async fn issue_certificate_with_bootstrap( let finalized_order = wait_for_order_completion(settings, &mut client, &order, finalized_order).await?; - if let Some(cert_url) = finalized_order.certificate { - info!("Downloading certificate from: {}", cert_url); - let cert_pem = client.download_certificate(&cert_url).await?; - info!("Certificate received. Saving to files..."); - - let (leaf_pem, chain) = if settings.trust.ca_bundle_path.is_some() { - split_leaf_and_chain(&cert_pem)? - } else { - (cert_pem.clone(), Vec::new()) - }; - let key_pem = cert_key.serialize_pem(); - let policy = crate::cert_group::CertGroupPolicy { - gid: profile.cert_group_gid, - }; - - // The chain first, the leaf after. An unpinned fingerprint or a - // CA bundle that cannot be read fails here, before anything is - // published, so no consumer is left holding a leaf whose issuer - // this host does not trust. With `ca_bundle_path` unconfigured - // there is nothing to verify or merge, and with an empty chain - // the bundle is left alone with a warning; neither is a refusal. - if let Some(bundle_path) = &settings.trust.ca_bundle_path { - if chain.is_empty() { - warn!("Certificate chain not present; CA bundle not updated."); - } else { - verify_chain_fingerprints(&chain, &settings.trust.trusted_ca_sha256)?; - write_merged_ca_bundle( - bundle_path, - &chain, - &settings.trust.trusted_ca_sha256, - policy, - ) - .await?; - info!("CA bundle saved to: {:?}", bundle_path); - } - } - - let cert_file_pem = match options.leaf_publication { - LeafPublication::LeafOnly => leaf_pem, - LeafPublication::LeafWithChain => { - let mut published = leaf_pem; - for der in &chain { - published.push_str(&encode_cert_pem(der)); - } - published - } - }; - - // Each file is staged and `rename(2)`d, so neither destination - // ever holds a truncated PEM. The *pair* is not atomic: the two - // renames are separate, so a reader landing between them sees - // the new leaf with the old key, or the old leaf with the new - // key. Closing that window is the reader's side of the contract - // — a bounded retry on a mismatched pair — and is not done here. - fs_util::write_cert_and_key( - &profile.paths.cert, - &profile.paths.key, - &cert_file_pem, - &key_pem, - policy, - ) - .await?; - info!("Certificate saved to: {:?}", profile.paths.cert); - info!("Private key saved to: {:?}", profile.paths.key); - } else { + let Some(cert_url) = finalized_order.certificate else { info!( "Order finalized, but certificate not yet ready (or failed). Status: {:?}", finalized_order.status ); - } + return Ok(None); + }; + info!("Downloading certificate from: {}", cert_url); + let cert_pem = client.download_certificate(&cert_url).await?; + info!("Certificate received."); - Ok(()) + let (leaf_pem, chain) = if settings.trust.ca_bundle_path.is_some() { + split_leaf_and_chain(&cert_pem)? + } else { + (cert_pem.clone(), Vec::new()) + }; + let cert_pem = match options.leaf_publication { + LeafPublication::LeafOnly => leaf_pem, + LeafPublication::LeafWithChain => { + let mut published = leaf_pem; + for der in &chain { + published.push_str(&encode_cert_pem(der)); + } + published + } + }; + + Ok(Some(IssuedMaterial { + cert_pem, + key_pem: cert_key.serialize_pem(), + chain, + })) } #[cfg(test)] diff --git a/src/daemon.rs b/src/daemon.rs index 42a53e65..c7302cbd 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -224,6 +224,18 @@ pub(crate) async fn run_daemon(invocation: DaemonInvocation) -> anyhow::Result<( handler, maintenance, } = built; + // Before the accept task, deliberately: the adapter initializes + // the per-leaf renewal state from the certificates start-time + // issuance has already made usable, and that reading happens + // before the endpoint begins serving. + spawn_registrar_cert_renewal( + &mut handles, + &settings, + Arc::clone(&endpoint), + insecure_mode, + &shutdown_rx, + ) + .await; spawn_registrar_endpoint( &mut handles, endpoint, @@ -848,6 +860,52 @@ fn spawn_registrar_endpoint( })); } +/// Spawns the adapter that keeps the registrar surface's two leaves +/// valid, when the endpoint is enabled. +/// +/// Called with the endpoint already activated and *before* the accept +/// task is spawned, so the per-leaf renewal state is initialized from +/// the certificates start-time issuance ensured, and is initialized +/// before the endpoint serves anything. The handle joins with every +/// other daemon task, and the adapter takes the same shutdown watch. +/// +/// A preparation that fails is a warning and not a refused start: the +/// endpoint is already up on material that is usable today, and taking +/// it down because renewal could not be armed would turn a future +/// problem into an immediate outage. Nothing is renewed until the next +/// daemon start in that case, which is what the warning says. +/// +/// Where the endpoint is disabled this is never reached at all: no +/// adapter, no state entry, no renewal work, no `OpenBao` request and no +/// CA request. +#[cfg(target_os = "linux")] +async fn spawn_registrar_cert_renewal( + handles: &mut Vec>>, + settings: &Arc, + endpoint: Arc, + insecure_mode: bool, + shutdown_rx: &watch::Receiver, +) { + let renewal = match crate::registrar_renewal::RegistrarCertRenewal::prepare( + Arc::clone(settings), + endpoint, + insecure_mode, + ) + .await + { + Ok(renewal) => renewal, + Err(err) => { + error!( + "Registrar certificate renewal could not be armed, so neither registrar leaf \ + will be renewed under this daemon: {err:#}" + ); + return; + } + }; + let shutdown_rx = shutdown_rx.clone(); + handles.push(tokio::spawn(renewal.run(shutdown_rx))); +} + /// Spawns the task that rotates `OpenBao`'s file audit device, when /// this host is one whose device the daemon owns. /// @@ -1278,7 +1336,30 @@ pub(crate) async fn should_renew( trust: &config::TrustSettings, renew_before: Duration, ) -> anyhow::Result { - let cert_bytes = match tokio::fs::read(&profile.paths.cert).await { + should_renew_certificate(&profile.paths.cert, trust, renew_before).await +} + +/// The same rule, over a certificate path alone. +/// +/// Split out so the registrar surface's two leaves — which have no +/// `[[profiles]]` entry and never will — are judged by this predicate +/// rather than by a second one written beside it. The lead-time gate, +/// the chain-drift gate and the `ca_bundle_path`-unconfigured opt-out +/// are all this function's, so there is one eligibility rule in the +/// daemon and not two. +/// +/// # Errors +/// +/// Returns an error if the certificate cannot be parsed or read for +/// reasons other than `NotFound`. Chain-verification failures or a +/// missing/unreadable bundle force a reissue rather than abort the +/// renewal loop. +pub(crate) async fn should_renew_certificate( + cert_path: &Path, + trust: &config::TrustSettings, + renew_before: Duration, +) -> anyhow::Result { + let cert_bytes = match tokio::fs::read(cert_path).await { Ok(bytes) => bytes, Err(err) if err.kind() == std::io::ErrorKind::NotFound => { info!("Certificate file not found. Issuing a new certificate."); @@ -1287,7 +1368,7 @@ pub(crate) async fn should_renew( Err(err) => { return Err(anyhow::anyhow!( "Failed to read certificate file {}: {err}", - profile.paths.cert.display() + cert_path.display() )); } }; @@ -1311,7 +1392,7 @@ pub(crate) async fn should_renew( Ok(false) => { warn!( "Certificate at {} no longer chains to CA bundle at {}; reissuing.", - profile.paths.cert.display(), + cert_path.display(), bundle_path.display() ); return Ok(true); @@ -1319,7 +1400,7 @@ pub(crate) async fn should_renew( Err(err) => { warn!( "Chain verification of {} against {} failed ({err}); reissuing.", - profile.paths.cert.display(), + cert_path.display(), bundle_path.display() ); return Ok(true); diff --git a/src/fs_util.rs b/src/fs_util.rs index a4ad9151..4c19b375 100644 --- a/src/fs_util.rs +++ b/src/fs_util.rs @@ -1085,6 +1085,20 @@ impl FixedOwner { } } + /// The uid and gid a rollback snapshot recorded off the file it is + /// about to put back. + /// + /// The one production owner that is not stated as `root:root`, and + /// it states nothing of its own: restoring a file means restoring + /// the owner it had, so the ids come off the file that was + /// captured rather than from a policy. A publish that cannot + /// establish them fails and leaves the destination untouched, + /// which is what a rollback that cannot restore has to do. + #[must_use] + pub fn restored(uid: u32, gid: u32) -> Self { + Self { uid, gid } + } + /// Whether this owner is `root:root`, which is what every /// production caller asks for and what the failure message names. fn is_root(self) -> bool { diff --git a/src/lib.rs b/src/lib.rs index 33247d6f..30c4f14f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,10 @@ pub mod openbao; pub mod profile; pub mod registrar; pub(crate) mod registrar_certs; +// The renewal adapter drives the activated endpoint's TLS swap, and the +// endpoint exists on Linux alone. +#[cfg(target_os = "linux")] +pub(crate) mod registrar_renewal; pub mod secret; pub mod service_material; pub mod tls; diff --git a/src/registrar/endpoint.rs b/src/registrar/endpoint.rs index bbf0ecaf..9f81ec3e 100644 --- a/src/registrar/endpoint.rs +++ b/src/registrar/endpoint.rs @@ -35,7 +35,12 @@ //! - [`tls`] is the mutually-authenticated transport the connection is //! wrapped in: the server material the endpoint presents, the //! verifier every client certificate is built against, and the -//! resolver a renewal swaps new material through. +//! assembly a renewal builds a whole replacement configuration with. +//! The replacement is exchanged here, through +//! [`ActivatedEndpoint::swap_active_tls`], and the accept loop loads +//! the active configuration immediately before each handshake — so a +//! renewal takes effect from the next connection with no restart, no +//! signal and no socket rebind. //! - [`serve`] is the accept loop, the bounded connection fleet and the //! drain-then-abort shutdown. //! - [`client`] is the other end of all of it: this repository's @@ -109,7 +114,7 @@ mod tests; use std::os::unix::io::{FromRawFd as _, RawFd}; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, PoisonError, RwLock}; use std::time::Duration; use anyhow::Context as _; @@ -218,15 +223,29 @@ pub(crate) struct ActivatedEndpoint { listener: UnixListener, socket_path: PathBuf, daemon_uid: u32, + // The acceptor and the resolver that built it, replaced together + // and never one without the other. Replacing only the resolver + // would leave the old acceptor — and with it the old *client* + // verifier — deciding every later handshake, so a trust-anchor + // rotation that changed who may connect would not take effect until + // a restart. One `Arc` is swapped wholesale instead, under a lock + // that is taken and released inside a synchronous accessor. + active: RwLock>, + domain: String, +} + +/// The endpoint's live TLS configuration: what a handshake is run +/// against, and the typed resolver inside it. +/// +/// The resolver is retained beside the acceptor because +/// [`rustls::ServerConfig`] keeps only an `Arc` +/// and the trait does not extend `Any`, so there is no way back to the +/// concrete type from a built configuration. Keeping the two together +/// is what makes [`ActivatedEndpoint::cert_resolver`] always the +/// resolver of the acceptor that is actually serving. +pub(crate) struct ActiveTls { acceptor: TlsAcceptor, - // The certificate resolver is retained for exactly one consumer, - // and that consumer — renewal for the endpoint leaf — is a sibling - // issue. It has to be retained now because there is no way back to - // the concrete resolver from a built `ServerConfig`, so a build - // that dropped it would leave renewal with nothing to swap - // through. Reachability is asserted through `cert_resolver`. resolver: Arc, - domain: String, } impl ActivatedEndpoint { @@ -245,25 +264,64 @@ impl ActivatedEndpoint { self.daemon_uid } + /// Returns the configuration in force at the moment of the call. + /// + /// Loaded immediately before each handshake rather than held for the + /// life of the accept loop, so a renewal that exchanged the active + /// configuration is in force from the very next handshake with no + /// restart. The read lock is taken and released inside this + /// synchronous call and never held across an `.await`; what comes + /// out is one `Arc`, so a swap concurrent with this call publishes + /// the whole replacement or none of it. + /// + /// A poisoned lock recovers the guard instead of panicking: a panic + /// elsewhere in the process must not take the endpoint's handshakes + /// down with it, and the value behind the lock is replaced wholesale + /// so there is no half-written state to inherit. + pub(crate) fn active_tls(&self) -> Arc { + Arc::clone(&self.active.read().unwrap_or_else(PoisonError::into_inner)) + } + /// Returns the acceptor every accepted connection is handed to. - pub(crate) fn tls_acceptor(&self) -> &TlsAcceptor { - &self.acceptor + pub(crate) fn tls_acceptor(&self) -> TlsAcceptor { + self.active_tls().acceptor.clone() } /// Returns the resolver that decides which certificate the next /// handshake presents. /// - /// This is the whole of the renewal seam. [`rustls::ServerConfig`] - /// keeps only an `Arc`, and the trait does - /// not extend `Any`, so code holding the configuration or the - /// acceptor cannot reach the concrete resolver at all — it survives - /// here or nowhere. - // The renewal work that calls this is a sibling issue, so until it - // lands nothing outside a test reaches the accessor. It exists now - // so that work needs nothing from this one. + /// Always the resolver of the acceptor currently serving, because + /// the two are swapped together. + // Renewal replaces the whole configuration rather than the + // presented material alone — the old acceptor would otherwise keep + // the old client verifier — so nothing outside a test reaches this + // accessor. It stays because it is the one way back to the concrete + // resolver, which a built `ServerConfig` does not offer. #[allow(dead_code)] - pub(crate) fn cert_resolver(&self) -> &Arc { - &self.resolver + pub(crate) fn cert_resolver(&self) -> Arc { + Arc::clone(&self.active_tls().resolver) + } + + /// Exchanges the whole active configuration, from the next handshake + /// onwards. + /// + /// Infallible by construction: the replacement is built in full by + /// the caller before any live file is touched, so by the time this + /// runs there is nothing left that can fail. Connections already + /// established, and handshakes already in flight, keep the + /// configuration they started with; nothing is dropped, the socket + /// is not rebound and no signal is sent. + pub(crate) fn swap_active_tls( + &self, + config: Arc, + resolver: Arc, + ) { + let replacement = Arc::new(ActiveTls { + acceptor: TlsAcceptor::from(config), + resolver, + }); + let mut guard = self.active.write().unwrap_or_else(PoisonError::into_inner); + *guard = replacement; } /// Returns the configured `network.domain` the presented client @@ -496,8 +554,10 @@ pub(crate) fn adopt( listener, socket_path, daemon_uid: effective_uid, - acceptor: TlsAcceptor::from(server_config), - resolver, + active: RwLock::new(Arc::new(ActiveTls { + acceptor: TlsAcceptor::from(server_config), + resolver, + })), domain, })) } diff --git a/src/registrar/endpoint/client.rs b/src/registrar/endpoint/client.rs index 397fe2c5..48c4a503 100644 --- a/src/registrar/endpoint/client.rs +++ b/src/registrar/endpoint/client.rs @@ -107,6 +107,31 @@ const DIAL_NAME: &str = "registrar-endpoint.invalid"; /// The one response member success and refusal are told apart by. const REFUSAL_DISCRIMINATOR: &str = "class"; +/// How many times one dial reads the client pair before it gives up on +/// finding a matching one. +/// +/// The writer publishes the pair with two separate renames, so a reader +/// that lands between them sees the new certificate beside the old key, +/// or the old certificate beside the new key. That window is the +/// writer's whole publication step and is over in microseconds, so what +/// closes it is a short bounded retry rather than a lock: the reader +/// re-reads both files and keeps whichever read finds a matching pair. +const CLIENT_MATERIAL_READS: usize = 5; + +/// The waits between those reads — one after each of the first four +/// mismatches, and none after the fifth, which is the read the typed +/// mismatch is reported from. +/// +/// This is a fixed local policy of the reader's, and deliberately not +/// the daemon's issuance-retry backoff: it is measured against one +/// `rename(2)` pair on the same host, not against a CA that is down. +const CLIENT_MATERIAL_BACKOFF: [Duration; CLIENT_MATERIAL_READS - 1] = [ + Duration::from_millis(1), + Duration::from_millis(2), + Duration::from_millis(4), + Duration::from_millis(8), +]; + /// Which half of the exchange an I/O failure interrupted. /// /// The one thing an [`io::Error`] cannot say by itself: whether the @@ -189,6 +214,26 @@ pub(crate) enum ClientMaterialError { /// The path that was handed to the client. path: PathBuf, }, + /// Every read of the pair found a key that is not the leaf's. + /// + /// A renewal in flight leaves that state for the length of one + /// `rename(2)`, and the bounded re-read above is what rides it out; + /// reaching this means the two files disagreed on every read, which + /// is a misconfiguration rather than a race. Neither path's bytes + /// are named: one of them is key material. + #[error( + "registrar client certificate {} and key {} did not match on any of {reads} reads", + .certificate_path.display(), + .key_path.display() + )] + KeyMismatch { + /// The certificate path that was handed to the client. + certificate_path: PathBuf, + /// The key path that was handed to the client. + key_path: PathBuf, + /// How many reads found a mismatch. + reads: usize, + }, /// The pair parsed but `rustls` will not authenticate with it. #[error( "registrar client certificate {} and key {} are not usable client authentication \ @@ -712,8 +757,12 @@ fn load_dial_config( certificate_path: &Path, key_path: &Path, ) -> Result { - let (chain, key) = load_client_material(certificate_path, key_path) - .map_err(|source| ExchangeError::Material { source })?; + // `std::thread::sleep` because this whole function is already on + // Tokio's blocking pool, where blocking the thread is what the pool + // is for. The waits are microseconds against one `rename(2)` pair. + let (chain, key) = + load_matching_client_material(certificate_path, key_path, std::thread::sleep) + .map_err(|source| ExchangeError::Material { source })?; build_client_config(pin_file, expected_endpoint_name, chain, key).map_err(|err| match err { ClientConfigError::Pin(source) => ExchangeError::Pin { source }, ClientConfigError::ClientAuth(source) => ExchangeError::Material { @@ -758,13 +807,89 @@ pub(crate) fn build_client_config( .map_err(ClientConfigError::ClientAuth) } +/// Reads the pair until the key is the leaf's, or reports that it never +/// was. +/// +/// This is the reader's half of the publication contract. The writer +/// stages the certificate and the key and `rename(2)`s them one after +/// the other, so the pair is not published atomically even though +/// neither file is ever truncated: a dial landing between the two +/// renames sees a certificate and a key that are not each other's. +/// Presenting that pair would fail the handshake with a signature error +/// that says nothing about what really happened, so it is never +/// presented — the reader re-reads instead, up to +/// [`CLIENT_MATERIAL_READS`] times with the +/// [`CLIENT_MATERIAL_BACKOFF`] waits between them, and returns the +/// typed [`ClientMaterialError::KeyMismatch`] only once every read has +/// disagreed. +/// +/// `wait` is a parameter so the policy is drivable without real waits: +/// a test asserts the exact delays it is asked for, and can publish a +/// matching pair from inside one of them. +/// +/// A read that fails for any *other* reason — an absent file, an +/// unreadable one, a certificate PEM the parser refuses — is returned +/// at once and is not retried. Those are not races, and re-reading them +/// only delays the diagnostic. +/// +/// # Errors +/// +/// Returns whatever [`load_client_material`] reported, or +/// [`ClientMaterialError::KeyMismatch`] when every read found a key +/// that is not the leaf's. +fn load_matching_client_material( + certificate_path: &Path, + key_path: &Path, + mut wait: impl FnMut(Duration), +) -> Result<(Vec>, PrivateKeyDer<'static>), ClientMaterialError> { + for read in 0..CLIENT_MATERIAL_READS { + let (chain, key) = load_client_material(certificate_path, key_path)?; + if client_pair_matches(&chain, &key) { + return Ok((chain, key)); + } + debug!( + certificate = %certificate_path.display(), + read = read + 1, + "The registrar client pair did not match; re-reading it." + ); + if let Some(delay) = CLIENT_MATERIAL_BACKOFF.get(read) { + wait(*delay); + } + } + Err(ClientMaterialError::KeyMismatch { + certificate_path: certificate_path.to_path_buf(), + key_path: key_path.to_path_buf(), + reads: CLIENT_MATERIAL_READS, + }) +} + +/// Reports whether the loaded key is the loaded leaf's. +/// +/// The proof is a signature, through the same helper the endpoint's own +/// loader uses, so there is one matching rule on both sides of the +/// connection rather than two. +/// +/// A key `rustls` cannot sign with at all is *not* reported as a +/// mismatch: re-reading it would find the same key five times and then +/// blame a race that never happened, so it is passed through for +/// `with_client_auth_cert` to refuse with +/// [`ClientMaterialError::Unusable`], which names the real fault. +fn client_pair_matches(chain: &[CertificateDer<'static>], key: &PrivateKeyDer<'static>) -> bool { + let Some(leaf) = chain.first() else { + return false; + }; + let Ok(signing_key) = rustls::crypto::ring::sign::any_supported_type(key) else { + return true; + }; + crate::tls::cert_key_matches(leaf, signing_key.as_ref()) +} + /// Loads the registrar client leaf and its key from disk. /// -/// The single seam every dial's material comes through. It does the -/// plain load and nothing more: it does not verify that the key matches -/// the leaf, does not retry a pair that is momentarily torn by a -/// renewal, and does not inspect `notAfter`. Later work that owns those -/// reasons adds them here rather than restructuring the dial path. +/// The plain load, and nothing more: it does not decide whether the key +/// is the leaf's and it does not inspect `notAfter`. The matching rule +/// and the bounded re-read that rides out a renewal's torn pair live in +/// [`load_matching_client_material`], which is what the dial path calls. /// /// # Errors /// diff --git a/src/registrar/endpoint/client/tests.rs b/src/registrar/endpoint/client/tests.rs index 2a5e6dbb..024ba17a 100644 --- a/src/registrar/endpoint/client/tests.rs +++ b/src/registrar/endpoint/client/tests.rs @@ -747,6 +747,186 @@ async fn a_pem_key_rustls_will_not_load_fails_before_the_dial() { assert_eq!(double.observed().connections, 0); } +// --------------------------------------------------------------------- +// The per-dial reload contract +// --------------------------------------------------------------------- + +/// A reader that lands between the writer's two renames sees a +/// certificate and a key that are not each other's, and re-reads until +/// it finds a pair that is — it never presents the mismatch. +/// +/// The torn state is published from inside the retry's own wait, so the +/// case is deterministic and costs no wall-clock time: nothing sleeps, +/// and the pair is repaired at exactly the point the reader is about to +/// look again. +#[test] +fn a_reader_between_the_two_renames_retries_until_the_pair_matches() { + let deployment = Deployment::new(); + + // The state the *first* rename leaves: the new certificate beside + // the old key. + let renewed_cert = deployment.pki.path("renewed.crt"); + let renewed_key = deployment.pki.path("renewed.key"); + write_leaf_material( + &deployment.pki.ca, + vec![dns_san(®istrar_client_name())], + &renewed_cert, + &renewed_key, + true, + ); + std::fs::copy(&renewed_cert, &deployment.certificate_path).expect("the first rename"); + + let waits = StdMutex::new(Vec::new()); + let (chain, key) = super::load_matching_client_material( + &deployment.certificate_path, + &deployment.key_path, + |delay| { + waits.lock().expect("not poisoned").push(delay); + // The second rename, landing while the reader waits. + std::fs::copy(&renewed_key, &deployment.key_path).expect("the second rename"); + }, + ) + .expect("the reader finds the matching pair"); + + assert_eq!( + waits.into_inner().expect("not poisoned"), + vec![Duration::from_millis(1)], + "one mismatch costs one wait, and the next read finds the pair" + ); + assert!( + super::client_pair_matches(&chain, &key), + "the loader never returns a pair it knows is mismatched" + ); + assert_eq!( + chain.first().map(|leaf| leaf.as_ref().to_vec()), + Some( + rustls_pemfile::certs(&mut std::io::BufReader::new( + std::fs::read(&renewed_cert) + .expect("read the renewed leaf") + .as_slice() + )) + .next() + .expect("one certificate") + .expect("it parses") + .as_ref() + .to_vec() + ), + "the pair it returns is the renewed one, not the pre-rename one" + ); +} + +/// A pair that never matches costs exactly five reads and the four +/// documented waits, and then returns the typed mismatch rather than +/// presenting anything. +#[test] +fn a_permanent_mismatch_reads_five_times_and_returns_the_typed_error() { + let deployment = Deployment::new(); + // A key from a different leaf, and nothing repairs it. + let (_certificate, key) = + issue_leaf(&deployment.pki.ca, vec![dns_san(®istrar_client_name())]); + std::fs::write(&deployment.key_path, key.serialize_pem()).expect("write the foreign key"); + + let waits = StdMutex::new(Vec::new()); + let err = super::load_matching_client_material( + &deployment.certificate_path, + &deployment.key_path, + |delay| waits.lock().expect("not poisoned").push(delay), + ) + .expect_err("a pair that never matches is never presented"); + + assert!( + matches!( + err, + ClientMaterialError::KeyMismatch { reads, .. } if reads == super::CLIENT_MATERIAL_READS + ), + "{err:?}" + ); + assert_eq!( + waits.into_inner().expect("not poisoned"), + vec![ + Duration::from_millis(1), + Duration::from_millis(2), + Duration::from_millis(4), + Duration::from_millis(8), + ], + "four waits after the first four of five reads, and none after the fifth" + ); +} + +/// The mismatch never reaches the socket: a dial whose pair is +/// permanently torn opens no connection. +#[tokio::test] +async fn a_permanently_torn_pair_never_reaches_the_socket() { + let deployment = Deployment::new(); + let double = deployment.double(answered(response_frame(MINT_SUCCESS))); + let (_certificate, key) = + issue_leaf(&deployment.pki.ca, vec![dns_san(®istrar_client_name())]); + std::fs::write(&deployment.key_path, key.serialize_pem()).expect("write the foreign key"); + + let err = deployment + .client() + .mint(register_request()) + .await + .expect_err("a torn pair is refused before the dial"); + + assert!( + matches!( + err, + ExchangeError::Material { + source: ClientMaterialError::KeyMismatch { .. } + } + ), + "{err:?}" + ); + assert_eq!(double.observed().connections, 0); +} + +/// The next dial rereads the pair, so a renewal that landed between two +/// requests is presented without the caller being recreated or told. +#[tokio::test] +async fn the_next_dial_presents_a_renewed_pair_without_recreating_the_caller() { + let deployment = Deployment::new(); + let double = deployment.double(answered(response_frame(MINT_SUCCESS))); + let client = deployment.client(); + + client + .mint(register_request()) + .await + .expect("the first exchange succeeds"); + let first = double + .observed() + .peer_leaves + .first() + .cloned() + .expect("the double saw the first client leaf"); + + // A renewal replaces the pair under the same paths, exactly as the + // publication does. + write_leaf_material( + &deployment.pki.ca, + vec![dns_san(®istrar_client_name())], + &deployment.certificate_path, + &deployment.key_path, + true, + ); + + client + .mint(register_request()) + .await + .expect("the second exchange succeeds over the renewed pair"); + let second = double + .observed() + .peer_leaves + .get(1) + .cloned() + .expect("the double saw a second client leaf"); + + assert_ne!( + first, second, + "the same client presented the renewed leaf on its next dial" + ); +} + #[tokio::test] async fn a_request_over_the_frame_bound_is_refused_locally() { let deployment = Deployment::new(); diff --git a/src/registrar/endpoint/serve.rs b/src/registrar/endpoint/serve.rs index fb4102d8..7dc435c7 100644 --- a/src/registrar/endpoint/serve.rs +++ b/src/registrar/endpoint/serve.rs @@ -381,7 +381,13 @@ async fn handshake( accepted_at: Instant, ) -> Option> { let deadline = accepted_at + HANDSHAKE_TIMEOUT; - match timeout_at(deadline, endpoint.tls_acceptor().accept(stream)).await { + // Loaded here, immediately before the handshake, rather than held + // for the life of the accept loop: a renewal that exchanged the + // active configuration is in force from this connection onwards, + // and one that lands while this handshake runs leaves it on the + // configuration it started with. + let acceptor = endpoint.tls_acceptor(); + match timeout_at(deadline, acceptor.accept(stream)).await { Ok(Ok(tls)) => Some(tls), Ok(Err(err)) => { warn!( diff --git a/src/registrar/endpoint/tests.rs b/src/registrar/endpoint/tests.rs index cd035bb3..5f7ba9ab 100644 --- a/src/registrar/endpoint/tests.rs +++ b/src/registrar/endpoint/tests.rs @@ -2968,9 +2968,7 @@ fn the_activated_endpoint_carries_no_handler() { "listener: UnixListener,", "socket_path: PathBuf,", "daemon_uid: u32,", - "acceptor: TlsAcceptor,", - "// The certificate resolver is retained for exactly one consumer,", - "resolver: Arc,", + "active: RwLock>,", "domain: String,", ], "the adopted endpoint holds the socket, its path, the daemon uid and the TLS material — \ @@ -3527,6 +3525,86 @@ async fn swapping_the_resolver_through_the_activated_endpoint_changes_the_presen running.stop().await; } +/// Exchanging the whole active configuration changes both halves of the +/// handshake at once: what the endpoint presents *and* which client +/// anchors it accepts. +/// +/// Replacing the resolver alone could not do this. The acceptor retains +/// the client verifier it was built with, so a trust-anchor rotation +/// that widened or narrowed who may connect would go on being decided by +/// the old verifier until a restart. The whole configuration is +/// replaced instead, and the accept loop loads it immediately before +/// each handshake. +/// +/// The socket is not rebound — its inode is asserted unchanged — no +/// signal is sent, and a connection established before the swap is still +/// connected after it. +#[tokio::test] +async fn exchanging_the_active_configuration_changes_the_presented_chain_and_the_accepted_anchors() +{ + let harness = Harness::bind().expect("harness"); + let running = RunningEndpoint::start( + &harness.endpoint, + Arc::new(EchoHandler { + response: b"served".to_vec(), + }), + ); + let inode_before = std::fs::metadata(&harness.socket_path) + .expect("stat the socket") + .ino(); + + let first = harness.round_trip(&frame_of(b"mint", b"")).await; + assert!(!first.is_empty(), "the original configuration serves"); + + // A connection opened before the exchange and held across it. + let held = tokio::net::UnixStream::connect(&harness.socket_path) + .await + .expect("connect before the exchange"); + + // The next trust generation: new server material for the same + // endpoint name, and a client leaf under the new anchor. The old + // generation's client is not in the new anchor set at all. + let rotated = Pki::new(); + let (config, resolver) = rotated.conforming(); + harness.endpoint.swap_active_tls(config, resolver); + + assert_eq!( + inode_before, + std::fs::metadata(&harness.socket_path) + .expect("stat the socket") + .ino(), + "a TLS replacement never rebinds or replaces the socket" + ); + assert!( + held.peer_addr().is_ok(), + "a connection already in flight is not dropped" + ); + + // The caller pinned to the old anchor, presenting the old client + // leaf, is refused on both counts. + let outcome = tls_connect(&harness.socket_path, harness.pki.registrar_client_config()).await; + assert!( + outcome.is_err(), + "the previous generation must no longer complete a handshake" + ); + + // The caller of the new generation is accepted: its leaf verifies + // against the rebuilt incoming verifier, and the chain it is + // presented matches the new pin file. + let config = client_config_pinning( + &rotated.pin_file_path(), + Some(rotated.registrar_client_material()), + ); + let served = tls_round_trip(&harness.socket_path, config, &frame_of(b"mint", b"")).await; + assert!( + !served.is_empty(), + "the exchanged configuration serves the new generation with no restart: {served:?}" + ); + + drop(held); + running.stop().await; +} + // --------------------------------------------------------------------- // The configuration builder: every startup refusal, over its inputs // --------------------------------------------------------------------- diff --git a/src/registrar/endpoint/tls.rs b/src/registrar/endpoint/tls.rs index b3be3880..45aef445 100644 --- a/src/registrar/endpoint/tls.rs +++ b/src/registrar/endpoint/tls.rs @@ -31,13 +31,22 @@ //! the outbound path applies. It is never built with //! `allow_unauthenticated()`: a caller presenting no certificate //! fails the handshake rather than arriving unauthenticated. -//! - **The swap seam.** [`EndpointCertResolver`] holds the presented -//! [`CertifiedKey`] behind an `RwLock` and swaps it under the next -//! handshake with no restart. [`rustls::ServerConfig`] keeps only an +//! - **The swap seam.** A renewal replaces the endpoint's *whole* +//! configuration rather than the material it presents, through +//! [`super::ActivatedEndpoint::swap_active_tls`]: the old acceptor +//! retains the old client verifier, so replacing only the presented +//! certificate would leave a trust-anchor rotation undecided until a +//! restart. [`build_server_config_from_staged`] is what builds that +//! replacement, from a pair and from the exact bytes the merged CA +//! bundle *will* hold, so the whole configuration exists before any +//! live path changes. +//! +//! [`EndpointCertResolver`] is the typed handle inside it. +//! [`rustls::ServerConfig`] keeps only an //! `Arc` and the trait does not extend `Any`, -//! so the typed handle is returned alongside the configuration and -//! retained by [`super::ActivatedEndpoint`]; there is no way back to -//! it from a built configuration. +//! so it is returned alongside the configuration and retained beside +//! the acceptor it built; there is no way back to it from a built +//! configuration. //! //! # Where a refusal is decided //! @@ -258,10 +267,17 @@ impl EndpointCertResolver { /// Replaces the certificate and key presented from the next /// handshake onwards, with no restart and without disturbing a /// connection already established. - // Certificate renewal for the endpoint leaf is a sibling issue, so - // until it lands the only caller of the seam is the test that - // proves it is reachable from the `ActivatedEndpoint` the daemon - // holds. + /// + /// This is **not** what a renewal uses, and it is documented here so + /// that stays visible: the acceptor around this resolver keeps the + /// client verifier it was built with, so a swap here changes what + /// the endpoint presents and nothing about who may connect. A + /// renewal exchanges the whole configuration through + /// [`super::ActivatedEndpoint::swap_active_tls`] instead. + // Nothing in production replaces the presented material alone, for + // the reason above; the method stays because it is the resolver's + // whole reason for being a typed handle, and it is exercised where + // that handle is reached from the endpoint the daemon holds. #[allow(dead_code)] pub(crate) fn swap(&self, certified_key: CertifiedKey) { let mut guard = self @@ -332,10 +348,80 @@ pub(crate) fn build_server_config( setting: TRUSTED_CA_SETTING, }); } + let bundle_bytes = read_file(bundle_path, CA_BUNDLE_SETTING)?; + assemble_server_config( + certified_key, + &expected_name, + cert_path, + bundle_path, + &bundle_bytes, + pins, + ) +} + +/// Builds the endpoint's *next* configuration from material a renewal +/// already has in hand, without reading the live CA bundle. +/// +/// The difference from [`build_server_config`] is the bundle: this takes +/// the exact bytes the merged bundle *will* hold once publication +/// succeeds, so the incoming client verifier is built from the +/// post-merge anchor set and the whole configuration is finished before +/// any live path changes. Everything else — the pair load, the key +/// match, the endpoint-SAN rule, the pinned-anchor restriction and the +/// caller's own self-check — is the same code on the same rules. +/// +/// The anchor set is still exactly the pinned subset of those bytes. +/// The endpoint pin file decides whether a *replacement server leaf* is +/// safe for pinned callers and is not consulted here; the returned +/// issuer chain on its own, the unpinned remainder of the bundle and +/// the system roots are all equally not anchors. +/// +/// # Errors +/// +/// The same [`EndpointTlsError`]s [`build_server_config`] returns, +/// except the ones about reading the bundle file: those bytes arrived +/// as an argument. +pub(crate) fn build_server_config_from_staged( + server_cert_path: &Path, + server_key_path: &Path, + ca_bundle_path: &Path, + ca_bundle_bytes: &[u8], + pins: &[String], + domain: &str, +) -> Result<(Arc, Arc), EndpointTlsError> { + tls::install_crypto_provider(); + + let certified_key = load_certified_key(server_cert_path, server_key_path)?; + let expected_name = endpoint_san(&certified_key, server_cert_path, domain)?; + if pins.is_empty() { + return Err(EndpointTlsError::MissingSetting { + setting: TRUSTED_CA_SETTING, + }); + } + assemble_server_config( + certified_key, + &expected_name, + server_cert_path, + ca_bundle_path, + ca_bundle_bytes, + pins, + ) +} + +/// The half both builders share: the pinned anchor set, the caller's own +/// self-check, and the mutually-authenticated configuration itself. +fn assemble_server_config( + certified_key: CertifiedKey, + expected_name: &str, + cert_path: &Path, + bundle_path: &Path, + bundle_bytes: &[u8], + pins: &[String], +) -> Result<(Arc, Arc), EndpointTlsError> { let pin_set: HashSet = pins.iter().map(|pin| pin.to_ascii_lowercase()).collect(); - let anchors = pinned_bundle_anchors(bundle_path, &pin_set)?; + let anchors = pinned_bundle_anchors(bundle_path, bundle_bytes, &pin_set)?; - self_check(&certified_key, cert_path, &expected_name, &pin_set)?; + self_check(&certified_key, cert_path, expected_name, &pin_set)?; let roots = tls::certs_to_root_store(&anchors).map_err(|err| EndpointTlsError::ClientVerifier { @@ -444,11 +530,11 @@ fn leaf_of<'a>( /// has no meaning for client authentication. fn pinned_bundle_anchors( bundle_path: &Path, + contents: &[u8], pins: &HashSet, ) -> Result>, EndpointTlsError> { - let contents = read_file(bundle_path, CA_BUNDLE_SETTING)?; let certs = - tls::parse_pem_to_cert_list(&contents).map_err(|_| EndpointTlsError::Unparsable { + tls::parse_pem_to_cert_list(contents).map_err(|_| EndpointTlsError::Unparsable { setting: CA_BUNDLE_SETTING, path: bundle_path.to_path_buf(), })?; diff --git a/src/registrar_certs.rs b/src/registrar_certs.rs index eb64aa72..b11f08f3 100644 --- a/src/registrar_certs.rs +++ b/src/registrar_certs.rs @@ -126,7 +126,7 @@ const SURFACE_LEAF_PUBLICATION: LeafPublication = LeafPublication::LeafWithChain /// The two are evaluated and issued independently: one being usable is /// never a reason to leave the other unusable, and one needing issuance /// is never a reason to re-issue the other. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) enum SurfaceLeaf { /// The leaf the endpoint presents. A server certificate, so it takes /// the ordinary CSR shape and requests no extended key usage. @@ -177,6 +177,17 @@ impl SurfaceLeaf { } } + /// How this leaf is named in a log line or a state entry. + // Read by the renewal adapter, which exists on Linux alone. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[must_use] + pub(crate) fn label(self) -> &'static str { + match self { + Self::EndpointServer => "the endpoint server leaf", + Self::RegistrarClient => "the registrar client leaf", + } + } + /// How the certificate path is spelled in a diagnostic. #[must_use] pub(crate) fn cert_setting(self) -> &'static str { @@ -807,6 +818,71 @@ pub(crate) async fn issue_surface_pair( }) } +/// Issues one pair **off-live**: the issuer chain, the candidate +/// certificate and a fresh key, with nothing written anywhere. +/// +/// The renewal path's issuance step. It runs the same outbound ACME +/// path [`issue_surface_pair`] runs, under the same reserved-name +/// profile and the same bootstrap-pin rule, and stops immediately before +/// publication: it writes neither the configured pair nor +/// `trust.ca_bundle_path`, so a candidate that turns out to be +/// unusable — a key that is not the leaf's, a SAN that drifted, a chain +/// that reaches no pinned anchor — costs the running endpoint and the +/// running caller nothing at all. +/// +/// Every call produces a fresh key. There is no reuse path and no +/// option to ask for one. +/// +/// # Errors +/// +/// Returns an error naming both configured paths and the failure, +/// including an order that finalized without a certificate — which is +/// merely "nothing yet" to a start-time issuance and is a failed +/// renewal attempt here. +// Called by the renewal adapter, which exists on Linux alone, and by +// this module's own tests everywhere. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub(crate) async fn issue_surface_pair_material( + settings: &Settings, + pair: &SurfacePairPaths, + host: &str, + inputs: &SurfaceAcmeInputs, + insecure_mode: bool, +) -> Result { + let issuance = issuance_settings(settings, pair, host, &inputs.responder_hmac); + let profile = issuance + .profiles + .first() + .ok_or_else(|| anyhow::anyhow!("the registrar surface issuance profile was not built"))?; + let bootstrap_pins = bootstrap_pins_for_mode(&issuance.trust, insecure_mode); + let material = crate::acme::issue_certificate_material( + &issuance, + profile, + inputs.eab.clone(), + insecure_mode, + pair.leaf.issuance_options(), + bootstrap_pins, + ) + .await + .with_context(|| { + format!( + "issuing a replacement {} for {} and {}", + pair.name, + pair.cert_path.display(), + pair.key_path.display() + ) + })?; + material.ok_or_else(|| { + anyhow::anyhow!( + "the order for {} finalized without a certificate, so there is no candidate to \ + validate for {} and {}", + pair.name, + pair.cert_path.display(), + pair.key_path.display() + ) + }) +} + /// Selects pin-only TLS only for a missing or parse-empty output bundle. /// /// A different read error remains on the ordinary path, whose existing diff --git a/src/registrar_certs/tests.rs b/src/registrar_certs/tests.rs index 534c2f39..0ab1a871 100644 --- a/src/registrar_certs/tests.rs +++ b/src/registrar_certs/tests.rs @@ -1474,6 +1474,97 @@ async fn the_published_key_is_never_group_or_world_readable() { } } +/// The off-live issuance the renewal path runs returns the chain, the +/// candidate certificate and a fresh key, and writes nothing: neither +/// configured path and not the CA bundle. +/// +/// This is the property the whole renewal transaction rests on. If this +/// call published anything, a candidate that later failed validation +/// would already have replaced material a running endpoint and a running +/// caller are reading. +#[tokio::test] +async fn the_off_live_issuance_returns_material_and_publishes_nothing() { + let mut host = Host::new(); + let acme = start_acme(Arc::clone(&host.ca)).await; + aim_at(&mut host.settings, &acme); + host.provision_both_pairs(); + let bundle_path = host + .settings + .trust + .ca_bundle_path + .clone() + .expect("a configured bundle"); + let before_bundle = digest_of(&bundle_path); + + let pairs = surface_pairs(host.endpoint(), TEST_HOST, TEST_DOMAIN).expect("pairs resolve"); + let inputs = openbao_inputs(); + for pair in &pairs { + let before_cert = digest_of(&pair.cert_path); + let before_key = digest_of(&pair.key_path); + + let material = issue_surface_pair_material(&host.settings, pair, TEST_HOST, &inputs, false) + .await + .expect("the off-live issuance produces material"); + + assert_eq!( + single_dns_san(&pem_to_der(&material.cert_pem)).expect("one DNS SAN"), + pair.name, + "the candidate carries the reserved name for its pair" + ); + assert!( + !material.chain.is_empty(), + "the issuer chain comes back so the merged bundle can be staged" + ); + assert!( + material.cert_pem.matches("BEGIN CERTIFICATE").count() >= 2, + "the candidate is published as the leaf followed by its issuer chain" + ); + assert_eq!( + before_cert, + digest_of(&pair.cert_path), + "the live certificate must be untouched" + ); + assert_eq!( + before_key, + digest_of(&pair.key_path), + "the live key must be untouched" + ); + assert_eq!( + before_bundle, + digest_of(&bundle_path), + "the live CA bundle must be untouched" + ); + } +} + +/// Every off-live issuance generates its own key, so a renewal can never +/// republish the key the leaf it is replacing was minted under. +#[tokio::test] +async fn two_off_live_issuances_of_the_same_name_produce_different_keys() { + let mut host = Host::new(); + let acme = start_acme(Arc::clone(&host.ca)).await; + aim_at(&mut host.settings, &acme); + let pairs = surface_pairs(host.endpoint(), TEST_HOST, TEST_DOMAIN).expect("pairs resolve"); + let client = pairs + .iter() + .find(|pair| pair.leaf == SurfaceLeaf::RegistrarClient) + .expect("the client pair"); + let inputs = openbao_inputs(); + + let first = issue_surface_pair_material(&host.settings, client, TEST_HOST, &inputs, false) + .await + .expect("first candidate"); + let second = issue_surface_pair_material(&host.settings, client, TEST_HOST, &inputs, false) + .await + .expect("second candidate"); + + assert_ne!( + first.key_pem, second.key_pem, + "each issuance must generate a key" + ); + assert_ne!(first.cert_pem, second.cert_pem); +} + /// Issuance leaves the endpoint anchor pin file untouched. The pin is /// over trust **anchors**, not over the leaf, and is written once by the /// provisioning tool. diff --git a/src/registrar_renewal.rs b/src/registrar_renewal.rs new file mode 100644 index 00000000..f9486e26 --- /dev/null +++ b/src/registrar_renewal.rs @@ -0,0 +1,1249 @@ +//! Renewal of the registrar surface's two leaves, on the daemon's own +//! loop. +//! +//! [`crate::registrar_certs`] mints both leaves once, at start, and +//! leaves a usable pair alone. Both expire. This module is what keeps +//! them valid afterwards: one daemon-owned adapter, created only where +//! the endpoint is enabled, owning both leaves, taking the daemon's +//! shutdown signal and joined with the daemon's other tasks. +//! +//! # What a pass does, and in what order +//! +//! Nothing on the live paths until everything that can fail has already +//! succeeded. For a leaf that is due: +//! +//! 1. **Issue off-live.** The same outbound ACME path start-time +//! issuance runs, under the same bootroot-internal credential, but +//! stopped before publication — the chain, the candidate certificate +//! and a fresh key come back as values. Nothing is written at +//! `[registrar_endpoint]`'s paths and nothing at +//! `[trust] ca_bundle_path`. +//! 2. **Validate the candidate.** Its key is the leaf's; a client +//! candidate carries the reserved registrar client SAN and +//! recognises to the same instance, host and domain as the name the +//! plan composed; a server candidate carries the exact endpoint SAN +//! and chains to an anchor already in the endpoint pin file. An +//! invalid or unpinned candidate is a refusal, not a repair: it is +//! discarded and no live or active state changes. +//! 3. **Stage the merged bundle.** The bytes `[trust] ca_bundle_path` +//! *would* hold — the existing bundle's pinned certificates plus the +//! candidate's chain — computed but not written. +//! 4. **Build the next configuration.** A complete +//! [`rustls::ServerConfig`], from the candidate server pair when +//! that is the leaf being renewed and from the live server pair when +//! it is not, and from the staged bundle in either case. The +//! incoming client verifier is rebuilt here, from the post-merge +//! bundle's pinned subset and from nothing else. +//! 5. **Snapshot, then publish.** The affected live paths' bytes, modes +//! and ownership are captured, then the merged bundle is replaced +//! atomically and the certificate and key are written through the +//! established two-rename contract. +//! 6. **Exchange the active configuration.** Only after every live +//! write succeeded, and infallibly, because the replacement was +//! built in step 4. The socket is not rebound, no signal is sent, +//! and connections and handshakes already in flight keep the +//! configuration they started with. +//! +//! A publication that fails restores every live path whose write may +//! have started, leaves the old configuration installed, records the +//! failure and lets the next tick try again. A rollback that itself +//! fails is recorded as what it is — both errors, and no claim that the +//! files were restored. +//! +//! # What it deliberately does not do +//! +//! It never rewrites the endpoint pin file and never adds a leaf +//! fingerprint to it: the pin file decides whether a *replacement server +//! leaf* is safe for pinned callers, and a renewal that could edit it +//! would be deciding that about itself. It never repairs the +//! bootroot-internal credential and imposes no ordering on a CA +//! rotation — a tick that reaches a leaf while that credential cannot +//! authenticate records an ordinary failed issuance and retries. It +//! reads no `role_id` and no `secret_id`, and never uses the per-service +//! ACME renewal path. +//! +//! # The accessor +//! +//! [`RegistrarCertRenewalState`] is the whole of this module's +//! interface to anything else in the process: per leaf, the observed +//! `notAfter`, the outcome of the last attempt, and when that attempt +//! ran. It is written on this loop's own tick and nowhere else, and +//! nothing here puts anything on the wire. + +use std::collections::BTreeMap; +use std::future::Future; +use std::os::unix::fs::MetadataExt as _; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use rustls::pki_types::{CertificateDer, UnixTime}; +use time::OffsetDateTime; +use tokio::sync::watch; +use tracing::{debug, error, info, warn}; + +use crate::cert_group::{CERT_FILE_MODE, CertGroupPolicy, KEY_FILE_MODE_DEFAULT}; +use crate::config::Settings; +use crate::fs_util::{Destination, FixedOwner, StagedMode}; +use crate::registrar::endpoint::ActivatedEndpoint; +use crate::registrar::endpoint::tls::build_server_config_from_staged; +use crate::registrar::endpoint_pin::{ + self, EndpointPinError, EndpointVerifyRejection, RegistrarEndpointVerifier, +}; +use crate::registrar::internal::{InternalPaths, load_internal_config}; +use crate::registrar::{ + recognize_registrar_client_name, recognize_registrar_endpoint_name, single_dns_san, +}; +use crate::registrar_certs::{ + SurfaceAcmeInputs, SurfaceLeaf, SurfacePairPaths, SurfacePlan, issue_surface_pair_material, + read_acme_inputs, resolve_surface_plan, +}; +use crate::{daemon, fs_util, tls, utils}; + +/// The basename the candidate certificate is staged under, inside the +/// attempt's private working directory. +const CANDIDATE_CERT_FILE: &str = "candidate.crt"; + +/// The basename the candidate key is staged under. +const CANDIDATE_KEY_FILE: &str = "candidate.key"; + +/// The prefix the attempt's private working directory is created with, +/// so an operator who finds one knows what left it. +const ARTIFACT_DIR_PREFIX: &str = "bootroot-registrar-renewal."; + +// --------------------------------------------------------------------- +// The accessor +// --------------------------------------------------------------------- + +/// How the last renewal attempt for one leaf ended. +/// +/// "No attempt yet" is a state of its own rather than an absent +/// outcome: it is what initialization records, and it is the only one +/// of the three that carries no timestamp. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum RenewalAttempt { + /// Initialization observed the leaf and nothing has been attempted + /// since. Reached without an `OpenBao` request or a CA request. + NeverAttempted, + /// The last attempt issued, validated and published a replacement, + /// and exchanged the active configuration. + Succeeded, + /// The last attempt failed, and the leaf still holds whatever it + /// held before. Every ordinary issuance, publication, rollback, + /// authentication and pin-file refusal lands here. + Failed { + /// What went wrong, rendered for an operator. + reason: String, + }, +} + +/// What the accessor holds about one leaf. +/// +/// These three values are exactly what the reporting sibling publishes, +/// so they are written here and read there. Until that sibling lands +/// nothing outside a test reads them, and they carry the same +/// justification [`RegistrarCertRenewalState::leaf`] records: a reader +/// that re-derived a lifetime from disk is the thing this seam exists +/// to forbid. +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub(crate) struct LeafRenewalState { + /// The `notAfter` of the certificate currently at the leaf's + /// configured path. Replaced by a success and retained by a failure. + pub(crate) not_after: OffsetDateTime, + /// How the last attempt ended. + pub(crate) attempt: RenewalAttempt, + /// When that attempt ran, absent while nothing has been attempted. + pub(crate) attempted_at: Option, +} + +/// The one place a registrar leaf's lifetime and last outcome are +/// discovered. +/// +/// Written on the renewal loop's own tick, never on a request path, and +/// never recomputed from disk by a reader: a member that stat'ed and +/// parsed a certificate per request would put filesystem work behind an +/// endpoint a caller can drive. +#[derive(Clone, Default)] +pub(crate) struct RegistrarCertRenewalState { + inner: Arc>>, +} + +impl RegistrarCertRenewalState { + /// Records a leaf's observed `notAfter` with no attempt behind it. + /// + /// The one write that is not an attempt: it reads a certificate that + /// is already on disk and contacts neither `OpenBao` nor the CA. + pub(crate) fn initialize(&self, leaf: SurfaceLeaf, not_after: OffsetDateTime) { + self.with(|entries| { + entries.insert( + leaf, + LeafRenewalState { + not_after, + attempt: RenewalAttempt::NeverAttempted, + attempted_at: None, + }, + ); + }); + } + + /// Records a successful attempt: the new `notAfter`, and when it ran. + /// + /// Creates the entry when initialization could not read the + /// certificate that was there — that leaf has no lifetime to retain + /// and no entry, and this attempt has just given it one. Every other + /// write updates in place. + pub(crate) fn record_success( + &self, + leaf: SurfaceLeaf, + not_after: OffsetDateTime, + at: OffsetDateTime, + ) { + self.with(|entries| { + entries.insert( + leaf, + LeafRenewalState { + not_after, + attempt: RenewalAttempt::Succeeded, + attempted_at: Some(at), + }, + ); + }); + } + + /// Records a failed attempt, retaining the `notAfter` already there. + /// + /// The leaf is still valid until that `notAfter`, and the whole + /// point of recording the failure while it is is that the window + /// between the two is the only interval in which anything can be + /// repaired quietly. + /// + /// A leaf with no entry gets none here: the entry's whole content is + /// a lifetime a failure retains, and a failed attempt has no + /// lifetime to put in one. + pub(crate) fn record_failure(&self, leaf: SurfaceLeaf, reason: &str, at: OffsetDateTime) { + self.with(|entries| { + if let Some(entry) = entries.get_mut(&leaf) { + entry.attempt = RenewalAttempt::Failed { + reason: reason.to_string(), + }; + entry.attempted_at = Some(at); + } + }); + } + + /// Returns one leaf's state, or `None` where the endpoint is + /// disabled and no entry was ever made. + /// + /// The read side of the accessor. Its consumer — the reporting + /// sibling that publishes these three values in the endpoint health + /// container — is a separate issue, so nothing outside a test reads + /// it yet. It exists now because the accessor is the whole interface + /// between the two, and a reader that had to re-derive a lifetime + /// from disk is exactly what that seam forbids. + #[allow(dead_code)] + #[must_use] + pub(crate) fn leaf(&self, leaf: SurfaceLeaf) -> Option { + self.with(|entries| entries.get(&leaf).cloned()) + } + + /// How many leaves have entries. Two on an enabled endpoint, zero + /// on a disabled one. + #[must_use] + pub(crate) fn len(&self) -> usize { + self.with(|entries| entries.len()) + } + + /// Runs `body` under the lock. + /// + /// A poisoned lock recovers the guard rather than panicking: a panic + /// elsewhere must not make the daemon's certificate state + /// unreadable, and every value behind the lock is replaced wholesale. + fn with(&self, body: impl FnOnce(&mut BTreeMap) -> T) -> T { + let mut guard = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + body(&mut guard) + } +} + +impl std::fmt::Debug for RegistrarCertRenewalState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RegistrarCertRenewalState") + .field("leaves", &self.len()) + .finish() + } +} + +// --------------------------------------------------------------------- +// Candidate validation +// --------------------------------------------------------------------- + +/// Why a freshly issued candidate is not publishable. +/// +/// Every variant is a refusal that changes nothing: the candidate is +/// discarded, the live pair, the live bundle, the active configuration +/// and the pin file are all left exactly as they were, and the accessor +/// records the reason against a retained `notAfter`. +#[derive(Debug, thiserror::Error)] +pub(crate) enum CandidateRejection { + /// The issued certificate or key does not parse, or the key is of a + /// type this build cannot sign with. + #[error("the issued candidate does not parse as a certificate and a usable private key")] + Malformed, + /// The fresh key is not the candidate leaf's. + #[error("the issued candidate's private key is not the key of the leaf beside it")] + KeyMismatched, + /// The candidate does not carry exactly one DNS SAN equal to the + /// reserved name this pair runs under. + #[error( + "the issued candidate's subject alternative name is {found:?} rather than the reserved \ + name {expected}" + )] + SanMismatched { + /// The name the plan composed for this pair. + expected: String, + /// What the candidate carried, when it carried one DNS SAN. + found: Option, + }, + /// The SAN is the reserved name but does not recognise as the + /// registrar identity it has to be in the configured domain. + #[error("the issued candidate's subject alternative name is not recognised: {source}")] + Unrecognised { + /// Which rule the name broke. + #[source] + source: crate::registrar::RegistrarIdentityError, + }, + /// The endpoint pin file could not be read or parsed, so no server + /// candidate can be judged safe for a pinned caller. + #[error("the endpoint pin file could not be read: {source}")] + PinFileUnusable { + /// What the pin loader reported. + #[source] + source: EndpointPinError, + }, + /// The server candidate does not chain to any anchor the pin file + /// already names, so every pinned caller would refuse it. + #[error( + "the issued server candidate does not chain to an anchor in the endpoint pin file at \ + {}: {source}", + .pin_file.display() + )] + Unpinned { + /// The pin file the anchor set came from. It is never rewritten. + pin_file: PathBuf, + /// The rejection a pinned caller's own verifier produced. + #[source] + source: EndpointVerifyRejection, + }, +} + +/// Holds a candidate to every rule that decides whether it may replace +/// live material. +/// +/// Runs before any live write, over bytes alone. `pin_file` is consulted +/// for a server candidate and only to read it: the anchor set it names +/// is the deployment's, and a renewal that could add its own leaf to it +/// would be pinning itself. +pub(crate) fn validate_candidate( + leaf: SurfaceLeaf, + cert_pem: &[u8], + key_pem: &[u8], + expected_name: &str, + domain: &str, + pin_file: &Path, +) -> Result<(), CandidateRejection> { + tls::install_crypto_provider(); + + let chain: Vec> = + rustls_pemfile::certs(&mut std::io::BufReader::new(cert_pem)) + .collect::, _>>() + .map_err(|_| CandidateRejection::Malformed)?; + let candidate = chain.first().ok_or(CandidateRejection::Malformed)?; + + // The key is key material, so nothing below quotes a byte of it. + let key = rustls_pemfile::private_key(&mut std::io::BufReader::new(key_pem)) + .ok() + .flatten() + .ok_or(CandidateRejection::Malformed)?; + let signing_key = rustls::crypto::ring::sign::any_supported_type(&key) + .map_err(|_| CandidateRejection::Malformed)?; + if !tls::cert_key_matches(candidate, signing_key.as_ref()) { + return Err(CandidateRejection::KeyMismatched); + } + + let san = single_dns_san(candidate.as_ref()).ok(); + let expected = expected_name.to_ascii_lowercase(); + if san.as_deref() != Some(expected.as_str()) { + return Err(CandidateRejection::SanMismatched { + expected, + found: san, + }); + } + let name = expected; + + match leaf { + // The client leaf keeps the identity the registrar already + // authenticates as. The name equality above proves the three + // labels are unchanged; this proves they are still the labels + // the rule admits, in the configured domain. + SurfaceLeaf::RegistrarClient => { + recognize_registrar_client_name(&name, domain) + .map_err(|source| CandidateRejection::Unrecognised { source })?; + } + SurfaceLeaf::EndpointServer => { + recognize_registrar_endpoint_name(&name, domain) + .map_err(|source| CandidateRejection::Unrecognised { source })?; + // The pinned caller's own rule, run against the replacement + // before it is published. A server leaf under an anchor the + // pin file does not name is one every correctly pinned + // caller would refuse, and publishing it would take the + // endpoint off the air with no local symptom. + let pins = endpoint_pin::load_anchor_pins(pin_file) + .map_err(|source| CandidateRejection::PinFileUnusable { source })?; + let verifier = RegistrarEndpointVerifier::new(pins.into_iter().collect(), &name) + .map_err(|source| CandidateRejection::PinFileUnusable { source })?; + verifier + .verify( + candidate, + chain.get(1..).unwrap_or_default(), + UnixTime::now(), + ) + .map_err(|source| CandidateRejection::Unpinned { + pin_file: pin_file.to_path_buf(), + source, + })?; + } + } + Ok(()) +} + +// --------------------------------------------------------------------- +// Private artifacts, and the restore snapshots +// --------------------------------------------------------------------- + +/// The private working directory one attempt creates and removes. +/// +/// Everything an attempt writes outside the live paths lives here: the +/// candidate certificate and key, and the snapshot of each live file the +/// publication may replace. The directory is `0700` and is removed on +/// every exit — issuance failure, validation refusal, publication +/// failure, rollback failure and TLS-swap alike — by +/// [`Artifacts::close`] on the ordinary paths and by `TempDir`'s own +/// `Drop` on an unwind. +struct Artifacts { + dir: tempfile::TempDir, +} + +impl Artifacts { + /// Creates the directory beside `neighbour`, which is a live path + /// this attempt will write, so the artifacts share its filesystem + /// and its access control. + fn create(neighbour: &Path) -> Result { + let parent = neighbour.parent().ok_or_else(|| { + anyhow::anyhow!( + "{} has no parent directory to stage renewal artifacts in", + neighbour.display() + ) + })?; + let dir = tempfile::Builder::new() + .prefix(ARTIFACT_DIR_PREFIX) + .tempdir_in(parent) + .with_context(|| { + format!( + "creating the private renewal working directory below {}", + parent.display() + ) + })?; + Ok(Self { dir }) + } + + /// Writes one artifact with the mode the finished file it stands for + /// requires, and returns its path. + /// + /// The mode is established as the file is created rather than + /// afterwards: a candidate key that is world-readable for the width + /// of one `set_permissions` is a key that leaked. + fn write(&self, name: &str, contents: &[u8], mode: u32) -> Result { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + + let path = self.dir.path().join(name); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(mode) + .open(&path) + .with_context(|| format!("creating the renewal artifact at {}", path.display()))?; + file.write_all(contents) + .with_context(|| format!("writing the renewal artifact at {}", path.display()))?; + Ok(path) + } + + /// Removes the directory and everything in it, reporting a failure + /// rather than swallowing it. + fn close(self) -> Result<()> { + self.dir + .close() + .context("removing the private renewal working directory") + } +} + +/// One live path's prior state, captured before the first live write. +/// +/// The bytes are held in a private artifact rather than in memory, the +/// mode and the ownership beside it, so a restore puts back exactly what +/// was there and not merely something with the same contents. An absent +/// path is captured as absent, and restoring one removes whatever the +/// publication put there. +pub(crate) struct Snapshot { + /// The live path this snapshot restores. + path: PathBuf, + /// The file that was there, or `None` when the path was absent. + prior: Option, +} + +/// What was at a live path, and how it was owned. +struct PriorFile { + /// The private copy of the bytes. + artifact: PathBuf, + /// The mode the file carried. + mode: u32, + /// The uid that owned it. + uid: u32, + /// The gid that owned it. + gid: u32, +} + +impl Snapshot { + /// Captures `path` into `artifacts` under the artifact basename + /// `name`. + /// + /// # Errors + /// + /// Returns an error when the file exists and cannot be read or + /// stat'ed, or when the artifact cannot be written. Reaching one of + /// those before the first live write is what keeps a publication + /// from starting without a way back. + fn capture(path: &Path, name: &str, artifacts: &Artifacts) -> Result { + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Ok(Self { + path: path.to_path_buf(), + prior: None, + }); + } + Err(err) => { + return Err(anyhow::Error::new(err).context(format!( + "reading {} to capture a rollback snapshot", + path.display() + ))); + } + }; + let metadata = std::fs::metadata(path).with_context(|| { + format!( + "reading the mode and ownership of {} to capture a rollback snapshot", + path.display() + ) + })?; + let mode = metadata.mode() & 0o7777; + // The snapshot is private whatever the file it copies is: it + // sits in a `0700` directory a rollback reads and nothing else, + // and a copy of a key at any wider mode is a second key file. + let artifact = artifacts.write(name, &bytes, KEY_FILE_MODE_DEFAULT)?; + Ok(Self { + path: path.to_path_buf(), + prior: Some(PriorFile { + artifact, + mode, + uid: metadata.uid(), + gid: metadata.gid(), + }), + }) + } + + /// The live path this snapshot restores. + pub(crate) fn path(&self) -> &Path { + &self.path + } +} + +// --------------------------------------------------------------------- +// The live writes, behind a seam a test can fail +// --------------------------------------------------------------------- + +/// One live write in flight. +type LiveWrite<'a> = Pin> + Send + 'a>>; + +/// The three live writes a publication performs. +/// +/// A trait rather than three direct calls so a test can fail exactly one +/// of them — the bundle write, the pair write, or the rollback — and +/// assert what the daemon does about it, without corrupting a +/// filesystem to get there. +pub(crate) trait LivePaths: Send + Sync { + /// Replaces the merged CA bundle atomically. + fn write_bundle<'a>(&'a self, path: &'a Path, contents: &'a str) -> LiveWrite<'a>; + + /// Writes the certificate and then the key, through the established + /// two-rename contract. + fn write_pair<'a>( + &'a self, + cert_path: &'a Path, + key_path: &'a Path, + cert_pem: &'a str, + key_pem: &'a str, + ) -> LiveWrite<'a>; + + /// Puts one captured snapshot back: its bytes, its mode and its + /// ownership, or its absence. + fn restore<'a>(&'a self, snapshot: &'a Snapshot) -> LiveWrite<'a>; +} + +/// The production writer: the crate's existing publication paths and +/// nothing else. +pub(crate) struct FilesystemPaths { + policy: CertGroupPolicy, +} + +impl FilesystemPaths { + /// The policy both surface leaves are issued under: root-owned + /// material with no cert group, exactly as start-time issuance + /// publishes them. + pub(crate) fn new() -> Self { + Self { + policy: CertGroupPolicy::none(), + } + } +} + +impl LivePaths for FilesystemPaths { + fn write_bundle<'a>(&'a self, path: &'a Path, contents: &'a str) -> LiveWrite<'a> { + Box::pin(async move { fs_util::write_ca_bundle(path, contents, self.policy).await }) + } + + fn write_pair<'a>( + &'a self, + cert_path: &'a Path, + key_path: &'a Path, + cert_pem: &'a str, + key_pem: &'a str, + ) -> LiveWrite<'a> { + Box::pin(async move { + fs_util::write_cert_and_key(cert_path, key_path, cert_pem, key_pem, self.policy).await + }) + } + + fn restore<'a>(&'a self, snapshot: &'a Snapshot) -> LiveWrite<'a> { + Box::pin(async move { + let Some(prior) = snapshot.prior.as_ref() else { + // The path was absent when the snapshot was taken, so + // restoring it means removing whatever the publication + // put there. An already-absent path is the state asked + // for and not a failure. + match tokio::fs::remove_file(&snapshot.path).await { + Ok(()) => return Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(err) => { + return Err(anyhow::Error::new(err).context(format!( + "removing {} to restore the absence a snapshot captured", + snapshot.path.display() + ))); + } + } + }; + let bytes = tokio::fs::read(&prior.artifact).await.with_context(|| { + format!( + "reading the rollback snapshot of {}", + snapshot.path.display() + ) + })?; + fs_util::atomic_write_fixed_owner( + Destination::bootroot_owned(&snapshot.path), + &bytes, + StagedMode::Policy(prior.mode), + FixedOwner::restored(prior.uid, prior.gid), + ) + .await + .with_context(|| format!("restoring {} from its snapshot", snapshot.path.display())) + }) + } +} + +// --------------------------------------------------------------------- +// The adapter +// --------------------------------------------------------------------- + +/// The cadence and the retry budget one adapter runs under. +/// +/// Every value comes from the rendered internal agent configuration's +/// sole profile. There is no second scheduler, no lead-time constant of +/// this module's own and no second retry policy. +#[derive(Debug, Clone)] +pub(crate) struct RenewalCadence { + /// How long between passes, before jitter. + pub(crate) check_interval: Duration, + /// How much either side of that interval a pass may fall. + pub(crate) check_jitter: Duration, + /// How far ahead of `notAfter` a leaf becomes due. + pub(crate) renew_before: Duration, + /// The issuance backoff, in seconds per retry. + pub(crate) retry_backoff: Vec, +} + +impl RenewalCadence { + /// Reads the cadence off the rendered internal agent config. + /// + /// # Errors + /// + /// Returns an error when that config is absent, unparseable or + /// fails the loader's invariants, or when it carries no profile. + pub(crate) fn from_internal_config(secrets_dir: &Path) -> Result { + let paths = InternalPaths::new(secrets_dir); + let internal = load_internal_config(&paths).with_context(|| { + format!( + "reading the registrar renewal cadence from the rendered internal agent config \ + at {}", + paths.agent_config().display() + ) + })?; + let profile = internal.profiles.first().ok_or_else(|| { + anyhow::anyhow!( + "the rendered internal agent config at {} carries no profile to take the \ + renewal cadence from", + paths.agent_config().display() + ) + })?; + Ok(Self { + check_interval: profile.daemon.check_interval, + check_jitter: profile.daemon.check_jitter, + renew_before: profile.daemon.renew_before, + retry_backoff: internal.retry.backoff_secs.clone(), + }) + } +} + +/// The daemon-owned adapter that keeps both registrar leaves valid. +/// +/// One per daemon invocation, and only where the endpoint is enabled. +pub(crate) struct RegistrarCertRenewal { + settings: Arc, + plan: SurfacePlan, + endpoint: Arc, + state: RegistrarCertRenewalState, + cadence: RenewalCadence, + insecure_mode: bool, + live: Box, +} + +impl RegistrarCertRenewal { + /// Resolves the plan, reads the cadence and initializes the + /// accessor from the certificates already on disk. + /// + /// Initialization is not an attempt: it reads each enabled leaf's + /// valid certificate, records its `notAfter` against + /// [`RenewalAttempt::NeverAttempted`] with no timestamp, and + /// contacts neither `OpenBao` nor the CA. A leaf whose certificate + /// cannot be read or parsed gets no entry and is left to the first + /// pass, which finds it due and issues a replacement — start-time + /// issuance has already ensured usable material, so reaching that is + /// a file that changed underneath the daemon. + /// + /// # Errors + /// + /// Returns an error when the deployment state file, the rendered + /// internal agent config or the configured material paths cannot be + /// resolved. + pub(crate) async fn prepare( + settings: Arc, + endpoint: Arc, + insecure_mode: bool, + ) -> Result { + let plan = resolve_surface_plan(&settings) + .context("resolving the registrar surface renewal plan")?; + let cadence = RenewalCadence::from_internal_config(&plan.secrets_dir)?; + Ok(Self::assemble(settings, plan, endpoint, cadence, insecure_mode).await) + } + + /// Initializes the accessor and assembles the adapter around an + /// already-resolved plan and cadence. + /// + /// Split from [`RegistrarCertRenewal::prepare`] so a test drives the + /// production initialization, pass and publication paths over a plan + /// it wrote under `tempfile::tempdir()`, without a deployment state + /// file to resolve them from. + async fn assemble( + settings: Arc, + plan: SurfacePlan, + endpoint: Arc, + cadence: RenewalCadence, + insecure_mode: bool, + ) -> Self { + let state = RegistrarCertRenewalState::default(); + for pair in &plan.pairs { + match observed_not_after(&pair.cert_path).await { + Ok(not_after) => state.initialize(pair.leaf, not_after), + Err(err) => warn!( + "Registrar renewal could not read {} at {} to record its lifetime ({err:#}); \ + the first pass will treat it as due.", + pair.leaf.label(), + pair.cert_path.display() + ), + } + } + Self { + settings, + plan, + endpoint, + state, + cadence, + insecure_mode, + live: Box::new(FilesystemPaths::new()), + } + } + + /// The assembly step alone, for a test that has its own plan. + #[cfg(test)] + pub(crate) async fn for_test( + settings: Arc, + plan: SurfacePlan, + endpoint: Arc, + cadence: RenewalCadence, + ) -> Self { + Self::assemble(settings, plan, endpoint, cadence, false).await + } + + /// Returns the accessor this adapter writes. + /// + /// Handed to the reporting sibling when it lands; until then only a + /// test reads it, for the same reason [`RegistrarCertRenewalState::leaf`] + /// records. + #[allow(dead_code)] + pub(crate) fn state(&self) -> RegistrarCertRenewalState { + self.state.clone() + } + + /// Replaces the live-write seam, so a test can fail one of the three + /// writes a publication performs. + #[cfg(test)] + pub(crate) fn with_live_paths(mut self, live: Box) -> Self { + self.live = live; + self + } + + /// Runs an immediate first pass, then one per jittered interval, + /// until the daemon's shutdown signal arrives. + /// + /// The same shape the per-profile loop has, over the same cadence + /// values, so there is one scheduling idiom in the daemon and not + /// two. A pass never ends the loop: an issuance that failed, a + /// candidate that was refused and a publication that was rolled back + /// are all recorded and retried on the next tick. + /// + /// # Errors + /// + /// Reserved for a failure that ends the adapter itself; there is + /// none today, so this returns `Ok` once shutdown is observed. + pub(crate) async fn run(self, mut shutdown: watch::Receiver) -> Result<()> { + info!( + "Registrar certificate renewal enabled. check_interval={:?}, renew_before={:?}, \ + check_jitter={:?}", + self.cadence.check_interval, self.cadence.renew_before, self.cadence.check_jitter + ); + let mut first_tick = true; + loop { + if *shutdown.borrow_and_update() { + break; + } + let delay = if first_tick { + first_tick = false; + Duration::from_secs(0) + } else { + utils::jittered_delay(self.cadence.check_interval, self.cadence.check_jitter) + }; + tokio::select! { + _ = shutdown.changed() => break, + () = tokio::time::sleep(delay) => self.run_pass().await, + } + } + info!("Shutdown signal received. Exiting registrar certificate renewal."); + Ok(()) + } + + /// Runs one pass over both leaves. + /// + /// Eligibility is decided per leaf and the two are independent: one + /// being due is never a reason to replace the other. The `OpenBao` + /// reads that supply the ACME inputs happen only once a leaf is + /// actually due, so a pass that finds nothing to do makes no + /// `OpenBao` and no CA request at all. + pub(crate) async fn run_pass(&self) { + let mut due = Vec::with_capacity(self.plan.pairs.len()); + for pair in &self.plan.pairs { + match daemon::should_renew_certificate( + &pair.cert_path, + &self.settings.trust, + self.cadence.renew_before, + ) + .await + { + Ok(true) => due.push(pair.clone()), + Ok(false) => debug!( + "Registrar renewal: {} at {} is still valid.", + pair.leaf.label(), + pair.cert_path.display() + ), + // Not an attempt, so it supplies the accessor with + // nothing: the certificate could not be read or parsed, + // which is a fact about this pass rather than about an + // issuance that was tried and failed. The daemon's own + // per-profile tick reports its eligibility failures the + // same way and keeps ticking. + Err(err) => error!( + "Registrar renewal could not decide whether {} at {} is due: {err:#}", + pair.leaf.label(), + pair.cert_path.display() + ), + } + } + if due.is_empty() { + return; + } + + // Read only now, and once for however many leaves are due. A + // failure here is an ordinary failed attempt for each of them — + // including the credential's own refusal inside a trust-rotation + // window, which the rotation's mandatory tail repairs and this + // tick neither orders nor works around. + let inputs = match read_acme_inputs( + &self.plan.secrets_dir, + &self.plan.openbao_url, + &self.plan.kv_mount, + ) + .await + { + Ok(inputs) => inputs, + Err(err) => { + let reason = format!("{err:#}"); + for pair in &due { + self.record_failure(pair.leaf, &reason); + } + return; + } + }; + + for pair in &due { + if let Err(err) = self.renew_leaf(pair, &inputs).await { + self.record_failure(pair.leaf, &format!("{err:#}")); + } + } + } + + /// Issues, validates, stages, publishes and exchanges one leaf. + /// + /// Every step that can fail happens before the first live write, and + /// the one step after the last live write cannot fail. + async fn renew_leaf(&self, pair: &SurfacePairPaths, inputs: &SurfaceAcmeInputs) -> Result<()> { + let material = self.issue_with_retry(pair, inputs).await?; + self.renew_leaf_with_material(pair, material).await + } + + /// Everything after the issuance: the private working directory, the + /// validation, the publication transaction, the cleanup and the + /// state write. + /// + /// Split at exactly the point the ACME exchange ends, so a test + /// drives the whole publication path over material it minted itself + /// without a CA to reach. + async fn renew_leaf_with_material( + &self, + pair: &SurfacePairPaths, + material: crate::acme::IssuedMaterial, + ) -> Result<()> { + let artifacts = Artifacts::create(&pair.key_path)?; + let outcome = self.publish_candidate(pair, &material, &artifacts).await; + // Removed on every exit, whatever the outcome was, and the + // removal's own failure is reported rather than hidden. + let cleanup = artifacts.close(); + let outcome = match (outcome, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), Ok(())) | (Ok(()), Err(err)) => Err(err), + (Err(err), Err(cleanup)) => Err(err.context(format!( + "and the private renewal artifacts could not be removed: {cleanup:#}" + ))), + }; + outcome?; + + let not_after = observed_not_after(&pair.cert_path).await?; + self.state + .record_success(pair.leaf, not_after, OffsetDateTime::now_utc()); + info!( + "Registrar renewal replaced {} at {}; the next handshake uses it.", + pair.leaf.label(), + pair.cert_path.display() + ); + Ok(()) + } + + /// Runs the off-live issuance under the internal config's retry + /// budget. + /// + /// The budget is the internal profile's own, not a second policy: + /// the backoff list is read off its rendered configuration and + /// driven through the same helper the per-profile loop drives. + async fn issue_with_retry( + &self, + pair: &SurfacePairPaths, + inputs: &SurfaceAcmeInputs, + ) -> Result { + let produced: Mutex> = Mutex::new(None); + utils::retry_with_backoff_and_sleep( + || async { + let material = issue_surface_pair_material( + &self.settings, + pair, + &self.plan.host, + inputs, + self.insecure_mode, + ) + .await?; + // The guard is taken and dropped inside this statement; + // nothing holds it across an `.await`. + *produced.lock().unwrap_or_else(PoisonError::into_inner) = Some(material); + Ok(()) + }, + |delay| tokio::time::sleep(delay), + |attempt, err| { + error!( + "Registrar renewal of {} failed (attempt {attempt}): {err:#}", + pair.leaf.label() + ); + }, + &self.cadence.retry_backoff, + ) + .await?; + produced + .into_inner() + .unwrap_or_else(PoisonError::into_inner) + .ok_or_else(|| { + anyhow::anyhow!( + "the registrar renewal of {} reported success without producing material", + pair.leaf.label() + ) + }) + } + + /// Validates the candidate, builds the whole replacement, and + /// publishes it as a recoverable transaction. + async fn publish_candidate( + &self, + pair: &SurfacePairPaths, + material: &crate::acme::IssuedMaterial, + artifacts: &Artifacts, + ) -> Result<()> { + let pin_file = self.pin_file(); + validate_candidate( + pair.leaf, + material.cert_pem.as_bytes(), + material.key_pem.as_bytes(), + &pair.name, + &self.settings.domain, + &pin_file, + )?; + + let bundle_path = self + .settings + .trust + .ca_bundle_path + .as_deref() + .ok_or_else(|| { + anyhow::anyhow!( + "trust.ca_bundle_path is unset, so the incoming client verifier has no anchor \ + set to be rebuilt from and no registrar leaf can be published" + ) + })?; + let staged_bundle = self.stage_bundle(bundle_path, &material.chain)?; + + // The candidate on disk, at the modes the finished files + // require, so the configuration is built from exactly the bytes + // that will be published. + let candidate_cert = artifacts.write( + CANDIDATE_CERT_FILE, + material.cert_pem.as_bytes(), + CERT_FILE_MODE, + )?; + let candidate_key = artifacts.write( + CANDIDATE_KEY_FILE, + material.key_pem.as_bytes(), + KEY_FILE_MODE_DEFAULT, + )?; + + // The whole replacement, before any live path changes. When the + // client leaf is the one being renewed the server pair is the + // live one, unchanged — but the verifier is still rebuilt, + // because the staged bundle is what decides who may connect. + let (server_cert, server_key) = match pair.leaf { + SurfaceLeaf::EndpointServer => (candidate_cert.clone(), candidate_key.clone()), + SurfaceLeaf::RegistrarClient => self.live_server_pair()?, + }; + let (next_config, next_resolver) = build_server_config_from_staged( + &server_cert, + &server_key, + bundle_path, + staged_bundle.as_bytes(), + &self.settings.trust.trusted_ca_sha256, + &self.settings.domain, + ) + .with_context(|| { + format!( + "building the registrar endpoint's next TLS configuration for the renewal of {}", + pair.leaf.label() + ) + })?; + + // Only now, with nothing left that can fail before a write. + let bundle_snapshot = Snapshot::capture(bundle_path, "bundle.snapshot", artifacts)?; + let cert_snapshot = Snapshot::capture(&pair.cert_path, "cert.snapshot", artifacts)?; + let key_snapshot = Snapshot::capture(&pair.key_path, "key.snapshot", artifacts)?; + + if let Err(err) = self.live.write_bundle(bundle_path, &staged_bundle).await { + return Err(self.roll_back(err, &[&bundle_snapshot]).await); + } + if let Err(err) = self + .live + .write_pair( + &pair.cert_path, + &pair.key_path, + &material.cert_pem, + &material.key_pem, + ) + .await + { + return Err(self + .roll_back(err, &[&bundle_snapshot, &cert_snapshot, &key_snapshot]) + .await); + } + + // Infallible, and last. The replacement was built above, so + // there is nothing here that can leave the endpoint holding new + // files behind an old configuration. + self.endpoint.swap_active_tls(next_config, next_resolver); + Ok(()) + } + + /// Restores every live path whose publication may have started, and + /// reports what actually happened. + /// + /// A rollback that succeeds says so. A rollback that fails does not + /// claim the files were restored: both errors travel out together, + /// the active configuration stays as it was, and a later tick + /// retries. + async fn roll_back( + &self, + publication: anyhow::Error, + snapshots: &[&Snapshot], + ) -> anyhow::Error { + let mut failures = Vec::new(); + for snapshot in snapshots { + if let Err(err) = self.live.restore(snapshot).await { + failures.push(format!("{}: {err:#}", snapshot.path().display())); + } + } + if failures.is_empty() { + return publication.context( + "the registrar renewal publication failed and every live path it may have \ + written was restored from its snapshot", + ); + } + publication.context(format!( + "the registrar renewal publication failed AND the rollback did not restore {}; the \ + live material is in a mixed state and the endpoint is still serving its previous \ + configuration", + failures.join("; ") + )) + } + + /// Computes the bytes the merged bundle would hold, without writing + /// them. + /// + /// The candidate's chain is held to `trust.trusted_ca_sha256` first, + /// so a chain carrying an unpinned issuer is refused before anything + /// is staged, and the merge keeps only the existing bundle's pinned + /// certificates — the same rule the ordinary publication applies, + /// reached through the same two helpers rather than restated. + fn stage_bundle(&self, bundle_path: &Path, chain: &[Vec]) -> Result { + if chain.is_empty() { + anyhow::bail!( + "the issued candidate carried no issuer chain, so {} could not be staged and \ + the incoming client verifier could not be rebuilt", + bundle_path.display() + ); + } + crate::acme::verify_chain_fingerprints(chain, &self.settings.trust.trusted_ca_sha256)?; + let existing = match std::fs::read(bundle_path) { + Ok(bytes) => Some(bytes), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, + Err(err) => { + return Err(anyhow::Error::new(err).context(format!( + "refusing to replace the unreadable CA bundle at {}", + bundle_path.display() + ))); + } + }; + Ok(crate::acme::merge_ca_bundle( + existing.as_deref(), + chain, + &self.settings.trust.trusted_ca_sha256, + )) + } + + /// The live server pair, for a pass renewing only the client leaf. + fn live_server_pair(&self) -> Result<(PathBuf, PathBuf)> { + self.plan + .pairs + .iter() + .find(|pair| pair.leaf == SurfaceLeaf::EndpointServer) + .map(|pair| (pair.cert_path.clone(), pair.key_path.clone())) + .ok_or_else(|| { + anyhow::anyhow!("the registrar surface plan carries no endpoint server pair") + }) + } + + /// The endpoint pin file, beside the configured client certificate. + /// + /// Read to judge a server candidate and never written. + fn pin_file(&self) -> PathBuf { + let client_cert = self + .plan + .pairs + .iter() + .find(|pair| pair.leaf == SurfaceLeaf::RegistrarClient) + .map_or_else(PathBuf::new, |pair| pair.cert_path.clone()); + endpoint_pin::anchor_pin_path_for_client_certificate(&client_cert) + } + + /// Records a failed attempt and says so in the log. + fn record_failure(&self, leaf: SurfaceLeaf, reason: &str) { + error!("Registrar renewal of {} failed: {reason}", leaf.label()); + self.state + .record_failure(leaf, reason, OffsetDateTime::now_utc()); + } +} + +/// Reads the `notAfter` of the certificate at `path`. +/// +/// # Errors +/// +/// Returns an error when the file cannot be read or does not parse as a +/// PEM certificate. +async fn observed_not_after(path: &Path) -> Result { + let bytes = tokio::fs::read(path) + .await + .with_context(|| format!("reading {} to observe its lifetime", path.display()))?; + daemon::parse_cert_not_after(&bytes) + .with_context(|| format!("parsing the certificate at {}", path.display())) +} + +#[cfg(test)] +mod tests; diff --git a/src/registrar_renewal/tests.rs b/src/registrar_renewal/tests.rs new file mode 100644 index 00000000..3837e293 --- /dev/null +++ b/src/registrar_renewal/tests.rs @@ -0,0 +1,1351 @@ +//! Tests for the registrar surface's renewal adapter. +//! +//! Everything here runs under `tempfile::tempdir()` against a listener +//! this harness binds on a temporary path. Nothing reaches the network, +//! nothing mutates the process environment, and no test waits on a wall +//! clock: a pass is driven by calling it, and the loop's cadence is +//! driven under `tokio::time::pause`. +//! +//! The one step no test here drives is the ACME exchange itself. That is +//! [`crate::registrar_certs`]'s, whose own tests run +//! `issue_surface_pair_material` end to end against a mock directory; +//! what this module drives is everything that happens to the material it +//! returns. + +use std::os::unix::fs::PermissionsExt as _; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use rcgen::{ + BasicConstraints, CertificateParams, CertifiedIssuer, DnType, IsCa, KeyPair, KeyUsagePurpose, + SanType, +}; +use tempfile::TempDir; + +use super::*; +use crate::acme::IssuedMaterial; +use crate::config::{RegistrarEndpointSettings, Settings}; +use crate::registrar::endpoint; +use crate::registrar::endpoint::activation::ActivationContract; +use crate::registrar::endpoint_pin::REGISTRAR_ENDPOINT_ANCHORS_FILE; + +const TEST_DOMAIN: &str = "corp.example.internal"; +const TEST_HOST: &str = "bootroot-01"; + +// --------------------------------------------------------------------- +// Certificate material +// --------------------------------------------------------------------- + +type Issuer = CertifiedIssuer<'static, KeyPair>; + +/// A one-level test CA: a self-signed root that signs leaves directly, +/// exactly the shape the deployment's own anchor set has. +struct TestCa { + issuer: Issuer, + pem: String, +} + +impl TestCa { + fn new(common_name: &str) -> Self { + let key = KeyPair::generate().expect("ca key"); + let mut params = CertificateParams::new(Vec::::new()).expect("ca params"); + params + .distinguished_name + .push(DnType::CommonName, common_name); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]; + let issuer = CertifiedIssuer::self_signed(params, key).expect("self-signed CA"); + let pem = issuer.pem(); + Self { issuer, pem } + } + + fn fingerprint(&self) -> String { + crate::tls::sha256_hex(self.issuer.der()) + } + + fn der(&self) -> Vec { + self.issuer.der().to_vec() + } + + /// Issues a leaf carrying `name` as its only DNS SAN, valid from + /// `not_before_days` to `not_after_days` measured from now. + fn issue(&self, name: &str, not_before_days: i64, not_after_days: i64) -> (String, String) { + let key = KeyPair::generate().expect("leaf key"); + let mut params = CertificateParams::new(Vec::::new()).expect("leaf params"); + params.distinguished_name.push(DnType::CommonName, name); + params.is_ca = IsCa::NoCa; + params.subject_alt_names = vec![SanType::DnsName( + name.to_string().try_into().expect("a valid DNS SAN"), + )]; + let now = time::OffsetDateTime::now_utc(); + params.not_before = now + time::Duration::days(not_before_days); + params.not_after = now + time::Duration::days(not_after_days); + let leaf = params.signed_by(&key, &self.issuer).expect("issued leaf"); + (leaf.pem(), key.serialize_pem()) + } + + /// The material an issuance would have returned for `name`: the leaf + /// followed by this CA's certificate, a fresh key, and the chain. + fn material(&self, name: &str, not_before_days: i64, not_after_days: i64) -> IssuedMaterial { + let (leaf_pem, key_pem) = self.issue(name, not_before_days, not_after_days); + IssuedMaterial { + cert_pem: format!("{leaf_pem}{}", self.pem), + key_pem, + chain: vec![self.der()], + } + } +} + +fn pem_to_der(pem: &str) -> Vec { + let (_, parsed) = x509_parser::pem::parse_x509_pem(pem.as_bytes()).expect("a PEM certificate"); + parsed.contents +} + +fn san_at(path: &Path) -> String { + let pem = std::fs::read_to_string(path).expect("read the published leaf"); + single_dns_san(&pem_to_der(&pem)).expect("the published leaf carries one DNS SAN") +} + +fn digest_of(path: &Path) -> String { + crate::tls::sha256_hex(&std::fs::read(path).expect("read the file to digest")) +} + +// --------------------------------------------------------------------- +// The host fixture +// --------------------------------------------------------------------- + +/// A provisioned endpoint-enabled host, an activated endpoint over a +/// listener the harness bound, and the renewal adapter above both. +struct Harness { + dir: TempDir, + _socket_dir: TempDir, + ca: TestCa, + settings: Arc, + plan: SurfacePlan, + endpoint: Arc, + socket_path: PathBuf, +} + +impl Harness { + fn build() -> Self { + Self::build_with_pins(None) + } + + /// `pin_file_over` replaces what the endpoint pin file names; `None` + /// pins the deployment CA, which is what a provisioned host has. + fn build_with_pins(pin_file_over: Option>) -> Self { + crate::tls::install_crypto_provider(); + let dir = TempDir::new().expect("tempdir"); + let ca = TestCa::new("Bootroot Renewal Test CA"); + let certs = dir.path().join("certs"); + std::fs::create_dir_all(&certs).expect("create the certs directory"); + + let bundle_path = certs.join("ca-bundle.pem"); + std::fs::write(&bundle_path, &ca.pem).expect("write the bundle"); + + let pins = pin_file_over.unwrap_or_else(|| vec![ca.fingerprint()]); + let pin_file = certs.join(REGISTRAR_ENDPOINT_ANCHORS_FILE); + std::fs::write( + &pin_file, + pins.iter().fold(String::new(), |mut acc, pin| { + use std::fmt::Write as _; + let _ = writeln!(acc, "{pin}"); + acc + }), + ) + .expect("write the pin file"); + + let mut settings = base_settings(); + settings.trust.ca_bundle_path = Some(bundle_path); + settings.trust.trusted_ca_sha256 = vec![ca.fingerprint()]; + settings.registrar_endpoint = RegistrarEndpointSettings { + enabled: true, + server_cert_path: Some(certs.join("endpoint.crt")), + server_key_path: Some(certs.join("endpoint.key")), + client_cert_path: Some(certs.join("client.crt")), + client_key_path: Some(certs.join("client.key")), + }; + + // The deployment's active root, so a pass that needs the ACME + // inputs gets as far as loading the bootroot-internal + // credential rather than stopping one step above it. + let secrets_dir = dir.path().join("secrets"); + let root_path = crate::registrar::internal::active_root_cert_path(&secrets_dir); + std::fs::create_dir_all(root_path.parent().expect("a certs directory")) + .expect("create the secrets certs directory"); + std::fs::write(&root_path, &ca.pem).expect("write the active root"); + + let plan = SurfacePlan { + secrets_dir, + openbao_url: "https://127.0.0.1:1".to_string(), + kv_mount: "bootroot-kv".to_string(), + host: TEST_HOST.to_string(), + pairs: crate::registrar_certs::surface_pairs( + &settings.registrar_endpoint, + TEST_HOST, + TEST_DOMAIN, + ) + .expect("both pairs resolve"), + }; + + // Live material at both configured pairs, in date. + for pair in &plan.pairs { + let (leaf, key) = ca.issue(&pair.name, -1, 30); + std::fs::write(&pair.cert_path, format!("{leaf}{}", ca.pem)) + .expect("write the live leaf"); + std::fs::write(&pair.key_path, key).expect("write the live key"); + std::fs::set_permissions(&pair.key_path, std::fs::Permissions::from_mode(0o600)) + .expect("narrow the live key"); + } + + let (socket_dir, socket_path, endpoint) = activate_over(&settings); + Self { + dir, + _socket_dir: socket_dir, + ca, + settings: Arc::new(settings), + plan, + endpoint, + socket_path, + } + } + + fn pair(&self, leaf: SurfaceLeaf) -> SurfacePairPaths { + self.plan + .pairs + .iter() + .find(|pair| pair.leaf == leaf) + .expect("both leaves are in the plan") + .clone() + } + + fn bundle_path(&self) -> PathBuf { + self.settings + .trust + .ca_bundle_path + .clone() + .expect("a configured bundle") + } + + fn pin_file(&self) -> PathBuf { + self.dir + .path() + .join("certs") + .join(REGISTRAR_ENDPOINT_ANCHORS_FILE) + } + + async fn renewal(&self) -> RegistrarCertRenewal { + RegistrarCertRenewal::for_test( + Arc::clone(&self.settings), + self.plan.clone(), + Arc::clone(&self.endpoint), + cadence(), + ) + .await + } +} + +/// Binds a listener on a private path and adopts it through the +/// production activation path, over `settings`' own TLS material. +fn activate_over(settings: &Settings) -> (TempDir, PathBuf, Arc) { + use std::os::fd::IntoRawFd as _; + + let socket_dir = TempDir::new().expect("socket tempdir"); + std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700)) + .expect("narrow the socket directory"); + let socket_path = socket_dir.path().join("registrar.sock"); + let listener = std::os::unix::net::UnixListener::bind(&socket_path).expect("bind"); + std::fs::set_permissions( + &socket_path, + std::fs::Permissions::from_mode(endpoint::REQUIRED_SOCKET_MODE), + ) + .expect("set the socket mode"); + let (config, resolver) = endpoint::tls::build_server_config( + settings.registrar_endpoint.server_cert_path.as_deref(), + settings.registrar_endpoint.server_key_path.as_deref(), + settings.trust.ca_bundle_path.as_deref(), + &settings.trust.trusted_ca_sha256, + &settings.domain, + ) + .expect("the fixture's material builds a configuration"); + let adopted = endpoint::adopt( + ActivationContract::from_test_descriptor(listener.into_raw_fd()), + endpoint::current_effective_uid(), + config, + resolver, + settings.domain.clone(), + ) + .expect("adopt the harness listener"); + (socket_dir, socket_path, adopted) +} + +fn base_settings() -> Settings { + Settings { + email: "ops@example.internal".to_string(), + server: "https://127.0.0.1:1/acme/acme/directory".to_string(), + domain: TEST_DOMAIN.to_string(), + eab: None, + acme: crate::config::AcmeSettings { + directory_fetch_attempts: 1, + directory_fetch_base_delay_secs: 1, + directory_fetch_max_delay_secs: 1, + poll_attempts: 1, + poll_interval_secs: 0, + account_key_path: None, + http_responder_url: "http://127.0.0.1:1".to_string(), + http_responder_hmac: "unused".into(), + http_responder_timeout_secs: 1, + http_responder_token_ttl_secs: 300, + }, + retry: crate::config::RetrySettings { + backoff_secs: Vec::new(), + }, + trust: crate::config::TrustSettings::default(), + scheduler: crate::config::SchedulerSettings { + max_concurrent_issuances: 1, + }, + profiles: Vec::new(), + openbao: None, + registrar_endpoint: RegistrarEndpointSettings::default(), + registrar: crate::config::RegistrarSettings::default(), + } +} + +/// A cadence a test can drive without waiting on anything real. +fn cadence() -> RenewalCadence { + RenewalCadence { + check_interval: Duration::from_hours(1), + check_jitter: Duration::from_secs(0), + renew_before: Duration::from_hours(16), + retry_backoff: Vec::new(), + } +} + +// --------------------------------------------------------------------- +// The injectable live-write seam +// --------------------------------------------------------------------- + +/// The production writer with any of its three writes forced to fail. +/// +/// Composing the three independently is what lets a test drive the +/// publication failure and the rollback failure of the *same* attempt, +/// which is the case where the two errors have to travel out together. +struct FaultyPaths { + inner: FilesystemPaths, + fail_bundle: bool, + fail_pair: bool, + fail_rollback: bool, + restores: Arc, +} + +impl FaultyPaths { + /// Not `Self`: the seam is a `Box`, and the counter + /// beside it is what a test asserts the rollback through. + #[allow(clippy::new_ret_no_self)] + fn new( + fail_bundle: bool, + fail_pair: bool, + fail_rollback: bool, + ) -> (Box, Arc) { + let restores = Arc::new(AtomicUsize::new(0)); + ( + Box::new(Self { + inner: FilesystemPaths::new(), + fail_bundle, + fail_pair, + fail_rollback, + restores: Arc::clone(&restores), + }), + restores, + ) + } +} + +impl LivePaths for FaultyPaths { + fn write_bundle<'a>(&'a self, path: &'a Path, contents: &'a str) -> LiveWrite<'a> { + if self.fail_bundle { + return Box::pin(async { anyhow::bail!("injected CA bundle write failure") }); + } + self.inner.write_bundle(path, contents) + } + + fn write_pair<'a>( + &'a self, + cert_path: &'a Path, + key_path: &'a Path, + cert_pem: &'a str, + key_pem: &'a str, + ) -> LiveWrite<'a> { + if self.fail_pair { + return Box::pin(async { anyhow::bail!("injected certificate and key write failure") }); + } + self.inner + .write_pair(cert_path, key_path, cert_pem, key_pem) + } + + fn restore<'a>(&'a self, snapshot: &'a Snapshot) -> LiveWrite<'a> { + self.restores.fetch_add(1, Ordering::SeqCst); + if self.fail_rollback { + return Box::pin(async { anyhow::bail!("injected rollback failure") }); + } + self.inner.restore(snapshot) + } +} + +/// Every live and active fact a refusal must leave untouched. +struct LiveFacts { + bundle: String, + cert: String, + key: String, + pin_file: String, + acceptor: Arc, + not_after: OffsetDateTime, +} + +impl LiveFacts { + fn capture(harness: &Harness, leaf: SurfaceLeaf, state: &RegistrarCertRenewalState) -> Self { + let pair = harness.pair(leaf); + Self { + bundle: digest_of(&harness.bundle_path()), + cert: digest_of(&pair.cert_path), + key: digest_of(&pair.key_path), + pin_file: digest_of(&harness.pin_file()), + acceptor: harness.endpoint.active_tls(), + not_after: state.leaf(leaf).expect("the leaf has an entry").not_after, + } + } + + fn assert_unchanged(&self, harness: &Harness, leaf: SurfaceLeaf) { + let pair = harness.pair(leaf); + assert_eq!( + self.bundle, + digest_of(&harness.bundle_path()), + "the live CA bundle must be byte-for-byte unchanged" + ); + assert_eq!( + self.cert, + digest_of(&pair.cert_path), + "the live certificate must be byte-for-byte unchanged" + ); + assert_eq!( + self.key, + digest_of(&pair.key_path), + "the live key must be byte-for-byte unchanged" + ); + assert_eq!( + self.pin_file, + digest_of(&harness.pin_file()), + "the endpoint pin file must never be rewritten" + ); + assert!( + Arc::ptr_eq(&self.acceptor, &harness.endpoint.active_tls()), + "the active TLS configuration must not have been exchanged" + ); + } +} + +// --------------------------------------------------------------------- +// The accessor and initialization +// --------------------------------------------------------------------- + +/// An enabled endpoint gets one entry per leaf, initialized from the +/// certificate already on disk, with no attempt behind it and no +/// timestamp. +#[tokio::test] +async fn initialization_records_both_leaves_without_attempting_anything() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let state = renewal.state(); + + assert_eq!(state.len(), 2, "an enabled endpoint has both entries"); + for leaf in [SurfaceLeaf::EndpointServer, SurfaceLeaf::RegistrarClient] { + let entry = state.leaf(leaf).expect("an entry for each leaf"); + assert_eq!(entry.attempt, RenewalAttempt::NeverAttempted); + assert!( + entry.attempted_at.is_none(), + "initialization is not an attempt and stamps no time" + ); + let pair = harness.pair(leaf); + let observed = observed_not_after(&pair.cert_path) + .await + .expect("the fixture leaf parses"); + assert_eq!( + entry.not_after, observed, + "the recorded lifetime is the one on disk" + ); + } +} + +/// The accessor is empty until something initializes it, which is what a +/// disabled endpoint leaves behind: no adapter is built, so no entry +/// exists and nothing is ever asked of `OpenBao` or the CA. +#[test] +fn a_state_accessor_nobody_initialized_holds_nothing() { + let state = RegistrarCertRenewalState::default(); + assert_eq!(state.len(), 0); + assert!(state.leaf(SurfaceLeaf::EndpointServer).is_none()); + assert!(state.leaf(SurfaceLeaf::RegistrarClient).is_none()); +} + +/// With the endpoint disabled there is no activated endpoint for an +/// adapter to be built over, so no adapter exists, no state entry is +/// made, no pass runs and nothing is asked of `OpenBao` or the CA. +/// +/// The adapter is spawned inside the same branch the accept task is, +/// which is reached only with an activated endpoint in hand; a disabled +/// endpoint activates to nothing without reading a certificate, a +/// descriptor or an activation variable. +#[test] +fn a_disabled_endpoint_leaves_no_adapter_to_build() { + let mut settings = base_settings(); + settings.registrar_endpoint = RegistrarEndpointSettings { + enabled: false, + server_cert_path: Some(PathBuf::from("/nonexistent/endpoint.crt")), + server_key_path: Some(PathBuf::from("/nonexistent/endpoint.key")), + client_cert_path: Some(PathBuf::from("/nonexistent/client.crt")), + client_key_path: Some(PathBuf::from("/nonexistent/client.key")), + }; + assert!( + endpoint::activate(&settings) + .expect("a disabled endpoint never fails") + .is_none(), + "a disabled endpoint yields nothing for a renewal adapter to own" + ); +} + +/// A success replaces `notAfter`; a failure retains it and records the +/// reason; neither loses the entry. +#[test] +fn a_failure_retains_the_lifetime_a_success_replaces() { + let state = RegistrarCertRenewalState::default(); + let first = OffsetDateTime::now_utc(); + let later = first + time::Duration::days(30); + let attempted = first + time::Duration::minutes(1); + state.initialize(SurfaceLeaf::RegistrarClient, first); + + state.record_success(SurfaceLeaf::RegistrarClient, later, attempted); + let entry = state.leaf(SurfaceLeaf::RegistrarClient).expect("an entry"); + assert_eq!(entry.not_after, later); + assert_eq!(entry.attempt, RenewalAttempt::Succeeded); + assert_eq!(entry.attempted_at, Some(attempted)); + + let failed_at = attempted + time::Duration::hours(1); + state.record_failure(SurfaceLeaf::RegistrarClient, "the CA said no", failed_at); + let entry = state.leaf(SurfaceLeaf::RegistrarClient).expect("an entry"); + assert_eq!( + entry.not_after, later, + "a failed attempt retains the lifetime the leaf still has" + ); + assert_eq!( + entry.attempt, + RenewalAttempt::Failed { + reason: "the CA said no".to_string() + } + ); + assert_eq!(entry.attempted_at, Some(failed_at)); +} + +// --------------------------------------------------------------------- +// Eligibility +// --------------------------------------------------------------------- + +/// A leaf outside lead time is not due; one inside it is; and the lead +/// time is checked before the chain, so a leaf that is both is reported +/// once. +#[tokio::test] +async fn lead_time_decides_before_chain_drift_does() { + let harness = Harness::build(); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + assert!( + !daemon::should_renew_certificate( + &pair.cert_path, + &harness.settings.trust, + Duration::from_secs(3600) + ) + .await + .expect("the fixture leaf parses"), + "a leaf 30 days out is not due at a one-hour lead time" + ); + assert!( + daemon::should_renew_certificate( + &pair.cert_path, + &harness.settings.trust, + Duration::from_hours(24 * 60) + ) + .await + .expect("the fixture leaf parses"), + "a leaf inside lead time is due" + ); +} + +/// An in-date leaf outside lead time whose chain no longer reaches the +/// configured bundle is due; with no bundle configured, the same leaf is +/// not — the operator opted out of bundle management and nothing is +/// reissued for want of an anchor set. +#[tokio::test] +async fn chain_drift_is_eligibility_only_where_a_bundle_is_configured() { + let harness = Harness::build(); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + // A destructive rotation: the bundle is replaced with a generation + // that never signed this leaf, which is still in date and still + // correctly named. + let rotated = TestCa::new("Bootroot Rotated CA"); + std::fs::write(harness.bundle_path(), &rotated.pem).expect("rewrite the bundle"); + + let short_lead = Duration::from_secs(3600); + assert!( + daemon::should_renew_certificate(&pair.cert_path, &harness.settings.trust, short_lead) + .await + .expect("the leaf parses"), + "a leaf that no longer chains to the configured bundle is due" + ); + + let opted_out = crate::config::TrustSettings::default(); + assert!( + opted_out.ca_bundle_path.is_none(), + "the opt-out is an unconfigured bundle path" + ); + assert!( + !daemon::should_renew_certificate(&pair.cert_path, &opted_out, short_lead) + .await + .expect("the leaf parses"), + "with no bundle configured, chain drift does not trigger renewal" + ); +} + +/// That opt-out cannot stand an endpoint up: `build_server_config` +/// requires both the bundle path and a non-empty pin list, and falls +/// back to no other root source for incoming mTLS. +#[tokio::test] +async fn the_no_bundle_opt_out_cannot_activate_an_enabled_endpoint() { + let harness = Harness::build(); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + + let missing_bundle = endpoint::tls::build_server_config( + Some(&pair.cert_path), + Some(&pair.key_path), + None, + &harness.settings.trust.trusted_ca_sha256, + TEST_DOMAIN, + ) + .expect_err("an enabled endpoint with no bundle must refuse"); + assert!( + matches!( + missing_bundle, + endpoint::tls::EndpointTlsError::MissingSetting { + setting: endpoint::tls::CA_BUNDLE_SETTING + } + ), + "{missing_bundle:#}" + ); + + let missing_pins = endpoint::tls::build_server_config( + Some(&pair.cert_path), + Some(&pair.key_path), + harness.settings.trust.ca_bundle_path.as_deref(), + &[], + TEST_DOMAIN, + ) + .expect_err("an enabled endpoint with no pins must refuse"); + assert!( + matches!( + missing_pins, + endpoint::tls::EndpointTlsError::MissingSetting { + setting: endpoint::tls::TRUSTED_CA_SETTING + } + ), + "{missing_pins:#}" + ); +} + +// --------------------------------------------------------------------- +// Candidate validation +// --------------------------------------------------------------------- + +/// A conforming candidate for each leaf passes every rule. +#[tokio::test] +async fn a_conforming_candidate_is_accepted_for_both_leaves() { + let harness = Harness::build(); + for leaf in [SurfaceLeaf::EndpointServer, SurfaceLeaf::RegistrarClient] { + let pair = harness.pair(leaf); + let material = harness.ca.material(&pair.name, -1, 30); + validate_candidate( + leaf, + material.cert_pem.as_bytes(), + material.key_pem.as_bytes(), + &pair.name, + TEST_DOMAIN, + &harness.pin_file(), + ) + .expect("a conforming candidate is publishable"); + } +} + +/// A key that is not the candidate leaf's is refused before anything is +/// staged, and so is a candidate whose SAN drifted off the reserved name +/// the pair runs under. +#[tokio::test] +async fn a_mismatched_key_or_a_drifted_name_is_refused() { + let harness = Harness::build(); + let pair = harness.pair(SurfaceLeaf::RegistrarClient); + let material = harness.ca.material(&pair.name, -1, 30); + let other = harness.ca.material(&pair.name, -1, 30); + + let rejection = validate_candidate( + SurfaceLeaf::RegistrarClient, + material.cert_pem.as_bytes(), + other.key_pem.as_bytes(), + &pair.name, + TEST_DOMAIN, + &harness.pin_file(), + ) + .expect_err("a foreign key must be refused"); + assert!( + matches!(rejection, CandidateRejection::KeyMismatched), + "{rejection:#}" + ); + + // The instance label moved, which is exactly what "the same + // identity" forbids. + let drifted_name = format!("002.bootroot-registrar.{TEST_HOST}.{TEST_DOMAIN}"); + let drifted = harness.ca.material(&drifted_name, -1, 30); + let rejection = validate_candidate( + SurfaceLeaf::RegistrarClient, + drifted.cert_pem.as_bytes(), + drifted.key_pem.as_bytes(), + &pair.name, + TEST_DOMAIN, + &harness.pin_file(), + ) + .expect_err("a drifted identity must be refused"); + assert!( + matches!( + rejection, + CandidateRejection::SanMismatched { ref found, .. } + if found.as_deref() == Some(drifted_name.as_str()) + ), + "{rejection:#}" + ); +} + +/// A server candidate under an anchor the pin file does not name is +/// refused, naming the pin file it was held against. +#[tokio::test] +async fn a_server_candidate_under_an_unpinned_anchor_is_refused() { + let harness = Harness::build(); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let foreign = TestCa::new("Bootroot Unpinned CA"); + let material = foreign.material(&pair.name, -1, 30); + + let rejection = validate_candidate( + SurfaceLeaf::EndpointServer, + material.cert_pem.as_bytes(), + material.key_pem.as_bytes(), + &pair.name, + TEST_DOMAIN, + &harness.pin_file(), + ) + .expect_err("an unpinned server candidate must be refused"); + assert!( + matches!(rejection, CandidateRejection::Unpinned { ref pin_file, .. } + if pin_file == &harness.pin_file()), + "{rejection:#}" + ); + + // The same material is fine for the *client* leaf's rules, which do + // not consult the pin file at all: the pin file decides only whether + // a replacement server leaf is safe for pinned callers. + let client = harness.pair(SurfaceLeaf::RegistrarClient); + let client_material = foreign.material(&client.name, -1, 30); + validate_candidate( + SurfaceLeaf::RegistrarClient, + client_material.cert_pem.as_bytes(), + client_material.key_pem.as_bytes(), + &client.name, + TEST_DOMAIN, + &harness.pin_file(), + ) + .expect("the pin file is not a client-candidate rule"); +} + +// --------------------------------------------------------------------- +// The staged bundle and the rebuilt verifier +// --------------------------------------------------------------------- + +/// The staged bundle is the existing bundle's pinned certificates plus +/// the candidate's chain, and an unpinned chain is refused before +/// anything is staged. +#[tokio::test] +async fn the_staged_bundle_merges_only_pinned_material() { + let mut harness = Harness::build(); + let rotated = TestCa::new("Bootroot Next Generation CA"); + + // Both generations pinned, which is the additive transitional trust + // a rotation publishes. + let mut settings = (*harness.settings).clone(); + settings.trust.trusted_ca_sha256 = vec![harness.ca.fingerprint(), rotated.fingerprint()]; + harness.settings = Arc::new(settings); + let renewal = harness.renewal().await; + + let staged = renewal + .stage_bundle(&harness.bundle_path(), &[rotated.der()]) + .expect("a pinned chain stages"); + let fingerprints: Vec = crate::tls::parse_pem_to_cert_list(staged.as_bytes()) + .expect("the staged bundle parses") + .into_iter() + .map(|cert| crate::tls::sha256_hex(cert.as_ref())) + .collect(); + assert!( + fingerprints.contains(&harness.ca.fingerprint()), + "the existing pinned anchor survives the merge" + ); + assert!( + fingerprints.contains(&rotated.fingerprint()), + "the candidate's chain is merged in" + ); + + // Nothing was written: staging is a computation. + assert_eq!( + std::fs::read_to_string(harness.bundle_path()).expect("read the bundle"), + harness.ca.pem, + "staging must not touch the live bundle" + ); + + let unpinned = TestCa::new("Bootroot Unpinned CA"); + let refused = renewal + .stage_bundle(&harness.bundle_path(), &[unpinned.der()]) + .expect_err("an unpinned chain must be refused"); + assert!( + format!("{refused:#}").contains("Untrusted CA fingerprint"), + "{refused:#}" + ); +} + +/// The rebuilt incoming verifier's anchor set is exactly the staged +/// post-merge bundle's pinned certificates. A pin-file-only anchor, an +/// unpinned bundle certificate and the returned chain alone cannot +/// substitute, and neither can the system roots. +#[tokio::test] +async fn the_incoming_verifier_is_rebuilt_from_the_staged_pinned_subset_alone() { + let harness = Harness::build(); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let bundle_path = harness.bundle_path(); + let foreign = TestCa::new("Bootroot Foreign CA"); + + // The pin file names the foreign anchor, and the staged bundle does + // not carry it: the verifier is built from the bundle, so this + // builds and admits nobody the bundle does not. + std::fs::write( + harness.pin_file(), + format!("{}\n", crate::tls::sha256_hex(&foreign.der())), + ) + .expect("rewrite the pin file"); + endpoint::tls::build_server_config_from_staged( + &pair.cert_path, + &pair.key_path, + &bundle_path, + harness.ca.pem.as_bytes(), + &harness.settings.trust.trusted_ca_sha256, + TEST_DOMAIN, + ) + .expect("the pin file is not a verifier source, so it changes nothing here"); + + // A staged bundle holding only an unpinned certificate has no anchor + // at all: it is refused rather than widened to the whole file or to + // the system roots. + let error = endpoint::tls::build_server_config_from_staged( + &pair.cert_path, + &pair.key_path, + &bundle_path, + foreign.pem.as_bytes(), + &harness.settings.trust.trusted_ca_sha256, + TEST_DOMAIN, + ) + .expect_err("an unpinned staged bundle has no anchor"); + assert!( + matches!( + error, + endpoint::tls::EndpointTlsError::NoPinnedAnchor { .. } + ), + "{error:#}" + ); + + // A staged bundle with nothing in it is refused rather than falling + // back to the system roots, which is the only other anchor source + // a `WebPkiClientVerifier` could have been given. + let error = endpoint::tls::build_server_config_from_staged( + &pair.cert_path, + &pair.key_path, + &bundle_path, + b"", + &harness.settings.trust.trusted_ca_sha256, + TEST_DOMAIN, + ) + .expect_err("an empty staged bundle has no anchor"); + assert!( + matches!(error, endpoint::tls::EndpointTlsError::Unparsable { .. }), + "{error:#}" + ); +} + +// --------------------------------------------------------------------- +// Publication +// --------------------------------------------------------------------- + +/// A due server leaf is published in full: the merged bundle, then the +/// certificate and key, then the active configuration — and the pin file +/// is not touched. +#[tokio::test] +async fn publishing_a_server_candidate_replaces_the_material_and_the_active_configuration() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let before_key = digest_of(&pair.key_path); + let before_pins = digest_of(&harness.pin_file()); + let before_active = harness.endpoint.active_tls(); + let material = harness.ca.material(&pair.name, -1, 60); + + let artifacts = Artifacts::create(&pair.key_path).expect("artifacts"); + renewal + .publish_candidate(&pair, &material, &artifacts) + .await + .expect("a conforming candidate publishes"); + artifacts.close().expect("artifacts are removed"); + + assert_eq!( + san_at(&pair.cert_path), + pair.name, + "the published leaf keeps the exact endpoint SAN" + ); + assert_ne!( + before_key, + digest_of(&pair.key_path), + "every successful issuance publishes a fresh key" + ); + assert_eq!( + before_pins, + digest_of(&harness.pin_file()), + "the endpoint pin file is never rewritten and gains no leaf pin" + ); + assert!( + !Arc::ptr_eq(&before_active, &harness.endpoint.active_tls()), + "the active TLS configuration is exchanged after a successful publication" + ); + assert_eq!( + std::fs::metadata(&pair.key_path) + .expect("stat the key") + .mode() + & 0o777, + 0o600, + "the published key is never group or world readable" + ); +} + +/// Renewing only the client leaf still rebuilds the whole configuration +/// — the staged bundle decides who may connect — and leaves the live +/// server pair exactly as it was. +#[tokio::test] +async fn renewing_the_client_leaf_leaves_the_server_pair_and_still_swaps() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let client = harness.pair(SurfaceLeaf::RegistrarClient); + let server = harness.pair(SurfaceLeaf::EndpointServer); + let before_server_cert = digest_of(&server.cert_path); + let before_server_key = digest_of(&server.key_path); + let before_active = harness.endpoint.active_tls(); + let material = harness.ca.material(&client.name, -1, 60); + + let artifacts = Artifacts::create(&client.key_path).expect("artifacts"); + renewal + .publish_candidate(&client, &material, &artifacts) + .await + .expect("a conforming client candidate publishes"); + artifacts.close().expect("artifacts are removed"); + + assert_eq!( + san_at(&client.cert_path), + client.name, + "the client leaf keeps its recognised instance, host and domain" + ); + assert_eq!( + before_server_cert, + digest_of(&server.cert_path), + "renewing the client leaf leaves the server certificate alone" + ); + assert_eq!( + before_server_key, + digest_of(&server.key_path), + "renewing the client leaf leaves the server key alone" + ); + assert!( + !Arc::ptr_eq(&before_active, &harness.endpoint.active_tls()), + "the incoming verifier is rebuilt from the staged bundle either way" + ); +} + +/// An unpinned server candidate changes nothing at all, and the accessor +/// records the pin-file reason against the lifetime the leaf still has. +#[tokio::test] +async fn an_unpinned_server_candidate_changes_no_live_or_active_state() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let facts = LiveFacts::capture(&harness, pair.leaf, &renewal.state()); + + let foreign = TestCa::new("Bootroot Unpinned CA"); + let material = foreign.material(&pair.name, -1, 60); + let artifacts = Artifacts::create(&pair.key_path).expect("artifacts"); + let err = renewal + .publish_candidate(&pair, &material, &artifacts) + .await + .expect_err("an unpinned candidate is a refusal"); + artifacts.close().expect("artifacts are removed"); + + renewal.record_failure(pair.leaf, &format!("{err:#}")); + facts.assert_unchanged(&harness, pair.leaf); + let entry = renewal.state().leaf(pair.leaf).expect("an entry"); + assert_eq!( + entry.not_after, facts.not_after, + "a refusal retains the lifetime the leaf still has" + ); + match entry.attempt { + RenewalAttempt::Failed { ref reason } => assert!( + reason.contains("endpoint pin file"), + "the recorded reason names the pin file: {reason}" + ), + ref other => panic!("expected a failed attempt, found {other:?}"), + } + assert!(entry.attempted_at.is_some(), "a refusal stamps its time"); +} + +/// A bundle write that fails restores the bundle, leaves the pair and +/// the active configuration alone, and cleans every private artifact. +#[tokio::test] +async fn an_injected_bundle_failure_rolls_back_and_keeps_the_old_configuration() { + let harness = Harness::build(); + let (live, restores) = FaultyPaths::new(true, false, false); + let renewal = harness.renewal().await.with_live_paths(live); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let facts = LiveFacts::capture(&harness, pair.leaf, &renewal.state()); + let material = harness.ca.material(&pair.name, -1, 60); + + let err = renewal + .renew_leaf_with_material(&pair, material) + .await + .expect_err("an injected bundle failure fails the attempt"); + let rendered = format!("{err:#}"); + assert!( + rendered.contains("injected CA bundle write failure"), + "{rendered}" + ); + assert!( + rendered.contains("was restored from its snapshot"), + "a successful rollback says so: {rendered}" + ); + assert_eq!( + restores.load(Ordering::SeqCst), + 1, + "only the bundle's publication had started, so only it is restored" + ); + facts.assert_unchanged(&harness, pair.leaf); + assert_no_artifacts_left(&pair.key_path); +} + +/// A certificate-and-key write that fails restores all three live paths, +/// with their bytes, modes and ownership. +#[tokio::test] +async fn an_injected_pair_failure_restores_every_path_the_publication_reached() { + let harness = Harness::build(); + let (live, restores) = FaultyPaths::new(false, true, false); + let renewal = harness.renewal().await.with_live_paths(live); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let facts = LiveFacts::capture(&harness, pair.leaf, &renewal.state()); + let key_mode = std::fs::metadata(&pair.key_path) + .expect("stat the key") + .mode() + & 0o777; + let material = harness.ca.material(&pair.name, -1, 60); + + let err = renewal + .renew_leaf_with_material(&pair, material) + .await + .expect_err("an injected pair failure fails the attempt"); + let rendered = format!("{err:#}"); + assert!( + rendered.contains("injected certificate and key write failure"), + "{rendered}" + ); + assert_eq!( + restores.load(Ordering::SeqCst), + 3, + "the bundle, the certificate and the key had all been reached" + ); + facts.assert_unchanged(&harness, pair.leaf); + assert_eq!( + std::fs::metadata(&pair.key_path) + .expect("stat the key") + .mode() + & 0o777, + key_mode, + "a restore puts the mode back, not only the bytes" + ); + assert_no_artifacts_left(&pair.key_path); +} + +/// A rollback that itself fails keeps the old configuration, reports +/// both errors, cleans the artifacts, and does not claim the live files +/// were restored. +#[tokio::test] +async fn an_injected_rollback_failure_reports_both_errors_and_claims_nothing() { + let harness = Harness::build(); + let (live, restores) = FaultyPaths::new(false, true, true); + let renewal = harness.renewal().await.with_live_paths(live); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let before_active = harness.endpoint.active_tls(); + let material = harness.ca.material(&pair.name, -1, 60); + + let err = renewal + .renew_leaf_with_material(&pair, material) + .await + .expect_err("a publication whose rollback also fails is a failed attempt"); + let rendered = format!("{err:#}"); + assert!( + rendered.contains("injected certificate and key write failure"), + "the publication error is carried: {rendered}" + ); + assert!( + rendered.contains("injected rollback failure"), + "the rollback error is carried too: {rendered}" + ); + assert!( + rendered.contains("mixed state"), + "a failed rollback names the state it left: {rendered}" + ); + assert!( + !rendered.contains("was restored from its snapshot"), + "a failed rollback is never described as restored: {rendered}" + ); + assert_eq!( + restores.load(Ordering::SeqCst), + 3, + "every reached path's restore was attempted" + ); + assert!( + Arc::ptr_eq(&before_active, &harness.endpoint.active_tls()), + "a failed publication never exchanges the active configuration" + ); + assert_no_artifacts_left(&pair.key_path); +} + +/// No renewal artifact survives an attempt, whichever way it ended. +fn assert_no_artifacts_left(neighbour: &Path) { + let parent = neighbour.parent().expect("a parent directory"); + let leftovers: Vec = std::fs::read_dir(parent) + .expect("read the material directory") + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.starts_with(ARTIFACT_DIR_PREFIX)) + .collect(); + assert!( + leftovers.is_empty(), + "private renewal artifacts must be removed on every exit: {leftovers:?}" + ); +} + +// --------------------------------------------------------------------- +// The pass and the loop +// --------------------------------------------------------------------- + +/// A pass over material that is not due changes nothing and asks +/// `OpenBao` for nothing — the fixture's `OpenBao` URL points at a port +/// nothing listens on, so a request would fail rather than pass. +#[tokio::test] +async fn a_no_op_pass_changes_nothing() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let before: Vec = harness + .plan + .pairs + .iter() + .map(|pair| digest_of(&pair.cert_path)) + .collect(); + let before_active = harness.endpoint.active_tls(); + + renewal.run_pass().await; + + for (pair, digest) in harness.plan.pairs.iter().zip(before) { + assert_eq!( + digest, + digest_of(&pair.cert_path), + "a no-op pass publishes nothing" + ); + } + assert!( + Arc::ptr_eq(&before_active, &harness.endpoint.active_tls()), + "a no-op pass exchanges nothing" + ); + for leaf in [SurfaceLeaf::EndpointServer, SurfaceLeaf::RegistrarClient] { + assert_eq!( + renewal.state().leaf(leaf).expect("an entry").attempt, + RenewalAttempt::NeverAttempted, + "a no-op pass is not an attempt" + ); + } +} + +/// A pass that finds a leaf due and cannot reach the internal credential +/// records an ordinary failed attempt for it, retains the lifetime, and +/// leaves the loop able to try again. +#[tokio::test] +async fn a_credential_failure_is_an_ordinary_failed_attempt() { + let harness = Harness::build(); + // Lead time wide enough that both leaves are due, so the pass + // reaches the credential. + let renewal = RegistrarCertRenewal::for_test( + Arc::clone(&harness.settings), + harness.plan.clone(), + Arc::clone(&harness.endpoint), + RenewalCadence { + renew_before: Duration::from_hours(24 * 365), + ..cadence() + }, + ) + .await; + let facts = LiveFacts::capture(&harness, SurfaceLeaf::EndpointServer, &renewal.state()); + + // The secrets directory has no credential at all, which is what an + // unreachable internal credential looks like from here. + renewal.run_pass().await; + + for leaf in [SurfaceLeaf::EndpointServer, SurfaceLeaf::RegistrarClient] { + let entry = renewal.state().leaf(leaf).expect("an entry"); + match entry.attempt { + RenewalAttempt::Failed { ref reason } => assert!( + reason.contains("bootroot-internal credential"), + "the pass reaches OpenBao through the internal credential's certificate login \ + and through nothing else — no AppRole, no role_id, no secret_id: {reason}" + ), + ref other => panic!("expected a failed attempt, found {other:?}"), + } + assert!(entry.attempted_at.is_some(), "a failure stamps its time"); + } + assert_eq!( + renewal + .state() + .leaf(SurfaceLeaf::EndpointServer) + .expect("an entry") + .not_after, + facts.not_after, + "a failed attempt retains the lifetime" + ); + facts.assert_unchanged(&harness, SurfaceLeaf::EndpointServer); +} + +/// The first pass is immediate, later passes wait the configured +/// interval, and the shutdown signal ends the loop so its handle joins. +#[tokio::test(start_paused = true)] +async fn the_first_pass_is_immediate_and_shutdown_ends_the_loop() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let (tx, rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(renewal.run(rx)); + + // Nothing is due, so a pass is observable only as the loop reaching + // its next sleep. Advancing by less than the interval must not end + // the loop, and the stop must. + tokio::time::advance(Duration::from_secs(1)).await; + assert!(!task.is_finished(), "the loop keeps ticking"); + tokio::time::advance(Duration::from_secs(3600)).await; + assert!( + !task.is_finished(), + "a scheduled pass does not end the loop" + ); + + tx.send(true).expect("the loop is still listening"); + task.await + .expect("the renewal task joins") + .expect("the loop ends cleanly"); +} + +/// A stop that arrives before the first pass ends the loop without +/// running one. +#[tokio::test(start_paused = true)] +async fn a_stop_before_the_first_pass_runs_none() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let state = renewal.state(); + let (tx, rx) = tokio::sync::watch::channel(true); + tokio::spawn(renewal.run(rx)) + .await + .expect("the renewal task joins") + .expect("the loop ends cleanly"); + drop(tx); + + for leaf in [SurfaceLeaf::EndpointServer, SurfaceLeaf::RegistrarClient] { + assert_eq!( + state.leaf(leaf).expect("an entry").attempt, + RenewalAttempt::NeverAttempted + ); + } +} + +// --------------------------------------------------------------------- +// The reload contract, end to end over the socket +// --------------------------------------------------------------------- + +/// The renewed server leaf is presented, and a caller under the rotated +/// anchor is accepted, from the next handshake onwards — with no +/// restart, no signal, the same socket inode, and a connection already +/// in flight left alone. +#[tokio::test] +async fn the_next_handshake_uses_the_renewed_material_without_a_restart() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let inode_before = std::fs::metadata(&harness.socket_path) + .expect("stat the socket") + .ino(); + + // A connection established before the swap, held open across it. + let held = tokio::net::UnixStream::connect(&harness.socket_path) + .await + .expect("connect before the swap"); + + let material = harness.ca.material(&pair.name, -1, 60); + let artifacts = Artifacts::create(&pair.key_path).expect("artifacts"); + renewal + .publish_candidate(&pair, &material, &artifacts) + .await + .expect("the candidate publishes"); + artifacts.close().expect("artifacts are removed"); + + assert_eq!( + inode_before, + std::fs::metadata(&harness.socket_path) + .expect("stat the socket") + .ino(), + "a TLS replacement never rebinds or replaces the socket" + ); + assert!( + held.peer_addr().is_ok(), + "a connection already in flight is not dropped" + ); + + // The acceptor the next handshake would load presents the renewed + // leaf: the resolver behind it is the one built from the candidate. + let renewed_leaf = std::fs::read_to_string(&pair.cert_path).expect("read the published leaf"); + assert!( + renewed_leaf.contains( + material + .cert_pem + .lines() + .nth(1) + .expect("the leaf PEM has a body line") + ), + "the published certificate is the candidate that was validated" + ); + drop(held); +} From 1207718170f0b5c0c0ac5038a9331e07e2d5bd3e Mon Sep 17 00:00:00 2001 From: sehkone Date: Sat, 29 Aug 2026 16:51:55 +0900 Subject: [PATCH 2/5] Keep a renewed leaf's own outcome truthful The accessor exists to state one thing correctly: the lifetime of the certificate actually at a leaf's path, and how the last attempt on it ended. Three paths could make it say otherwise. An issuance carried its fresh private key as a bare String inside a struct that derives Debug, so the key of every certificate this crate issues would print into whatever log line, trace field or error chain first rendered one. It is wrapped now, in the newtype the internal leaf's key already used. A publication that reached the active configuration really did renew the leaf. Failing the attempt because the 0700 working directory beside the pair would not unlink would retain the old notAfter against a certificate that has just been replaced, so the cleanup is folded in rather than substituted: reported beside the success, and attached to the error a failed attempt already carries. The lifetime a success records is read off the candidate before anything is published rather than off the file it becomes. They are the same bytes, but a read-back is one more thing that can fail after the swap has already happened, and a failure there would record the leaf as failed at exactly the moment it was renewed. Rollback's owner restoration is pub(crate) with the same reasoning that keeps every other protected write root-owned: it is the one way to publish under an owner that is not root, and it exists for a snapshot putting back ids it read off the file itself. Part of #768 --- src/acme/flow.rs | 12 +++- src/daemon.rs | 4 +- src/fs_util.rs | 18 ++++-- src/registrar/internal/material.rs | 6 +- src/registrar_certs/tests.rs | 3 +- src/registrar_renewal.rs | 72 +++++++++++++++++++----- src/registrar_renewal/tests.rs | 90 ++++++++++++++++++++++++++++-- 7 files changed, 174 insertions(+), 31 deletions(-) diff --git a/src/acme/flow.rs b/src/acme/flow.rs index 44787049..225537bd 100644 --- a/src/acme/flow.rs +++ b/src/acme/flow.rs @@ -12,6 +12,7 @@ use crate::acme::responder_client; use crate::acme::types::{AuthorizationStatus, ChallengeStatus, ChallengeType, OrderStatus}; use crate::cert_group::CertGroupPolicy; use crate::fs_util; +use crate::registrar::internal::PrivateKeyPem; fn contact_from_email(email: &str) -> String { if email.starts_with("mailto:") { @@ -511,7 +512,12 @@ pub(crate) struct IssuedMaterial { /// [`IssuanceOptions::leaf_publication`] selects. pub(crate) cert_pem: String, /// The private key, as PEM. Every issuance generates a fresh one. - pub(crate) key_pem: String, + /// + /// Wrapped rather than bare, because this value is now carried in a + /// struct: the `Debug` above would otherwise print the key of every + /// certificate this crate issues into whatever log line, trace field + /// or error chain first rendered one. + pub(crate) key_pem: PrivateKeyPem, /// The issuer chain the CA returned, as DER, with the leaf removed. /// /// Empty when `trust.ca_bundle_path` is unconfigured, where the @@ -626,7 +632,7 @@ async fn publish_issued_material( &profile.paths.cert, &profile.paths.key, &material.cert_pem, - &material.key_pem, + material.key_pem.expose(), policy, ) .await?; @@ -722,7 +728,7 @@ async fn run_issuance( Ok(Some(IssuedMaterial { cert_pem, - key_pem: cert_key.serialize_pem(), + key_pem: PrivateKeyPem::new(cert_key.serialize_pem()), chain, })) } diff --git a/src/daemon.rs b/src/daemon.rs index c7302cbd..0b326745 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -869,11 +869,11 @@ fn spawn_registrar_endpoint( /// before the endpoint serves anything. The handle joins with every /// other daemon task, and the adapter takes the same shutdown watch. /// -/// A preparation that fails is a warning and not a refused start: the +/// A preparation that fails is logged and not a refused start: the /// endpoint is already up on material that is usable today, and taking /// it down because renewal could not be armed would turn a future /// problem into an immediate outage. Nothing is renewed until the next -/// daemon start in that case, which is what the warning says. +/// daemon start in that case, which is what the error line says. /// /// Where the endpoint is disabled this is never reached at all: no /// adapter, no state entry, no renewal work, no `OpenBao` request and no diff --git a/src/fs_util.rs b/src/fs_util.rs index 4c19b375..7f18f399 100644 --- a/src/fs_util.rs +++ b/src/fs_util.rs @@ -1053,9 +1053,11 @@ pub enum StagedOwner { /// The owner is an input rather than a constant so the real staging, /// chown and rename path can be driven in a test that is not root. It /// is not an input an operator can reach: [`FixedOwner::root`] is the -/// only constructor outside `cfg(test)`, so no configuration key, -/// environment variable or public API can move a protected file off -/// uid 0 / gid 0. +/// only `pub` constructor, so no configuration key, environment +/// variable or public API can move a protected file off uid 0 / gid 0. +/// `restored` is `pub(crate)` and states nothing of its own — it carries +/// back the ids a rollback snapshot read off the file it is putting +/// back — and `current_process` is `cfg(test)`-gated. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct FixedOwner { uid: u32, @@ -1094,8 +1096,16 @@ impl FixedOwner { /// captured rather than from a policy. A publish that cannot /// establish them fails and leaves the destination untouched, /// which is what a rollback that cannot restore has to do. + /// + /// `pub(crate)` rather than `pub`: it is reached from the registrar + /// renewal's rollback and from nowhere else, and widening it would + /// hand a caller outside this crate the one way to publish a + /// protected file under an owner that is not root. + // That one caller is the renewal adapter, which exists on Linux + // alone. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] #[must_use] - pub fn restored(uid: u32, gid: u32) -> Self { + pub(crate) fn restored(uid: u32, gid: u32) -> Self { Self { uid, gid } } diff --git a/src/registrar/internal/material.rs b/src/registrar/internal/material.rs index 05b54304..b941997d 100644 --- a/src/registrar/internal/material.rs +++ b/src/registrar/internal/material.rs @@ -21,7 +21,11 @@ const FINGERPRINT_HEX_LEN: usize = 64; /// The opening of every PEM block header. const PEM_OPENING: &str = "-----BEGIN "; -/// The internal leaf's private key, in PEM. +/// A private key held in memory, in PEM. +/// +/// Named here because the internal leaf's key was the first of them, +/// and reached from wherever else this crate carries one — an ACME +/// issuance's fresh key on its way to publication is the other. /// /// A newtype whose `Debug` prints ``, following the verb /// layer's `WrappedSecretIdToken`: a `#[derive(Debug)]` on anything that diff --git a/src/registrar_certs/tests.rs b/src/registrar_certs/tests.rs index 0ab1a871..48e3a7cc 100644 --- a/src/registrar_certs/tests.rs +++ b/src/registrar_certs/tests.rs @@ -1559,7 +1559,8 @@ async fn two_off_live_issuances_of_the_same_name_produce_different_keys() { .expect("second candidate"); assert_ne!( - first.key_pem, second.key_pem, + first.key_pem.expose(), + second.key_pem.expose(), "each issuance must generate a key" ); assert_ne!(first.cert_pem, second.cert_pem); diff --git a/src/registrar_renewal.rs b/src/registrar_renewal.rs index f9486e26..4b94f50d 100644 --- a/src/registrar_renewal.rs +++ b/src/registrar_renewal.rs @@ -956,26 +956,41 @@ impl RegistrarCertRenewal { /// Split at exactly the point the ACME exchange ends, so a test /// drives the whole publication path over material it minted itself /// without a CA to reach. + /// + /// The cleanup is not part of the attempt's outcome. A publication + /// that reached the active configuration renewed the leaf, and + /// recording it as failed because a `0700` directory beside the pair + /// would not unlink would retain the *old* `notAfter` against a leaf + /// that has just been replaced — which is the one thing the accessor + /// exists to state correctly. The removal's own failure is reported + /// rather than hidden either way: attached to the error a failed + /// attempt already carries, and logged beside the success otherwise. async fn renew_leaf_with_material( &self, pair: &SurfacePairPaths, material: crate::acme::IssuedMaterial, ) -> Result<()> { + // Read off the candidate rather than off the file it becomes, + // and read before the publication rather than after it, for the + // reason above: these are the exact bytes `write_pair` puts at + // `cert_path`, so it is the published certificate's lifetime + // either way, but a *read-back* is one more thing that can fail + // after the swap has already happened — and a failure there + // would record `failed` against a leaf that had in fact just + // been replaced. + let not_after = + daemon::parse_cert_not_after(material.cert_pem.as_bytes()).with_context(|| { + format!( + "parsing the candidate {} to record the lifetime a publication would give it", + pair.leaf.label() + ) + })?; let artifacts = Artifacts::create(&pair.key_path)?; let outcome = self.publish_candidate(pair, &material, &artifacts).await; - // Removed on every exit, whatever the outcome was, and the - // removal's own failure is reported rather than hidden. + // Removed on every exit, whatever the outcome was. let cleanup = artifacts.close(); - let outcome = match (outcome, cleanup) { - (Ok(()), Ok(())) => Ok(()), - (Err(err), Ok(())) | (Ok(()), Err(err)) => Err(err), - (Err(err), Err(cleanup)) => Err(err.context(format!( - "and the private renewal artifacts could not be removed: {cleanup:#}" - ))), - }; - outcome?; + fold_cleanup(outcome, cleanup, pair.leaf)?; - let not_after = observed_not_after(&pair.cert_path).await?; self.state .record_success(pair.leaf, not_after, OffsetDateTime::now_utc()); info!( @@ -1046,7 +1061,7 @@ impl RegistrarCertRenewal { validate_candidate( pair.leaf, material.cert_pem.as_bytes(), - material.key_pem.as_bytes(), + material.key_pem.expose().as_bytes(), &pair.name, &self.settings.domain, &pin_file, @@ -1075,7 +1090,7 @@ impl RegistrarCertRenewal { )?; let candidate_key = artifacts.write( CANDIDATE_KEY_FILE, - material.key_pem.as_bytes(), + material.key_pem.expose().as_bytes(), KEY_FILE_MODE_DEFAULT, )?; @@ -1116,7 +1131,7 @@ impl RegistrarCertRenewal { &pair.cert_path, &pair.key_path, &material.cert_pem, - &material.key_pem, + material.key_pem.expose(), ) .await { @@ -1231,6 +1246,35 @@ impl RegistrarCertRenewal { } } +/// Folds a publication's outcome and its cleanup's into the attempt's. +/// +/// The cleanup is not part of the attempt. A publication that reached +/// the active configuration renewed the leaf, and calling that a failed +/// attempt because a `0700` directory beside the pair would not unlink +/// would retain the *old* `notAfter` against a leaf that has just been +/// replaced — the one thing the accessor exists to state correctly. +/// +/// The removal's own failure is reported rather than hidden either way: +/// attached to the error a failed attempt already carries, and logged +/// beside the success otherwise. +fn fold_cleanup(publication: Result<()>, cleanup: Result<()>, leaf: SurfaceLeaf) -> Result<()> { + match (publication, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(cleanup)) => { + error!( + "Registrar renewal replaced {} but could not remove its private working \ + directory: {cleanup:#}", + leaf.label() + ); + Ok(()) + } + (Err(err), Ok(())) => Err(err), + (Err(err), Err(cleanup)) => Err(err.context(format!( + "and the private renewal artifacts could not be removed: {cleanup:#}" + ))), + } +} + /// Reads the `notAfter` of the certificate at `path`. /// /// # Errors diff --git a/src/registrar_renewal/tests.rs b/src/registrar_renewal/tests.rs index 3837e293..8b1b2f3a 100644 --- a/src/registrar_renewal/tests.rs +++ b/src/registrar_renewal/tests.rs @@ -27,6 +27,7 @@ use crate::config::{RegistrarEndpointSettings, Settings}; use crate::registrar::endpoint; use crate::registrar::endpoint::activation::ActivationContract; use crate::registrar::endpoint_pin::REGISTRAR_ENDPOINT_ANCHORS_FILE; +use crate::registrar::internal::PrivateKeyPem; const TEST_DOMAIN: &str = "corp.example.internal"; const TEST_HOST: &str = "bootroot-01"; @@ -93,7 +94,7 @@ impl TestCa { let (leaf_pem, key_pem) = self.issue(name, not_before_days, not_after_days); IssuedMaterial { cert_pem: format!("{leaf_pem}{}", self.pem), - key_pem, + key_pem: PrivateKeyPem::new(key_pem), chain: vec![self.der()], } } @@ -675,7 +676,7 @@ async fn a_conforming_candidate_is_accepted_for_both_leaves() { validate_candidate( leaf, material.cert_pem.as_bytes(), - material.key_pem.as_bytes(), + material.key_pem.expose().as_bytes(), &pair.name, TEST_DOMAIN, &harness.pin_file(), @@ -697,7 +698,7 @@ async fn a_mismatched_key_or_a_drifted_name_is_refused() { let rejection = validate_candidate( SurfaceLeaf::RegistrarClient, material.cert_pem.as_bytes(), - other.key_pem.as_bytes(), + other.key_pem.expose().as_bytes(), &pair.name, TEST_DOMAIN, &harness.pin_file(), @@ -715,7 +716,7 @@ async fn a_mismatched_key_or_a_drifted_name_is_refused() { let rejection = validate_candidate( SurfaceLeaf::RegistrarClient, drifted.cert_pem.as_bytes(), - drifted.key_pem.as_bytes(), + drifted.key_pem.expose().as_bytes(), &pair.name, TEST_DOMAIN, &harness.pin_file(), @@ -743,7 +744,7 @@ async fn a_server_candidate_under_an_unpinned_anchor_is_refused() { let rejection = validate_candidate( SurfaceLeaf::EndpointServer, material.cert_pem.as_bytes(), - material.key_pem.as_bytes(), + material.key_pem.expose().as_bytes(), &pair.name, TEST_DOMAIN, &harness.pin_file(), @@ -763,7 +764,7 @@ async fn a_server_candidate_under_an_unpinned_anchor_is_refused() { validate_candidate( SurfaceLeaf::RegistrarClient, client_material.cert_pem.as_bytes(), - client_material.key_pem.as_bytes(), + client_material.key_pem.expose().as_bytes(), &client.name, TEST_DOMAIN, &harness.pin_file(), @@ -1021,6 +1022,51 @@ async fn an_unpinned_server_candidate_changes_no_live_or_active_state() { assert!(entry.attempted_at.is_some(), "a refusal stamps its time"); } +/// A publication that reached the active configuration is recorded as a +/// success: the accessor takes the *published* certificate's lifetime, +/// stamps the attempt, and no artifact survives it. +/// +/// The three injections below drive the same entry point on their +/// failure halves. This is the other half, and the only place +/// `record_success` is reached from a real publication rather than from +/// the accessor's own unit test — the lifetime it writes has to be the +/// new certificate's and not the one initialization observed. +#[tokio::test] +async fn a_published_renewal_records_the_new_lifetime_and_a_success() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let state = renewal.state(); + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let before = state.leaf(pair.leaf).expect("initialization made an entry"); + assert_eq!(before.attempt, RenewalAttempt::NeverAttempted); + assert!(before.attempted_at.is_none()); + + // The live leaf runs to 30 days; the candidate to 60. + let material = harness.ca.material(&pair.name, -1, 60); + renewal + .renew_leaf_with_material(&pair, material) + .await + .expect("a conforming candidate publishes"); + + let after = state.leaf(pair.leaf).expect("the entry survives"); + assert_eq!(after.attempt, RenewalAttempt::Succeeded); + assert!( + after.not_after > before.not_after, + "a success replaces the lifetime with the published certificate's, not the one \ + initialization observed" + ); + assert!(after.attempted_at.is_some(), "a success stamps its time"); + assert_eq!( + state + .leaf(SurfaceLeaf::RegistrarClient) + .expect("an entry") + .attempt, + RenewalAttempt::NeverAttempted, + "renewing one leaf is never an attempt against the other" + ); + assert_no_artifacts_left(&pair.key_path); +} + /// A bundle write that fails restores the bundle, leaves the pair and /// the active configuration alone, and cleans every private artifact. #[tokio::test] @@ -1155,6 +1201,38 @@ fn assert_no_artifacts_left(neighbour: &Path) { ); } +/// A cleanup that fails is reported but does not turn a publication +/// into a failed attempt. +/// +/// The leaf really was replaced and the active configuration really was +/// exchanged; calling that failed would retain the lifetime the *old* +/// certificate had against the new one, which is the single value the +/// accessor exists to state correctly. A publication that failed carries +/// the cleanup's error alongside its own instead. +#[test] +fn a_cleanup_failure_does_not_make_a_published_renewal_a_failed_attempt() { + fold_cleanup(Ok(()), Ok(()), SurfaceLeaf::EndpointServer).expect("a clean attempt succeeds"); + fold_cleanup( + Ok(()), + Err(anyhow::anyhow!("the working directory would not unlink")), + SurfaceLeaf::EndpointServer, + ) + .expect("a published renewal is a success whatever the cleanup did"); + + let err = fold_cleanup( + Err(anyhow::anyhow!("the bundle write failed")), + Err(anyhow::anyhow!("the working directory would not unlink")), + SurfaceLeaf::EndpointServer, + ) + .expect_err("a failed publication stays failed"); + let rendered = format!("{err:#}"); + assert!(rendered.contains("the bundle write failed"), "{rendered}"); + assert!( + rendered.contains("the working directory would not unlink"), + "the cleanup's own failure is reported rather than hidden: {rendered}" + ); +} + // --------------------------------------------------------------------- // The pass and the loop // --------------------------------------------------------------------- From 3d31db25ea0729d41600f691ebcabdc3e46d0f50 Mon Sep 17 00:00:00 2001 From: sehkone Date: Sat, 29 Aug 2026 17:47:20 +0900 Subject: [PATCH 3/5] Arm registrar renewal before the endpoint serves An enabled endpoint owes both leaves a renewal adapter and both accessor entries, and a failed attempt has no lifetime with which to create a missing entry. Logging a failed preparation and spawning the accept task anyway therefore produced exactly the state the adapter exists to prevent: an endpoint serving two certificates that nothing would renew, with no entry to report on them either. Preparation now happens beside the handler's own resolution, before anything is spawned, and a failure fails the invocation. That is the same rule the accept task's other dependencies already follow, and the material it needs is already guaranteed by start-time issuance, which runs before the daemon does. Initialization no longer tolerates a leaf it cannot observe, for the same reason. The cadence's retry budget also goes through the daemon's own selection now. The internal config's sole profile may carry a `[profiles.retry]` of its own, and copying the top-level table would have issued the registrar leaves under a different budget from the profile every other cadence value comes from. Part of #768 --- CHANGELOG.md | 5 +- docs/en/operations.md | 6 ++ docs/ko/operations.md | 6 ++ src/daemon.rs | 110 +++++++++++++--------- src/daemon/registrar_handler_tests.rs | 81 ++++++++++++++++ src/registrar_renewal.rs | 91 +++++++++++++----- src/registrar_renewal/tests.rs | 128 ++++++++++++++++++++++++++ 7 files changed, 355 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 619c9399..05367078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,10 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm never gains a leaf fingerprint. A caller reloads per dial: it rereads the pair every time and, because the two files are published by separate renames, retries a momentarily mismatched pair up to five - reads before failing rather than presenting it. + reads before failing rather than presenting it. A host that enables the + endpoint but cannot arm that loop — no usable internal agent + configuration, or a leaf on disk that no longer parses — now fails to + start and says so, instead of serving certificates nothing would renew. - `bootroot-agent` now rotates `OpenBao`'s file audit device on a host whose registrar endpoint is enabled, so the deployment no longer needs an external rotator against it. Every 60 seconds the daemon renames the diff --git a/docs/en/operations.md b/docs/en/operations.md index bac5432d..3cc38e0b 100644 --- a/docs/en/operations.md +++ b/docs/en/operations.md @@ -2021,6 +2021,12 @@ is what a trust-anchor rotation under a running daemon leaves behind. Where the endpoint is disabled nothing of this exists: no pass runs and nothing is asked of OpenBao or the CA. +Arming that loop is part of starting. An enabled endpoint whose renewal cannot +be prepared — a rendered internal agent configuration that is missing or no +longer loads, a deployment state file that cannot be resolved, a leaf +certificate on disk that no longer parses — fails the daemon start with a named +error instead of serving on two certificates nothing would renew. + A renewal is published as a transaction. The replacement is issued, validated and turned into a complete next TLS configuration **before** any live file is written: its key must be the leaf's, a client replacement must carry the same diff --git a/docs/ko/operations.md b/docs/ko/operations.md index dbc68164..6a96b68f 100644 --- a/docs/ko/operations.md +++ b/docs/ko/operations.md @@ -1920,6 +1920,12 @@ step-ca로 나가는 ACME 경로를 통해서입니다. 프로비저닝 도구 이 중 아무것도 존재하지 않습니다. 패스는 돌지 않고, OpenBao와 CA에 아무것도 요청하지 않습니다. +이 루프를 준비하는 일은 기동의 일부입니다. 엔드포인트가 활성화되어 있는데 갱신을 +준비할 수 없는 호스트는 — 렌더링된 내부 에이전트 설정이 없거나 더 이상 읽히지 +않을 때, 배포 상태 파일을 해석할 수 없을 때, 디스크의 리프 인증서가 더 이상 +파싱되지 않을 때 — 아무것도 갱신하지 않을 인증서 두 장으로 서비스를 시작하는 +대신, 이름이 붙은 오류와 함께 데몬 기동을 실패시킵니다. + 갱신은 하나의 트랜잭션으로 발행됩니다. 대체 자재는 **살아 있는 파일을 하나라도 쓰기 전에** 발급되고 검증되어 완전한 다음 TLS 설정으로 조립됩니다. 키는 그 리프의 키여야 하고, 클라이언트 대체본은 원래 갖고 있던 인스턴스·호스트·도메인을 diff --git a/src/daemon.rs b/src/daemon.rs index 0b326745..1787c89b 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -194,7 +194,19 @@ pub(crate) async fn run_daemon(invocation: DaemonInvocation) -> anyhow::Result<( let semaphore = Arc::new(Semaphore::new(max_concurrent)); #[cfg(target_os = "linux")] - let registrar_service = resolve_registrar_service(®istrar_endpoint, &settings).await?; + let registrar_service = match resolve_registrar_service(®istrar_endpoint, &settings).await? { + // Armed here, beside the other dependency the accept task cannot + // run without, and before anything at all is spawned: an enabled + // endpoint owes both leaves a renewal adapter and both accessor + // entries, so one that cannot arm them fails the invocation + // rather than serving without them. + Some((endpoint, built)) => { + let renewal = + prepare_registrar_cert_renewal(&settings, &endpoint, insecure_mode).await?; + Some((endpoint, built, renewal)) + } + None => None, + }; #[cfg(not(target_os = "linux"))] let _ = ®istrar_endpoint; let profile_locks = Arc::new(ProfileLocks::new()); @@ -219,23 +231,16 @@ pub(crate) async fn run_daemon(invocation: DaemonInvocation) -> anyhow::Result<( let mut handles = Vec::new(); #[cfg(target_os = "linux")] - let registrar_maintenance = if let Some((endpoint, built)) = registrar_service { + let registrar_maintenance = if let Some((endpoint, built, renewal)) = registrar_service { let BuiltRegistrarHandler { handler, maintenance, } = built; - // Before the accept task, deliberately: the adapter initializes - // the per-leaf renewal state from the certificates start-time - // issuance has already made usable, and that reading happens - // before the endpoint begins serving. - spawn_registrar_cert_renewal( - &mut handles, - &settings, - Arc::clone(&endpoint), - insecure_mode, - &shutdown_rx, - ) - .await; + // Before the accept task, deliberately: the adapter has already + // initialized the per-leaf renewal state from the certificates + // start-time issuance made usable, and its loop starts before + // the endpoint begins serving. + spawn_registrar_cert_renewal(&mut handles, renewal, &shutdown_rx); spawn_registrar_endpoint( &mut handles, endpoint, @@ -860,48 +865,54 @@ fn spawn_registrar_endpoint( })); } -/// Spawns the adapter that keeps the registrar surface's two leaves +/// Arms the adapter that keeps the registrar surface's two leaves /// valid, when the endpoint is enabled. /// -/// Called with the endpoint already activated and *before* the accept -/// task is spawned, so the per-leaf renewal state is initialized from -/// the certificates start-time issuance ensured, and is initialized -/// before the endpoint serves anything. The handle joins with every -/// other daemon task, and the adapter takes the same shutdown watch. +/// Called with the endpoint already activated and **before** anything is +/// spawned, for the reason [`resolve_registrar_service`] gives about its +/// own `?`: an enabled endpoint owes both leaves a renewal adapter and +/// both accessor entries, and one that serves without them is a surface +/// whose two certificates expire under a daemon that will never notice. +/// A rendered internal `agent.toml` that is missing or malformed, a +/// deployment state file that cannot be resolved or a leaf whose +/// certificate no longer parses is therefore a loud, named startup +/// failure and not a warning over a running endpoint. /// -/// A preparation that fails is logged and not a refused start: the -/// endpoint is already up on material that is usable today, and taking -/// it down because renewal could not be armed would turn a future -/// problem into an immediate outage. Nothing is renewed until the next -/// daemon start in that case, which is what the error line says. +/// Preparing here also does the reading the ordering requires: the +/// per-leaf renewal state is initialized from the certificates start-time +/// issuance ensured, before the endpoint serves anything. /// -/// Where the endpoint is disabled this is never reached at all: no -/// adapter, no state entry, no renewal work, no `OpenBao` request and no -/// CA request. +/// # Errors +/// +/// Returns whatever [`crate::registrar_renewal::RegistrarCertRenewal::prepare`] +/// could not resolve or observe. #[cfg(target_os = "linux")] -async fn spawn_registrar_cert_renewal( - handles: &mut Vec>>, +async fn prepare_registrar_cert_renewal( settings: &Arc, - endpoint: Arc, + endpoint: &Arc, insecure_mode: bool, - shutdown_rx: &watch::Receiver, -) { - let renewal = match crate::registrar_renewal::RegistrarCertRenewal::prepare( +) -> anyhow::Result { + crate::registrar_renewal::RegistrarCertRenewal::prepare( Arc::clone(settings), - endpoint, + Arc::clone(endpoint), insecure_mode, ) .await - { - Ok(renewal) => renewal, - Err(err) => { - error!( - "Registrar certificate renewal could not be armed, so neither registrar leaf \ - will be renewed under this daemon: {err:#}" - ); - return; - } - }; + .context("arming certificate renewal for the enabled registrar endpoint") +} + +/// Spawns the already-armed renewal adapter's loop. +/// +/// The handle joins with every other daemon task, and the adapter takes +/// the same shutdown watch. Where the endpoint is disabled this is never +/// reached at all: no adapter, no state entry, no renewal work, no +/// `OpenBao` request and no CA request. +#[cfg(target_os = "linux")] +fn spawn_registrar_cert_renewal( + handles: &mut Vec>>, + renewal: crate::registrar_renewal::RegistrarCertRenewal, + shutdown_rx: &watch::Receiver, +) { let shutdown_rx = shutdown_rx.clone(); handles.push(tokio::spawn(renewal.run(shutdown_rx))); } @@ -1267,7 +1278,14 @@ fn reload_profile_or_fallback( } } -fn select_retry_backoff<'a>( +/// Returns the issuance backoff one profile is renewed under. +/// +/// The profile's own `[profiles.retry]` wins where it has one, and the +/// top-level `[retry]` is the fallback. Shared with the registrar +/// surface's renewal adapter, whose two leaves have no `[[profiles]]` +/// entry of their own but are governed by the same policy read off the +/// rendered internal agent config's sole profile. +pub(crate) fn select_retry_backoff<'a>( settings: &'a config::Settings, profile: &'a config::DaemonProfileSettings, ) -> &'a [u64] { diff --git a/src/daemon/registrar_handler_tests.rs b/src/daemon/registrar_handler_tests.rs index 168fdd9c..77a101ca 100644 --- a/src/daemon/registrar_handler_tests.rs +++ b/src/daemon/registrar_handler_tests.rs @@ -210,6 +210,35 @@ fn configure_valid_profile_certificate(settings: &mut Settings, deployment: &Dep settings.profiles[0].paths.cert = certificate; } +/// Provisions the four registrar surface material paths an enabled +/// endpoint has by the time `run_daemon` is reached. +/// +/// A production start reaches the daemon only through +/// `ensure_registrar_surface_certificates`, which has already made both +/// pairs usable; a test that calls `run_daemon` directly bypasses that +/// step, and the renewal adapter — which reads both certificates before +/// the endpoint serves — refuses an endpoint it cannot observe. What is +/// written is a parsable self-signed pair per path, because +/// initialization records a lifetime and validates nothing else. +fn configure_registrar_surface_material(settings: &mut Settings, deployment: &Deployment) { + let key = KeyPair::generate().expect("generate surface material key"); + let certificate_pem = CertificateParams::new(Vec::new()) + .expect("build surface certificate") + .self_signed(&key) + .expect("self-sign surface certificate") + .pem(); + let key_pem = key.serialize_pem(); + let write = |name: &str, contents: &str| { + let path = deployment.path().join(name); + std::fs::write(&path, contents).expect("write surface material"); + path + }; + settings.registrar_endpoint.server_cert_path = Some(write("endpoint.crt", &certificate_pem)); + settings.registrar_endpoint.server_key_path = Some(write("endpoint.key", &key_pem)); + settings.registrar_endpoint.client_cert_path = Some(write("client.crt", &certificate_pem)); + settings.registrar_endpoint.client_key_path = Some(write("client.key", &key_pem)); +} + /// The projection reads exactly three members and tolerates every other /// one, so the CLI's inventory can grow a field without breaking a /// daemon that never looks at it. @@ -763,6 +792,7 @@ async fn an_unmounted_store_keeps_daemon_duties_running() { settings.registrar.audit_record_dir = unmounted_store.join("records"); configure_valid_profile_certificate(&mut settings, &deployment); + configure_registrar_surface_material(&mut settings, &deployment); let role_id = deployment.path().join("role-id"); let secret_id = deployment.path().join("secret-id"); @@ -1131,6 +1161,7 @@ async fn an_enabled_directory_endpoint_starts_the_ordinary_daemon_duties() { settings.registrar.audit_store_enforcement = AuditStoreEnforcement::Directory; settings.registrar.open_audit_store_as_test_user = true; configure_valid_profile_certificate(&mut settings, &deployment); + configure_registrar_surface_material(&mut settings, &deployment); let endpoint = crate::registrar::endpoint::DaemonTestEndpoint::bind() .expect("bind a test endpoint through the production adoption seam"); @@ -1212,6 +1243,56 @@ async fn an_enabled_directory_endpoint_starts_the_ordinary_daemon_duties() { .expect("the directory daemon shuts down cleanly"); } +/// An enabled endpoint whose certificate renewal cannot be armed is a +/// daemon failure too, and for the same reason the handler's own +/// dependencies are: the endpoint owes both leaves an adapter and both +/// accessor entries, and one that served without them would run two +/// certificates to expiry under a daemon that never noticed. The +/// refusal happens before anything is spawned, so the accept task is +/// never reached. +#[tokio::test] +async fn an_unarmable_registrar_renewal_stops_the_daemon() { + let deployment = Deployment::arrange(); + let mut settings = deployment.settings_with_endpoint(true); + settings.registrar.audit_store_enforcement = AuditStoreEnforcement::Directory; + settings.registrar.open_audit_store_as_test_user = true; + configure_valid_profile_certificate(&mut settings, &deployment); + configure_registrar_surface_material(&mut settings, &deployment); + + // The one thing start-time issuance guarantees, taken back: a leaf + // on disk whose lifetime cannot be observed. + let client_cert = settings + .registrar_endpoint + .client_cert_path + .clone() + .expect("the fixture configured a client certificate"); + std::fs::write(&client_cert, "not a certificate\n").expect("clobber the client leaf"); + + let endpoint = crate::registrar::endpoint::DaemonTestEndpoint::bind() + .expect("bind a test endpoint through the production adoption seam"); + let error = run_daemon(DaemonInvocation { + settings: Arc::new(settings), + default_eab: None, + eab_refresh_path: None, + config_path: Some(deployment.path().join("agent.toml")), + insecure_mode: false, + cli_overrides: crate::config::CliOverrides::default(), + shutdown: DaemonShutdown::new(), + registrar_endpoint: endpoint.registrar_endpoint(), + }) + .await + .expect_err("an endpoint whose renewal cannot be armed must stop the daemon"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("arming certificate renewal for the enabled registrar endpoint"), + "the failure names what could not be armed: {rendered}" + ); + assert!( + rendered.contains(&client_cert.display().to_string()), + "the failure names the leaf it could not observe: {rendered}" + ); +} + /// A production audit-store failure remains a daemon failure: directory mode /// reaches the real builder without consulting the mount predicate, and a /// regular file cannot become the record directory it needs. diff --git a/src/registrar_renewal.rs b/src/registrar_renewal.rs index 4b94f50d..cfb20700 100644 --- a/src/registrar_renewal.rs +++ b/src/registrar_renewal.rs @@ -82,7 +82,7 @@ use anyhow::{Context as _, Result}; use rustls::pki_types::{CertificateDer, UnixTime}; use time::OffsetDateTime; use tokio::sync::watch; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info}; use crate::cert_group::{CERT_FILE_MODE, CertGroupPolicy, KEY_FILE_MODE_DEFAULT}; use crate::config::Settings; @@ -191,10 +191,11 @@ impl RegistrarCertRenewalState { /// Records a successful attempt: the new `notAfter`, and when it ran. /// - /// Creates the entry when initialization could not read the - /// certificate that was there — that leaf has no lifetime to retain - /// and no entry, and this attempt has just given it one. Every other - /// write updates in place. + /// Written as an insert rather than an in-place update because it is + /// the one write that carries its own lifetime; initialization has + /// already made the entry on every enabled endpoint, since an + /// adapter whose accessor was missing one is refused before the + /// endpoint serves. pub(crate) fn record_success( &self, leaf: SurfaceLeaf, @@ -222,7 +223,9 @@ impl RegistrarCertRenewalState { /// /// A leaf with no entry gets none here: the entry's whole content is /// a lifetime a failure retains, and a failed attempt has no - /// lifetime to put in one. + /// lifetime to put in one. Nothing in the daemon reaches that case — + /// preparation refuses an endpoint whose leaves it could not + /// observe, so both entries exist before the first pass. pub(crate) fn record_failure(&self, leaf: SurfaceLeaf, reason: &str, at: OffsetDateTime) { self.with(|entries| { if let Some(entry) = entries.get_mut(&leaf) { @@ -688,7 +691,10 @@ pub(crate) struct RenewalCadence { pub(crate) check_jitter: Duration, /// How far ahead of `notAfter` a leaf becomes due. pub(crate) renew_before: Duration, - /// The issuance backoff, in seconds per retry. + /// The issuance backoff, in seconds per retry: the sole profile's + /// own `[profiles.retry]` where it has one, and the config's + /// top-level `[retry]` otherwise, selected exactly as the daemon + /// selects it for a `[[profiles]]` entry. pub(crate) retry_backoff: Vec, } @@ -719,7 +725,13 @@ impl RenewalCadence { check_interval: profile.daemon.check_interval, check_jitter: profile.daemon.check_jitter, renew_before: profile.daemon.renew_before, - retry_backoff: internal.retry.backoff_secs.clone(), + // The daemon's own selection, not a copy of the top-level + // table: the loader lets that profile carry a + // `[profiles.retry]` of its own, and a registrar leaf issued + // under a different budget from the profile it takes every + // other cadence value from would be a second retry policy + // wearing the first one's name. + retry_backoff: daemon::select_retry_backoff(&internal, profile).to_vec(), }) } } @@ -744,17 +756,23 @@ impl RegistrarCertRenewal { /// Initialization is not an attempt: it reads each enabled leaf's /// valid certificate, records its `notAfter` against /// [`RenewalAttempt::NeverAttempted`] with no timestamp, and - /// contacts neither `OpenBao` nor the CA. A leaf whose certificate - /// cannot be read or parsed gets no entry and is left to the first - /// pass, which finds it due and issues a replacement — start-time - /// issuance has already ensured usable material, so reaching that is - /// a file that changed underneath the daemon. + /// contacts neither `OpenBao` nor the CA. + /// + /// A leaf whose certificate cannot be read or parsed fails the + /// preparation instead of being left to the first pass. An enabled + /// endpoint owes both leaves an entry, and a failure has no lifetime + /// to create one with — so tolerating it here is how an endpoint + /// comes to serve with a leaf nothing reports on. Start-time + /// issuance has already made both pairs usable, so reaching this is + /// a file that changed underneath the daemon, and the caller's + /// refusal to serve names it. /// /// # Errors /// /// Returns an error when the deployment state file, the rendered /// internal agent config or the configured material paths cannot be - /// resolved. + /// resolved, or when either leaf's certificate cannot be read or + /// parsed. pub(crate) async fn prepare( settings: Arc, endpoint: Arc, @@ -763,7 +781,7 @@ impl RegistrarCertRenewal { let plan = resolve_surface_plan(&settings) .context("resolving the registrar surface renewal plan")?; let cadence = RenewalCadence::from_internal_config(&plan.secrets_dir)?; - Ok(Self::assemble(settings, plan, endpoint, cadence, insecure_mode).await) + Self::assemble(settings, plan, endpoint, cadence, insecure_mode).await } /// Initializes the accessor and assembles the adapter around an @@ -773,26 +791,31 @@ impl RegistrarCertRenewal { /// production initialization, pass and publication paths over a plan /// it wrote under `tempfile::tempdir()`, without a deployment state /// file to resolve them from. + /// + /// # Errors + /// + /// Returns an error when a leaf's certificate cannot be read or + /// parsed, so that no adapter exists whose accessor is missing one + /// of the two entries an enabled endpoint owes. async fn assemble( settings: Arc, plan: SurfacePlan, endpoint: Arc, cadence: RenewalCadence, insecure_mode: bool, - ) -> Self { + ) -> Result { let state = RegistrarCertRenewalState::default(); for pair in &plan.pairs { - match observed_not_after(&pair.cert_path).await { - Ok(not_after) => state.initialize(pair.leaf, not_after), - Err(err) => warn!( - "Registrar renewal could not read {} at {} to record its lifetime ({err:#}); \ - the first pass will treat it as due.", + let not_after = observed_not_after(&pair.cert_path).await.with_context(|| { + format!( + "recording the lifetime of the {} at {}", pair.leaf.label(), pair.cert_path.display() - ), - } + ) + })?; + state.initialize(pair.leaf, not_after); } - Self { + Ok(Self { settings, plan, endpoint, @@ -800,7 +823,7 @@ impl RegistrarCertRenewal { cadence, insecure_mode, live: Box::new(FilesystemPaths::new()), - } + }) } /// The assembly step alone, for a test that has its own plan. @@ -811,6 +834,24 @@ impl RegistrarCertRenewal { endpoint: Arc, cadence: RenewalCadence, ) -> Self { + Self::try_for_test(settings, plan, endpoint, cadence) + .await + .expect("the fixture writes both leaves before the adapter reads them") + } + + /// The same step, for a test that drives its refusal. + /// + /// # Errors + /// + /// Returns whatever [`RegistrarCertRenewal::assemble`] could not + /// observe. + #[cfg(test)] + pub(crate) async fn try_for_test( + settings: Arc, + plan: SurfacePlan, + endpoint: Arc, + cadence: RenewalCadence, + ) -> Result { Self::assemble(settings, plan, endpoint, cadence, false).await } diff --git a/src/registrar_renewal/tests.rs b/src/registrar_renewal/tests.rs index 8b1b2f3a..bd859b52 100644 --- a/src/registrar_renewal/tests.rs +++ b/src/registrar_renewal/tests.rs @@ -480,6 +480,36 @@ async fn initialization_records_both_leaves_without_attempting_anything() { } } +/// An enabled endpoint owes both leaves an entry, and a failed attempt +/// has no lifetime to make a missing one with. A leaf that cannot be +/// observed therefore refuses the adapter, so the daemon refuses to +/// serve rather than running an endpoint one of whose certificates +/// nothing tracks. +#[tokio::test] +async fn a_leaf_that_cannot_be_observed_refuses_the_adapter() { + let harness = Harness::build(); + let pair = harness.pair(SurfaceLeaf::RegistrarClient); + std::fs::write(&pair.cert_path, "not a certificate\n").expect("clobber the live leaf"); + + let refusal = RegistrarCertRenewal::try_for_test( + Arc::clone(&harness.settings), + harness.plan.clone(), + Arc::clone(&harness.endpoint), + cadence(), + ) + .await; + + let Err(error) = refusal else { + panic!("a leaf that cannot be observed must refuse the adapter"); + }; + + let rendered = format!("{error:#}"); + assert!( + rendered.contains(&pair.cert_path.display().to_string()), + "the refusal names the certificate it could not observe: {rendered}" + ); +} + /// The accessor is empty until something initializes it, which is what a /// disabled endpoint leaves behind: no adapter is built, so no entry /// exists and nothing is ever asked of `OpenBao` or the CA. @@ -549,6 +579,104 @@ fn a_failure_retains_the_lifetime_a_success_replaces() { assert_eq!(entry.attempted_at, Some(failed_at)); } +/// The adapter is armed before the invocation spawns anything, which is +/// what makes the refusal above a refusal to serve: `run_daemon` cannot +/// reach the accept task with a preparation it could not complete. The +/// same guarantee `the_registrar_service_is_resolved_before_anything_is_spawned` +/// makes about the handler, asserted the same way, because the ordering +/// lives in one function and nothing else can observe it. +#[test] +fn the_adapter_is_armed_before_anything_is_spawned() { + let source = include_str!("../daemon.rs"); + let body = source + .split("pub(crate) async fn run_daemon") + .nth(1) + .expect("run_daemon is in that file"); + let armed_at = body + .find("prepare_registrar_cert_renewal(") + .expect("run_daemon arms the renewal adapter"); + let first_spawn = body + .find("tokio::spawn") + .expect("run_daemon spawns something"); + assert!( + armed_at < first_spawn, + "renewal is armed before anything is spawned, so an endpoint whose adapter could not \ + be prepared never reaches the accept task" + ); +} + +// --------------------------------------------------------------------- +// The cadence +// --------------------------------------------------------------------- + +/// Writes the rendered internal agent config the cadence is read off, +/// with `extra` appended to its sole profile. +fn write_internal_agent_config(secrets_dir: &Path, extra: &str) { + let paths = InternalPaths::new(secrets_dir); + std::fs::create_dir_all(paths.agent_config().parent().expect("a config directory")) + .expect("create the internal credential directory"); + let base = crate::registrar::internal::render_internal_agent_config( + &paths, + &crate::registrar::internal::InternalAgentConfigParams { + email: "ops@example.internal", + server: "https://127.0.0.1:1/acme/acme/directory", + domain: TEST_DOMAIN, + hostname: TEST_HOST, + responder_url: "http://127.0.0.1:1", + responder_hmac: &"hmac".into(), + eab_kid: None, + eab_hmac: None, + trusted_ca_sha256: &["a".repeat(64)], + }, + ); + std::fs::write(paths.agent_config(), format!("{base}{extra}")) + .expect("write the internal agent config"); +} + +/// The interval, the jitter and the lead time come off the sole +/// profile, and the retry budget is the daemon's own selection: a +/// `[profiles.retry]` there wins over the config's top-level `[retry]`, +/// exactly as it does for a `[[profiles]]` entry the daemon renews. +#[test] +fn the_cadence_takes_the_profiles_own_retry_budget() { + let dir = TempDir::new().expect("tempdir"); + write_internal_agent_config( + dir.path(), + "\n[profiles.daemon]\ncheck_interval = \"2h\"\ncheck_jitter = \"30s\"\n\ + renew_before = \"24h\"\n\n[profiles.retry]\nbackoff_secs = [7, 11]\n", + ); + + let cadence = RenewalCadence::from_internal_config(dir.path()) + .expect("the rendered internal config loads"); + + assert_eq!( + cadence.retry_backoff, + vec![7, 11], + "the profile's own retry budget governs registrar issuance" + ); + assert_eq!(cadence.check_interval, Duration::from_hours(2)); + assert_eq!(cadence.check_jitter, Duration::from_secs(30)); + assert_eq!(cadence.renew_before, Duration::from_hours(24)); +} + +/// Without one, the top-level `[retry]` is the budget — the same +/// fallback the daemon applies, and no second policy of this module's +/// own. +#[test] +fn the_cadence_falls_back_to_the_top_level_retry_budget() { + let dir = TempDir::new().expect("tempdir"); + write_internal_agent_config(dir.path(), ""); + + let cadence = RenewalCadence::from_internal_config(dir.path()) + .expect("the rendered internal config loads"); + + assert_eq!( + cadence.retry_backoff, + vec![5, 15, 60], + "the rendered config's own [retry] table is the fallback" + ); +} + // --------------------------------------------------------------------- // Eligibility // --------------------------------------------------------------------- From baf29b2d21f0f9e94cc7dddc9c4dbf5c57a0f2e0 Mon Sep 17 00:00:00 2001 From: sehkone Date: Sat, 29 Aug 2026 18:24:11 +0900 Subject: [PATCH 4/5] Serialize every in-process writer of the CA bundle The merged CA bundle at [trust] ca_bundle_path has three writers inside one daemon: each profile's publication reads it, merges its issued chain in and writes it back, the fast-poll trust apply replaces it wholesale, and the registrar surface's renewal stages a merge, snapshots it, publishes it and restores it when the publication that follows fails. Nothing serialized them. ProfileLocks cannot: its keys are profile labels, and this file has no profile. So a merge computed from bytes another writer had already replaced overwrote that writer's anchor, and one landing after the registrar transaction's snapshot was discarded by that transaction's rollback. Either way the host went on serving and validating against a bundle missing a CA it had been told to trust, with nothing recording that it went missing. Up to three profiles issue concurrently by default, so the window is not a corner case. All three writers now take one lock per bundle path across the whole read-merge-write span. The lock is a static keyed on the configured path rather than something a caller is handed: the per-profile publication is reached from the CLI as well as from the daemon loop, several frames below anything that knows a daemon exists, and threading a registry through all of them would put the lock's correctness in the hands of every future call site. It is deliberately not taken inside fs_util::write_ca_bundle, which the holding transactions call. Part of #768 --- CHANGELOG.md | 14 ++ docs/en/concepts.md | 8 + docs/ko/concepts.md | 8 + src/acme/flow.rs | 10 +- src/ca_bundle_lock.rs | 138 ++++++++++++++++ src/fast_poll.rs | 74 ++++++++- src/fs_util.rs | 6 + src/lib.rs | 1 + src/registrar_renewal.rs | 16 ++ src/registrar_renewal/tests.rs | 291 +++++++++++++++++++++++++++++++++ 10 files changed, 561 insertions(+), 5 deletions(-) create mode 100644 src/ca_bundle_lock.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 05367078..e88c26c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -384,6 +384,20 @@ byte for byte. ### Fixed +- Two of the daemon's own loops writing the CA bundle at the same moment + can no longer lose one of them. Up to three profiles issue + concurrently by default, and each publication reads + `[trust] ca_bundle_path`, merges its issued chain into it and writes + the result back; the fast-poll loop replaces the same file wholesale + when a trust update arrives. Nothing serialised them, so a merge + computed from bytes another writer had already replaced overwrote that + writer's anchor, leaving the host serving and validating against a + bundle missing a CA it had been told to trust, with nothing recording + that it went missing. Every in-process writer now holds one lock per + bundle path across its whole read-merge-write span. Hosts are + otherwise unaffected: the file's contents, mode and owner are what + they were, and a writer waits only for another writer of the same + bundle. - Prometheus can now scrape step-ca. The bundled monitoring stack has always declared a `step-ca` scrape target, but step-ca serves metrics only when its `ca.json` carries a top-level `metricsAddress`, nothing diff --git a/docs/en/concepts.md b/docs/en/concepts.md index a6f86d0e..c398bb9e 100644 --- a/docs/en/concepts.md +++ b/docs/en/concepts.md @@ -72,6 +72,14 @@ root that consumers need to terminate the chain — overwriting with the chain alone would strip the root and break default-config TLS clients (`Node tls.connect`, `openssl verify` without `-partial_chain`). +The bundle is shared by everything in the daemon that writes it — every +profile's renewal, the fast-poll trust apply, and the registrar surface's +own renewal — so each of them holds it exclusively for the whole +read-merge-write, and one writer's anchors can never be overwritten by +another's merge of bytes it had already replaced. A writer waits only for +another writer of the same bundle; readers are never blocked, because +every write is published by an atomic rename. + ## CSR (Certificate Signing Request) A **CSR** is generated by the party that wants to obtain a certificate from the diff --git a/docs/ko/concepts.md b/docs/ko/concepts.md index 3b60fe71..e24dca63 100644 --- a/docs/ko/concepts.md +++ b/docs/ko/concepts.md @@ -65,6 +65,14 @@ bootroot-agent는 PEM 블록을 순서대로 파싱해 **첫 블록은 리프** `-partial_chain` 없는 `openssl verify`)가 체인을 끝까지 검증할 수 있습니다. 체인만으로 덮어쓰면 루트가 사라져 검증이 실패합니다. +이 번들은 데몬 안에서 이를 기록하는 모든 주체가 공유합니다. 각 프로파일의 +갱신, fast-poll 신뢰 갱신 적용, 레지스트라 표면 자체의 갱신이 모두 +같은 파일을 씁니다. 그래서 각 주체는 읽기-병합-쓰기 전 구간 동안 번들을 +배타적으로 점유하며, 이미 다른 주체가 교체한 바이트를 기준으로 계산된 +병합이 그 주체의 신뢰 앵커를 덮어쓰는 일은 발생하지 않습니다. 대기는 같은 +번들을 기록하는 다른 주체에 대해서만 발생하고, 모든 쓰기는 원자적 +이름 바꾸기로 게시되므로 읽는 쪽은 대기하지 않습니다. + ## CSR (Certificate Signing Request) CSR은 CA로부터 인증서를 발급받으려는 주체가 생성하는 요청서입니다. diff --git a/src/acme/flow.rs b/src/acme/flow.rs index 225537bd..c0d45543 100644 --- a/src/acme/flow.rs +++ b/src/acme/flow.rs @@ -263,12 +263,20 @@ pub(crate) fn merge_ca_bundle( /// intermediate-only state this PR hardens against would silently /// reappear whenever mode/ACL/ownership drift makes the file /// unreadable but still writeable. -async fn write_merged_ca_bundle( +/// +/// The read and the write are one transaction, held under +/// [`crate::ca_bundle_lock`] for the whole span. The bundle is shared +/// with the registrar surface's renewal and with the fast-poll trust +/// apply, and a merge computed from bytes another writer has already +/// replaced would publish a bundle missing an anchor this host was told +/// to trust. +pub(crate) async fn write_merged_ca_bundle( bundle_path: &Path, chain: &[Vec], trusted: &[String], policy: CertGroupPolicy, ) -> Result<()> { + let _bundle = crate::ca_bundle_lock::hold(bundle_path).await; let existing = match tokio::fs::read(bundle_path).await { Ok(bytes) => Some(bytes), Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, diff --git a/src/ca_bundle_lock.rs b/src/ca_bundle_lock.rs new file mode 100644 index 00000000..c7514dd2 --- /dev/null +++ b/src/ca_bundle_lock.rs @@ -0,0 +1,138 @@ +//! The one lock every in-process writer of `[trust] ca_bundle_path` +//! takes. +//! +//! The merged CA bundle is not a per-profile file. The per-profile +//! renewal loop merges its issued chain into it +//! (`acme::flow::write_merged_ca_bundle`), the fast-poll loop +//! replaces it wholesale when a trust update lands, and the registrar +//! surface's renewal reads it, stages a merge, snapshots it, publishes +//! it and — when the publication fails afterwards — restores it. All +//! three run as independent tasks of one daemon, and the daemon's +//! `ProfileLocks` serialises none of them against each other: its keys +//! are profile labels, and this file has no profile. +//! +//! Unserialised, the read-merge-write pairs interleave and lose writes. +//! A profile merge landing after the registrar's snapshot but before its +//! rollback is discarded by that rollback, which puts back bytes that +//! were already stale; a profile merge landing after the registrar's +//! staged read but before its publication is overwritten by it. Either +//! way the daemon ends up serving a bundle missing an anchor it was told +//! to trust, with nothing recording that it went missing. +//! +//! So each of those transactions holds this lock for its whole span — +//! from the read the merge is computed against through the last write or +//! restore that span can perform — and the file changes under none of +//! them. +//! +//! # What it is not +//! +//! It is process-wide, and only process-wide. A second `bootroot` +//! invocation writing the same bundle is outside its reach entirely; +//! that is what the atomic rename in [`crate::fs_util::write_ca_bundle`] +//! is for, and the two answer different questions. +//! +//! The lock is deliberately *not* taken inside +//! [`crate::fs_util::write_ca_bundle`]. A transaction that has already +//! taken it goes on to call that writer, and a second acquisition there +//! would deadlock it against itself. Ownership belongs to the +//! transaction, which is the only layer that knows where its critical +//! section begins. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, LazyLock, Mutex as StdMutex, PoisonError}; + +use tokio::sync::{Mutex as TokioMutex, OwnedMutexGuard}; + +/// The one bundle-path lock map the whole process shares. +/// +/// A `static` rather than something a caller is handed, because the +/// writers have no common owner to be handed it by: the per-profile +/// publication is reached from the daemon loop and from the CLI's +/// issuance alike, several call frames below anything that knows a +/// daemon exists. Threading a registry down all of them would put the +/// correctness of the lock in the hands of every future call site, +/// where forgetting the argument compiles. +static BUNDLE_LOCKS: LazyLock = LazyLock::new(BundleLocks::default); + +/// Exclusive access to one bundle path, released when dropped. +pub(crate) type BundleGuard = OwnedMutexGuard<()>; + +/// Takes `bundle_path`'s lock, waiting for whichever transaction holds +/// it to finish. +/// +/// Hold the guard across the whole transaction — the read the merge is +/// computed from, every live write, and any rollback — and not merely +/// across the write itself. A guard scoped to the write alone still +/// lets another writer land between this one's read and its write. +pub(crate) async fn hold(bundle_path: &Path) -> BundleGuard { + BUNDLE_LOCKS.for_path(bundle_path).lock_owned().await +} + +/// The path-keyed registry behind [`hold`]. +#[derive(Default)] +struct BundleLocks { + entries: StdMutex>>>, +} + +impl BundleLocks { + /// Returns `bundle_path`'s mutex, creating it on first access. + /// + /// Keyed on the configured path as written, exactly as the daemon's + /// `ProfileLocks` keys on the configured profile label: every writer + /// reads it from the same + /// `[trust] ca_bundle_path` of the same `Settings`, so the keys + /// coincide by construction. Entries are never removed — a daemon + /// has one bundle path and a reload can only ever introduce another + /// one — so a lock is never dropped while a waiter is queued on it. + fn for_path(&self, bundle_path: &Path) -> Arc> { + let mut entries = self + .entries + .lock() + // A lookup and an insert cannot leave the map torn, so an + // unrelated panic must not make the bundle permanently + // unwritable. + .unwrap_or_else(PoisonError::into_inner); + Arc::clone( + entries + .entry(bundle_path.to_path_buf()) + .or_insert_with(|| Arc::new(TokioMutex::new(()))), + ) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::hold; + + /// One path hands out one lock, so a second acquisition of it waits + /// while a different path does not. + #[tokio::test] + async fn one_path_is_one_lock_and_two_paths_are_two() { + let first = PathBuf::from("/tmp/bootroot-ca-bundle-lock-test/first.pem"); + let second = PathBuf::from("/tmp/bootroot-ca-bundle-lock-test/second.pem"); + let held = hold(&first).await; + + let other = tokio::spawn({ + let second = second.clone(); + async move { hold(&second).await } + }); + other + .await + .expect("a different path is a different lock and does not wait"); + + let same = tokio::spawn({ + let first = first.clone(); + async move { hold(&first).await } + }); + tokio::task::yield_now().await; + assert!( + !same.is_finished(), + "a second acquisition of one path waits for the guard to drop" + ); + drop(held); + same.await.expect("the queued acquisition proceeds"); + } +} diff --git a/src/fast_poll.rs b/src/fast_poll.rs index 96e816f8..4a9cbcbd 100644 --- a/src/fast_poll.rs +++ b/src/fast_poll.rs @@ -1207,7 +1207,9 @@ pub(crate) struct LiveFastPollHooks { /// /// The CA bundle goes through [`fs_util::write_ca_bundle`] (`0o644` + /// cert-group gid policy) so it stays world-readable and cert-group -/// consistent like the renewal-time bundle; the `agent.toml` rewrite goes +/// consistent like the renewal-time bundle, under +/// [`crate::ca_bundle_lock`] so it cannot land inside another in-process +/// writer's read-merge-write span; the `agent.toml` rewrite goes /// through [`fs_util::atomic_write`] at `0o600` (the #613-safe writer) so a /// crash cannot leave the daemon's hot-read config half-written. The /// `upsert_section_keys` round-trip preserves operator-tuned sections. @@ -1223,9 +1225,21 @@ async fn apply_trust_to_disk( ) })?; let policy = resolve_service_cert_group_policy(settings, service)?; - fs_util::write_ca_bundle(ca_bundle_path, &payload.ca_bundle_pem, policy) - .await - .with_context(|| format!("Failed to write CA bundle to {}", ca_bundle_path.display()))?; + // The bundle is shared with the per-profile renewal and with the + // registrar surface's renewal transaction, and this write replaces + // it wholesale. Landing inside one of their read-merge-write spans + // would either be discarded by a rollback or overwrite the merge + // that follows it, so it waits for whichever holds the file. The + // guard is scoped to this write alone: the `agent.toml` rewrite + // below is not the bundle's, and no other writer touches it. + { + let _bundle = crate::ca_bundle_lock::hold(ca_bundle_path).await; + fs_util::write_ca_bundle(ca_bundle_path, &payload.ca_bundle_pem, policy) + .await + .with_context(|| { + format!("Failed to write CA bundle to {}", ca_bundle_path.display()) + })?; + } let current = fs::read_to_string(config_path) .await @@ -3581,6 +3595,58 @@ mod tests { ); } + /// The fast-poll trust apply replaces the shared bundle wholesale, + /// so it waits for whichever in-process writer holds it rather than + /// landing inside that writer's read-merge-write span. + #[tokio::test] + async fn apply_trust_to_disk_waits_for_the_shared_bundle_lock() { + let dir = tempfile::tempdir().unwrap(); + let bundle_path = dir.path().join("ca-bundle.pem"); + let config_path = dir.path().join("agent.toml"); + std::fs::write(&config_path, "email = \"a@b.c\"\n").unwrap(); + std::fs::write(&bundle_path, "ORIGINAL\n").unwrap(); + + let payload = TrustPayload { + trusted_ca_sha256: vec!["a".repeat(64)], + ca_bundle_pem: "-----BEGIN CERTIFICATE-----\npem\n-----END CERTIFICATE-----\n" + .to_string(), + }; + + let held = crate::ca_bundle_lock::hold(&bundle_path).await; + let apply = tokio::spawn({ + let bundle_path = bundle_path.clone(); + async move { + let mut settings = test_settings("edge-proxy"); + settings.trust.ca_bundle_path = Some(bundle_path); + apply_trust_to_disk(&settings, &config_path, "edge-proxy", &payload).await + } + }); + // The task is parked on the mutex and cannot finish while the + // guard is alive; this only gives it the chance to get there. + for _ in 0..16 { + tokio::task::yield_now().await; + } + + assert!( + !apply.is_finished(), + "the trust apply waits for whichever writer holds the shared bundle" + ); + assert_eq!( + std::fs::read_to_string(&bundle_path).unwrap(), + "ORIGINAL\n", + "the bundle is not replaced while another writer holds it" + ); + + drop(held); + apply.await.unwrap().expect("apply trust"); + assert!( + std::fs::read_to_string(&bundle_path) + .unwrap() + .contains("BEGIN CERTIFICATE"), + "the trust apply lands once the bundle is free" + ); + } + #[tokio::test] async fn apply_trust_to_disk_writes_bundle_and_upserts_trust_section() { use std::os::unix::fs::PermissionsExt; diff --git a/src/fs_util.rs b/src/fs_util.rs index 7f18f399..7005e7c7 100644 --- a/src/fs_util.rs +++ b/src/fs_util.rs @@ -1638,6 +1638,12 @@ pub async fn write_cert_and_key( /// store while a rotation may be rewriting it, so the destination name /// must never hold a truncated chain. /// +/// This is one write and takes no lock. Serialising it against the +/// other in-process writers of the same bundle is the caller's, because +/// only the caller knows where its read-merge-write span begins; the +/// lock to take is `crate::ca_bundle_lock`, and taking it here as well +/// would deadlock every caller that already holds it. +/// /// # Errors /// Returns an error if the directory cannot be created, the bundle /// cannot be written, or the mode/owner cannot be applied. diff --git a/src/lib.rs b/src/lib.rs index 30c4f14f..f2e9d404 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ use std::sync::Arc; pub mod acme; pub mod agent_args; +pub(crate) mod ca_bundle_lock; pub mod cert_chain; pub mod cert_group; pub mod config; diff --git a/src/registrar_renewal.rs b/src/registrar_renewal.rs index cfb20700..ac445a2e 100644 --- a/src/registrar_renewal.rs +++ b/src/registrar_renewal.rs @@ -1092,6 +1092,15 @@ impl RegistrarCertRenewal { /// Validates the candidate, builds the whole replacement, and /// publishes it as a recoverable transaction. + /// + /// Everything the merged CA bundle is involved in — the read the + /// staged bytes are computed from, the snapshot, the write and the + /// restore a failure performs — happens under + /// [`crate::ca_bundle_lock`], because that file is shared with the + /// per-profile renewal loop and the fast-poll trust apply. Without + /// it a profile merge landing after this snapshot is thrown away by + /// this rollback, and one landing after this staged read is + /// overwritten by this publication. async fn publish_candidate( &self, pair: &SurfacePairPaths, @@ -1119,6 +1128,13 @@ impl RegistrarCertRenewal { set to be rebuilt from and no registrar leaf can be published" ) })?; + // Taken before the read the merge is computed from and held + // past the last restore a failure can perform, so no other + // writer can move the file inside the transaction. The + // candidate is validated above it: that reads the pin file and + // the plan, never the bundle, and refusing an unpinned + // candidate must not wait on another writer's publication. + let _bundle = crate::ca_bundle_lock::hold(bundle_path).await; let staged_bundle = self.stage_bundle(bundle_path, &material.chain)?; // The candidate on disk, at the modes the finished files diff --git a/src/registrar_renewal/tests.rs b/src/registrar_renewal/tests.rs index bd859b52..82aafe10 100644 --- a/src/registrar_renewal/tests.rs +++ b/src/registrar_renewal/tests.rs @@ -1555,3 +1555,294 @@ async fn the_next_handshake_uses_the_renewed_material_without_a_restart() { ); drop(held); } + +// --------------------------------------------------------------------- +// The shared CA bundle +// --------------------------------------------------------------------- + +/// The production writer with the pair write held open on a gate. +/// +/// `write_pair` announces that the transaction has passed its snapshots +/// and its bundle write, waits to be released, and then fails. That is +/// exactly the window another writer's bundle write must not land in: +/// the rollback that follows would put back bytes taken before it and +/// discard it. +struct GatedPairWrite { + inner: FilesystemPaths, + reached: Mutex>>, + release: Mutex>>, +} + +impl GatedPairWrite { + /// Not `Self`: the seam is a `Box`. + #[allow(clippy::new_ret_no_self)] + fn new( + reached: tokio::sync::oneshot::Sender<()>, + release: tokio::sync::oneshot::Receiver<()>, + ) -> Box { + Box::new(Self { + inner: FilesystemPaths::new(), + reached: Mutex::new(Some(reached)), + release: Mutex::new(Some(release)), + }) + } + + /// Takes whichever half the gate still holds, without a guard alive + /// across the await that follows. + fn take(slot: &Mutex>) -> Option { + slot.lock().unwrap_or_else(PoisonError::into_inner).take() + } +} + +impl LivePaths for GatedPairWrite { + fn write_bundle<'a>(&'a self, path: &'a Path, contents: &'a str) -> LiveWrite<'a> { + self.inner.write_bundle(path, contents) + } + + fn write_pair<'a>( + &'a self, + _cert_path: &'a Path, + _key_path: &'a Path, + _cert_pem: &'a str, + _key_pem: &'a str, + ) -> LiveWrite<'a> { + Box::pin(async move { + if let Some(reached) = Self::take(&self.reached) { + let _ = reached.send(()); + } + if let Some(release) = Self::take(&self.release) { + let _ = release.await; + } + anyhow::bail!("injected certificate and key write failure") + }) + } + + fn restore<'a>(&'a self, snapshot: &'a Snapshot) -> LiveWrite<'a> { + self.inner.restore(snapshot) + } +} + +/// Every certificate fingerprint the bundle at `path` currently holds. +fn bundle_fingerprints(path: &Path) -> Vec { + let bytes = std::fs::read(path).expect("read the bundle"); + x509_parser::pem::Pem::iter_from_buffer(&bytes) + .filter_map(Result::ok) + .filter(|pem| pem.label == "CERTIFICATE") + .map(|pem| crate::tls::sha256_hex(&pem.contents)) + .collect() +} + +/// The harness's settings with `extra` pinned as well, so a second +/// writer's chain survives the merge and is visible afterwards. +fn also_trusting(harness: &Harness, extra: &TestCa) -> Arc { + let mut settings = (*harness.settings).clone(); + settings.trust.trusted_ca_sha256.push(extra.fingerprint()); + Arc::new(settings) +} + +/// The publication takes the shared bundle lock before it reads the +/// bundle, so it writes nothing at all while another in-process writer +/// holds it — and proceeds once that writer is done. +#[tokio::test] +async fn the_registrar_publication_waits_for_the_shared_bundle_lock() { + let harness = Harness::build(); + let renewal = harness.renewal().await; + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let cert_path = pair.cert_path.clone(); + let before_cert = digest_of(&cert_path); + let before_bundle = digest_of(&harness.bundle_path()); + let material = harness.ca.material(&pair.name, -1, 60); + + let held = crate::ca_bundle_lock::hold(&harness.bundle_path()).await; + let publication = tokio::spawn(async move { + let artifacts = Artifacts::create(&pair.key_path).expect("artifacts"); + let outcome = renewal + .publish_candidate(&pair, &material, &artifacts) + .await; + artifacts.close().expect("artifacts are removed"); + outcome + }); + // Not a wait on anything real: the task is parked on the mutex and + // can never finish while the guard is alive, so this only gives it + // the chance to get there. + for _ in 0..16 { + tokio::task::yield_now().await; + } + + assert!( + !publication.is_finished(), + "the publication waits for whichever writer holds the shared bundle" + ); + assert_eq!( + before_bundle, + digest_of(&harness.bundle_path()), + "nothing is written to the bundle while another writer holds it" + ); + assert_eq!( + before_cert, + digest_of(&cert_path), + "no live path is written before the transaction owns the bundle" + ); + + drop(held); + publication + .await + .expect("the publication task") + .expect("the candidate publishes once the bundle is free"); + assert_ne!( + before_cert, + digest_of(&cert_path), + "the publication proceeds as soon as the lock is released" + ); +} + +/// The per-profile publication takes the same lock, so its merge is +/// computed against bytes no other writer can replace underneath it. +#[tokio::test] +async fn the_profile_bundle_writer_waits_for_the_shared_bundle_lock() { + let harness = Harness::build(); + let other = TestCa::new("Bootroot Second Anchor CA"); + let settings = also_trusting(&harness, &other); + let bundle_path = harness.bundle_path(); + let before = digest_of(&bundle_path); + + let held = crate::ca_bundle_lock::hold(&bundle_path).await; + let merge = tokio::spawn({ + let bundle_path = bundle_path.clone(); + let trusted = settings.trust.trusted_ca_sha256.clone(); + let chain = vec![other.der()]; + async move { + crate::acme::flow::write_merged_ca_bundle( + &bundle_path, + &chain, + &trusted, + CertGroupPolicy::none(), + ) + .await + } + }); + for _ in 0..16 { + tokio::task::yield_now().await; + } + + assert!( + !merge.is_finished(), + "the per-profile merge waits for whichever writer holds the bundle" + ); + assert_eq!( + before, + digest_of(&bundle_path), + "the merge reads nothing and writes nothing while the bundle is held" + ); + + drop(held); + merge + .await + .expect("the merge task") + .expect("the merge lands once the bundle is free"); + assert!( + bundle_fingerprints(&bundle_path).contains(&other.fingerprint()), + "the merge that waited still adds its anchor" + ); +} + +/// A per-profile merge that arrives while the registrar transaction is +/// mid-publication is not lost by that transaction's rollback. +/// +/// The lock is what makes the assertion hold whichever way the two +/// tasks are scheduled: the merge can only run entirely before the +/// transaction takes the bundle — in which case the snapshot the +/// rollback restores already contains its anchor — or entirely after +/// the rollback released it. Unserialised, the merge lands between the +/// snapshot and the rollback, and the rollback puts back a bundle that +/// never had the anchor in it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_interleaved_profile_merge_survives_a_registrar_rollback() { + let harness = Harness::build(); + let other = TestCa::new("Bootroot Second Anchor CA"); + let settings = also_trusting(&harness, &other); + let bundle_path = harness.bundle_path(); + + let (reached_tx, reached_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let renewal = RegistrarCertRenewal::for_test( + Arc::clone(&settings), + harness.plan.clone(), + Arc::clone(&harness.endpoint), + cadence(), + ) + .await + .with_live_paths(GatedPairWrite::new(reached_tx, release_rx)); + + let pair = harness.pair(SurfaceLeaf::EndpointServer); + let cert_path = pair.cert_path.clone(); + let before_cert = digest_of(&cert_path); + let material = harness.ca.material(&pair.name, -1, 60); + let publication = tokio::spawn(async move { + let artifacts = Artifacts::create(&pair.key_path).expect("artifacts"); + let outcome = renewal + .publish_candidate(&pair, &material, &artifacts) + .await; + artifacts.close().expect("artifacts are removed"); + outcome + }); + + reached_rx + .await + .expect("the transaction reaches its pair write"); + let merge = tokio::spawn({ + let bundle_path = bundle_path.clone(); + let trusted = settings.trust.trusted_ca_sha256.clone(); + let chain = vec![other.der()]; + async move { + crate::acme::flow::write_merged_ca_bundle( + &bundle_path, + &chain, + &trusted, + CertGroupPolicy::none(), + ) + .await + } + }); + // Awaits the condition rather than a clock: the merge either + // finishes — which is what it does when nothing serialises the two, + // and is the interleaving this test exists to reject — or parks on + // a bundle it cannot have until the transaction is done with it. + // The cap is only ever reached in the second case, where it costs a + // few thousand yields to a runtime with nothing else ready. + for _ in 0..10_000 { + if merge.is_finished() { + break; + } + tokio::task::yield_now().await; + } + assert!( + !merge.is_finished(), + "the merge must not reach the bundle between the transaction's snapshot and its rollback" + ); + release_tx.send(()).expect("the transaction is released"); + + let err = publication + .await + .expect("the publication task") + .expect_err("the injected pair write fails the publication"); + merge + .await + .expect("the merge task") + .expect("the merge lands either side of the transaction"); + + let fingerprints = bundle_fingerprints(&bundle_path); + assert!( + fingerprints.contains(&other.fingerprint()), + "a concurrent merge is never discarded by the rollback: {err:#}" + ); + assert!( + fingerprints.contains(&harness.ca.fingerprint()), + "the rollback still restores the anchor the transaction snapshotted" + ); + assert_eq!( + before_cert, + digest_of(&cert_path), + "the failed publication leaves the live pair as it was" + ); +} From b748619ec2215dcb2c18dea07e5a8d34fa5174f9 Mon Sep 17 00:00:00 2001 From: sehkone Date: Sat, 29 Aug 2026 18:54:15 +0900 Subject: [PATCH 5/5] Prove the swap over a served endpoint The reload contract's server half was asserted off the disk: the test opened an unserved raw socket and compared certificate files, so a publication that renewed the live material and never exchanged the active configuration would have passed it. That is the one failure the whole transaction exists to prevent. Both cases now run the production accept loop over the harness endpoint and read the answer off the wire. One holds a connection handshaken before the swap open across it, publishes a server candidate, and asserts the next handshake is presented the renewed leaf while the older connection finishes under the configuration it handshook with. The other dials with a caller under a rotated anchor, watches the live verifier refuse it, publishes the client leaf that carries that anchor, and dials again -- the same material, now admitted by the verifier the publication rebuilt. Removing the swap fails both: the first on the presented leaf, the second on an UnknownCA alert. Part of #768 --- src/registrar_renewal/tests.rs | 320 ++++++++++++++++++++++++++++++--- 1 file changed, 292 insertions(+), 28 deletions(-) diff --git a/src/registrar_renewal/tests.rs b/src/registrar_renewal/tests.rs index 82aafe10..07c89bc7 100644 --- a/src/registrar_renewal/tests.rs +++ b/src/registrar_renewal/tests.rs @@ -19,7 +19,11 @@ use rcgen::{ BasicConstraints, CertificateParams, CertifiedIssuer, DnType, IsCa, KeyPair, KeyUsagePurpose, SanType, }; +use rustls::ClientConfig; +use rustls::pki_types::ServerName; use tempfile::TempDir; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio_rustls::TlsConnector; use super::*; use crate::acme::IssuedMaterial; @@ -28,6 +32,7 @@ use crate::registrar::endpoint; use crate::registrar::endpoint::activation::ActivationContract; use crate::registrar::endpoint_pin::REGISTRAR_ENDPOINT_ANCHORS_FILE; use crate::registrar::internal::PrivateKeyPem; +use crate::registrar::verbs::outcome::CallerIdentity; const TEST_DOMAIN: &str = "corp.example.internal"; const TEST_HOST: &str = "bootroot-01"; @@ -1499,31 +1504,204 @@ async fn a_stop_before_the_first_pass_runs_none() { } // --------------------------------------------------------------------- -// The reload contract, end to end over the socket +// The reload contract, end to end over a served socket // --------------------------------------------------------------------- -/// The renewed server leaf is presented, and a caller under the rotated -/// anchor is accepted, from the next handshake onwards — with no -/// restart, no signal, the same socket inode, and a connection already -/// in flight left alone. +/// The handler the served endpoint dispatches to, which answers with +/// the caller identity the transport authenticated. +/// +/// The payload is never decoded: what these tests drive is the TLS +/// material, not the protocol. Answering with the identity is what +/// makes acceptance assertable — those bytes are written only after the +/// incoming verifier accepted the caller's chain and the accept loop +/// recognized its SAN, so a round trip that returns them is proof that +/// both halves of the active configuration admitted the caller. +struct EchoingCallerIdentity; + +impl endpoint::handler::RegistrarRequestHandler for EchoingCallerIdentity { + fn handle<'a>( + &'a self, + _operation: endpoint::frame::Operation, + _payload: &'a [u8], + caller: CallerIdentity, + ) -> Pin, endpoint::handler::HandlerRefusal>> + Send + 'a>> + { + Box::pin(async move { Ok(caller.as_str().as_bytes().to_vec()) }) + } +} + +/// The harness endpoint with the production accept loop running over +/// it. +/// +/// The loop is the one the daemon spawns, over the same +/// [`ActivatedEndpoint`] the renewal adapter holds, so a swap performed +/// by a publication is observed exactly where a real caller would +/// observe it: on the next handshake. +struct Serving { + shutdown: watch::Sender, + handle: tokio::task::JoinHandle>, +} + +impl Serving { + fn start(harness: &Harness) -> Self { + let (shutdown, receiver) = watch::channel(false); + let endpoint = Arc::clone(&harness.endpoint); + let handle = tokio::spawn(async move { + endpoint::serve::run(endpoint, Arc::new(EchoingCallerIdentity), receiver).await + }); + Self { shutdown, handle } + } + + async fn stop(self) { + let _ = self.shutdown.send(true); + self.handle + .await + .expect("the accept task joins") + .expect("the accept loop ends cleanly"); + } +} + +/// A caller's TLS stream over the endpoint's `AF_UNIX` socket. +type CallerStream = tokio_rustls::client::TlsStream; + +/// The pinned, authenticating configuration one dial builds from the +/// pair at `certificate_path` and `key_path`. +/// +/// Composed by [`endpoint::client::build_client_config`], the one place +/// in this tree the registrar caller's TLS configuration is built, over +/// material read from the paths a publication writes to. A dial after a +/// renewal therefore presents whatever the renewal published, with no +/// caller state carried across. +fn dial_config(harness: &Harness, certificate_path: &Path, key_path: &Path) -> ClientConfig { + let chain = crate::tls::parse_pem_to_cert_list( + &std::fs::read(certificate_path).expect("read the caller's chain"), + ) + .expect("the caller's chain parses"); + let key_bytes = std::fs::read(key_path).expect("read the caller's key"); + let key = rustls_pemfile::private_key(&mut std::io::BufReader::new(key_bytes.as_slice())) + .expect("the caller's key parses") + .expect("the caller's key file holds a key"); + endpoint::client::build_client_config( + &harness.pin_file(), + &harness.pair(SurfaceLeaf::EndpointServer).name, + chain, + key, + ) + .expect("a pinned, authenticating caller") +} + +/// Connects and completes the handshake, leaving the request unsent. +/// +/// The dial name is a placeholder: over `AF_UNIX` there is no +/// meaningful server name, and the pinned verifier decides on the +/// presented leaf's SAN instead of on this. +async fn dial(socket_path: &Path, config: ClientConfig) -> std::io::Result { + let stream = tokio::net::UnixStream::connect(socket_path).await?; + TlsConnector::from(Arc::new(config)) + .connect( + ServerName::try_from("localhost").expect("a valid dial name"), + stream, + ) + .await +} + +/// The end-entity certificate the endpoint presented on this stream, +/// as its DER fingerprint. +/// +/// A fingerprint rather than the DER itself so that a failure names two +/// digests instead of printing two certificates at each other. +fn presented_leaf(stream: &CallerStream) -> String { + stream + .get_ref() + .1 + .peer_certificates() + .and_then(<[_]>::first) + .map(|leaf| crate::tls::sha256_hex(leaf.as_ref())) + .expect("the endpoint presents a leaf on every completed handshake") +} + +/// The DER fingerprint of the first certificate in `pem`. +fn leaf_fingerprint(pem: &str) -> String { + crate::tls::sha256_hex(&pem_to_der(pem)) +} + +/// Sends one request over an already-handshaken stream and reads the +/// answer. +/// +/// In TLS 1.3 the client finishes its own handshake before the server +/// has validated the client certificate, so a refused caller learns of +/// the refusal here rather than at [`dial`]. Acceptance is therefore +/// asserted through a round trip and never through a connect. +async fn round_trip(stream: &mut CallerStream) -> std::io::Result> { + let name = endpoint::frame::Operation::Mint.as_str(); + let mut request = 0u32.to_be_bytes().to_vec(); + request.push(u8::try_from(name.len()).expect("a short operation name")); + request.extend_from_slice(name.as_bytes()); + stream.write_all(&request).await?; + stream.flush().await?; + + let mut prefix = [0u8; 4]; + stream.read_exact(&mut prefix).await?; + let declared = usize::try_from(u32::from_be_bytes(prefix)).expect("a test answer fits usize"); + let mut body = vec![0u8; declared]; + stream.read_exact(&mut body).await?; + Ok(body) +} + +/// One whole exchange: handshake, one request, one answer. Returns the +/// leaf the endpoint presented alongside what it answered. +async fn exchange(socket_path: &Path, config: ClientConfig) -> std::io::Result<(String, Vec)> { + let mut stream = dial(socket_path, config).await?; + let presented = presented_leaf(&stream); + let answer = round_trip(&mut stream).await?; + Ok((presented, answer)) +} + +/// The identity the accept loop renders for the caller `name`. +fn caller_of(name: &str) -> String { + format!("registrar-client:{}", name.to_ascii_lowercase()) +} + +/// The next handshake over the running endpoint presents the renewed +/// server leaf — same socket inode, no restart, no signal — and a +/// connection that handshook before the swap keeps the configuration it +/// handshook under and is not dropped. +/// +/// This runs the production accept loop and completes real handshakes, +/// so it fails if a publication renews the live files and leaves the +/// active acceptor alone: the certificate the second caller is +/// presented is read off the wire, not off the disk. #[tokio::test] -async fn the_next_handshake_uses_the_renewed_material_without_a_restart() { +async fn the_next_handshake_presents_the_renewed_server_leaf() { let harness = Harness::build(); let renewal = harness.renewal().await; - let pair = harness.pair(SurfaceLeaf::EndpointServer); + let serving = Serving::start(&harness); + let server = harness.pair(SurfaceLeaf::EndpointServer); + let client = harness.pair(SurfaceLeaf::RegistrarClient); + let live_leaf = + leaf_fingerprint(&std::fs::read_to_string(&server.cert_path).expect("read the leaf")); let inode_before = std::fs::metadata(&harness.socket_path) .expect("stat the socket") .ino(); - // A connection established before the swap, held open across it. - let held = tokio::net::UnixStream::connect(&harness.socket_path) - .await - .expect("connect before the swap"); + // Handshaken before the swap and held open across it, with its + // request still unsent. + let mut in_flight = dial( + &harness.socket_path, + dial_config(&harness, &client.cert_path, &client.key_path), + ) + .await + .expect("the endpoint serves the caller it was provisioned for"); + assert_eq!( + presented_leaf(&in_flight), + live_leaf, + "before the renewal the endpoint presents the leaf it was activated with" + ); - let material = harness.ca.material(&pair.name, -1, 60); - let artifacts = Artifacts::create(&pair.key_path).expect("artifacts"); + let material = harness.ca.material(&server.name, -1, 60); + let artifacts = Artifacts::create(&server.key_path).expect("artifacts"); renewal - .publish_candidate(&pair, &material, &artifacts) + .publish_candidate(&server, &material, &artifacts) .await .expect("the candidate publishes"); artifacts.close().expect("artifacts are removed"); @@ -1535,25 +1713,111 @@ async fn the_next_handshake_uses_the_renewed_material_without_a_restart() { .ino(), "a TLS replacement never rebinds or replaces the socket" ); + assert_eq!( + String::from_utf8( + round_trip(&mut in_flight) + .await + .expect("a connection already in flight is not dropped by the swap") + ) + .expect("the caller identity is UTF-8"), + caller_of(&client.name), + "the connection that handshook before the swap finishes under its own configuration" + ); + + let (presented, answer) = exchange( + &harness.socket_path, + dial_config(&harness, &client.cert_path, &client.key_path), + ) + .await + .expect("the next handshake completes without a restart"); + assert_eq!( + presented, + leaf_fingerprint(&material.cert_pem), + "the next handshake is served the renewed leaf, from the acceptor the swap installed" + ); + assert_ne!( + presented, live_leaf, + "the renewed leaf is not the one the endpoint was activated with" + ); + assert_eq!( + String::from_utf8(answer).expect("the caller identity is UTF-8"), + caller_of(&client.name), + "the renewed configuration still admits the caller it was serving" + ); + + serving.stop().await; +} + +/// A caller under a rotated anchor is refused before the client leaf is +/// renewed and accepted after it, over the same running endpoint. +/// +/// The rebuilt incoming verifier is the only thing that changes: the +/// anchor is pinned in configuration from the start and reaches the +/// bundle the verifier is built from only when the publication stages +/// it. The caller that is accepted afterwards is the renewed pair +/// itself, read off the live paths by the same dial-time load a +/// production caller performs. +#[tokio::test] +async fn the_next_handshake_accepts_a_caller_under_the_rebuilt_anchor() { + let mut harness = Harness::build(); + let rotated = TestCa::new("Bootroot Rotated Anchor CA"); + let settings = also_trusting(&harness, &rotated); + harness.settings = settings; + let renewal = harness.renewal().await; + let serving = Serving::start(&harness); + let client = harness.pair(SurfaceLeaf::RegistrarClient); + let server = harness.pair(SurfaceLeaf::EndpointServer); + let live_leaf = + leaf_fingerprint(&std::fs::read_to_string(&server.cert_path).expect("read the leaf")); + + // The renewed client pair, staged where the live one is not yet, so + // the same material can be dialled with before it is published. + let material = rotated.material(&client.name, -1, 60); + let staged_cert = harness.dir.path().join("rotated-client.crt"); + let staged_key = harness.dir.path().join("rotated-client.key"); + std::fs::write(&staged_cert, &material.cert_pem).expect("stage the rotated chain"); + std::fs::write(&staged_key, material.key_pem.expose()).expect("stage the rotated key"); + + let refused = exchange( + &harness.socket_path, + dial_config(&harness, &staged_cert, &staged_key), + ) + .await + .expect_err("the live verifier has no anchor for the rotated generation"); assert!( - held.peer_addr().is_ok(), - "a connection already in flight is not dropped" + format!("{refused:?}").contains("UnknownCA"), + "the endpoint refuses the rotated generation as an issuer its verifier has no anchor for, \ + which is the state the publication has to change: {refused:?}" ); - // The acceptor the next handshake would load presents the renewed - // leaf: the resolver behind it is the one built from the candidate. - let renewed_leaf = std::fs::read_to_string(&pair.cert_path).expect("read the published leaf"); + let artifacts = Artifacts::create(&client.key_path).expect("artifacts"); + renewal + .publish_candidate(&client, &material, &artifacts) + .await + .expect("a conforming client candidate publishes"); + artifacts.close().expect("artifacts are removed"); + + let (presented, answer) = exchange( + &harness.socket_path, + dial_config(&harness, &client.cert_path, &client.key_path), + ) + .await + .expect("the rebuilt verifier admits the renewed caller without a restart"); + assert_eq!( + String::from_utf8(answer).expect("the caller identity is UTF-8"), + caller_of(&client.name), + "the caller refused a moment ago is accepted under the anchor the publication staged" + ); + assert_eq!( + presented, live_leaf, + "renewing the client leaf leaves the server leaf the endpoint presents alone" + ); assert!( - renewed_leaf.contains( - material - .cert_pem - .lines() - .nth(1) - .expect("the leaf PEM has a body line") - ), - "the published certificate is the candidate that was validated" + bundle_fingerprints(&harness.bundle_path()).contains(&rotated.fingerprint()), + "the anchor the verifier was rebuilt from is the one the publication merged" ); - drop(held); + + serving.stop().await; } // ---------------------------------------------------------------------