From faabe891edbf6b7d1850265b082480d8b57ead56 Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 11:22:41 +0000 Subject: [PATCH 01/10] typeck: accept scalar value in stream_store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds check_stream_store separate from check_store. Allows i16/u16/i32/u32/i64/u64 scalar values in addition to vectors, prerequisite for q4k_repack-style scalar streaming writes that cannot use the vector form. Codegen still scalarizes through .into_vector_value() — next commit completes the scalar codegen path. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/typeck/intrinsics.rs | 2 +- src/typeck/intrinsics_memory.rs | 70 +++++++++++++++++++++++++++++++++ tests/stream_store_tests.rs | 38 ++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/typeck/intrinsics.rs b/src/typeck/intrinsics.rs index 594f7e1..8ebcc02 100644 --- a/src/typeck/intrinsics.rs +++ b/src/typeck/intrinsics.rs @@ -184,7 +184,7 @@ impl TypeChecker { "permute_runtime" => Some(self.check_permute_runtime(args, locals, span)), "scatter" => Some(self.check_scatter(args, locals, span)), "load_masked" => Some(self.check_load_masked(args, locals, type_hint, span)), - "stream_store" => Some(self.check_store(args, locals, span)), + "stream_store" => Some(self.check_stream_store(args, locals, span)), "store_masked" => Some(self.check_store_masked(args, locals, span)), "movemask" => Some(self.check_movemask(args, locals, span)), "min" | "max" => Some(self.check_min_max(name, args, locals, span)), diff --git a/src/typeck/intrinsics_memory.rs b/src/typeck/intrinsics_memory.rs index d60cbae..e3e8541 100644 --- a/src/typeck/intrinsics_memory.rs +++ b/src/typeck/intrinsics_memory.rs @@ -106,6 +106,76 @@ impl TypeChecker { } } + pub(super) fn check_stream_store( + &self, + args: &[Expr], + locals: &HashMap, + span: &Span, + ) -> crate::error::Result { + if args.len() != 3 { + return Err(CompileError::type_error( + "stream_store expects 3 arguments (ptr, index, value)", + span.clone(), + )); + } + let ptr_type = self.check_expr(&args[0], locals)?; + let idx_type = self.check_expr(&args[1], locals)?; + let val_type = self.check_expr(&args[2], locals)?; + + if !idx_type.is_integer() { + return Err(CompileError::type_error( + "stream_store index must be integer", + args[1].span().clone(), + )); + } + match (ptr_type, val_type) { + ( + Type::Pointer { + mutable: true, + inner, + .. + }, + Type::Vector { elem, .. }, + ) => { + if !types::types_compatible(&elem, &inner) { + return Err(CompileError::type_error( + format!( + "stream_store type mismatch: pointer element type is {inner}, but vector element type is {elem}" + ), + span.clone(), + )); + } + Ok(Type::Void) + } + ( + Type::Pointer { + mutable: true, + inner, + .. + }, + scalar @ (Type::I16 | Type::U16 | Type::I32 | Type::U32 | Type::I64 | Type::U64), + ) => { + if !types::types_compatible(&scalar, &inner) { + return Err(CompileError::type_error( + format!( + "stream_store type mismatch: pointer element type is {inner}, but scalar value type is {scalar}" + ), + span.clone(), + )); + } + Ok(Type::Void) + } + (Type::Pointer { mutable: false, .. }, _) => Err(CompileError::type_error( + "stream_store requires mutable pointer. Declare as *mut to allow writes", + args[0].span().clone(), + )), + (_, _) => Err(CompileError::type_error( + "stream_store expects (mut ptr, index, vector or i16/u16/i32/u32/i64/u64 scalar)", + span.clone(), + )), + } + } + pub(super) fn check_gather( &self, args: &[Expr], diff --git a/tests/stream_store_tests.rs b/tests/stream_store_tests.rs index bd2be01..68a7bc9 100644 --- a/tests/stream_store_tests.rs +++ b/tests/stream_store_tests.rs @@ -135,4 +135,42 @@ mod tests { "50 60 0 0", ); } + + // --- stream_store: scalar value support (typeck acceptance) --- + + #[test] + fn test_stream_store_scalar_i32_typecheck() { + // Scalar i32 stream_store must typecheck (extension over vector-only). + // Use typeck-only pipeline because scalar codegen lands in Task 2. + let source = r#" + export func test(out: *mut i32, v: i32) { + stream_store(out, 0, v) + } + "#; + let tokens = ea_compiler::tokenize(source).unwrap(); + let stmts = ea_compiler::parse(tokens).unwrap(); + let stmts = ea_compiler::desugar(stmts).unwrap(); + let result = ea_compiler::check_types(&stmts); + assert!( + result.is_ok(), + "scalar i32 stream_store should typecheck, got: {result:?}" + ); + } + + #[test] + fn test_stream_store_scalar_i64_typecheck() { + let source = r#" + export func test(out: *mut i64, v: i64) { + stream_store(out, 0, v) + } + "#; + let tokens = ea_compiler::tokenize(source).unwrap(); + let stmts = ea_compiler::parse(tokens).unwrap(); + let stmts = ea_compiler::desugar(stmts).unwrap(); + let result = ea_compiler::check_types(&stmts); + assert!( + result.is_ok(), + "scalar i64 stream_store should typecheck, got: {result:?}" + ); + } } From 1f41103557fc70409842c01626dc5001724751d6 Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 11:31:12 +0000 Subject: [PATCH 02/10] codegen: scalar stream_store branch compile_stream_store now branches on scalar vs. vector value type. Scalar path stores a single i16/u16/i32/u32/i64/u64 with the same !nontemporal metadata as the vector form. Lowers to movnti on x86 SSE2 for i32/i64; i16/u16 fall through to regular mov on x86 since no dedicated NT scalar instruction exists at that width. Closes the q4k_repack scalar-write surface gap identified during v1.15 brainstorm consumer audit (Olorin v2.0.3). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/codegen/simd_memory.rs | 32 ++++++++++++++++++++++---- tests/stream_store_tests.rs | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/codegen/simd_memory.rs b/src/codegen/simd_memory.rs index dd3c1e3..9a666e6 100644 --- a/src/codegen/simd_memory.rs +++ b/src/codegen/simd_memory.rs @@ -130,10 +130,25 @@ impl<'ctx> CodeGenerator<'ctx> { ) -> crate::error::Result> { let ptr_val = self.compile_expr(&args[0], function)?.into_pointer_value(); let idx_val = self.compile_expr(&args[1], function)?.into_int_value(); - let vec_val = self.compile_expr(&args[2], function)?.into_vector_value(); + let val_basic = self.compile_expr(&args[2], function)?; + + // Determine element type for GEP and the value to store. + // Vector path keeps the existing behavior; scalar path is the v1.15 addition. + let (elem_ty, store_val): (BasicTypeEnum<'ctx>, BasicValueEnum<'ctx>) = match val_basic { + BasicValueEnum::VectorValue(vec_val) => { + let vec_ty = vec_val.get_type(); + (vec_ty.get_element_type(), vec_val.into()) + } + BasicValueEnum::IntValue(int_val) => { + (int_val.get_type().into(), int_val.into()) + } + _ => { + return Err(CompileError::codegen_error( + "stream_store value must be a vector or i16/u16/i32/u32/i64/u64 scalar", + )); + } + }; - let vec_ty = vec_val.get_type(); - let elem_ty = vec_ty.get_element_type(); let elem_ptr = unsafe { self.builder .build_gep(elem_ty, ptr_val, &[idx_val], "nt_store_gep") @@ -142,10 +157,17 @@ impl<'ctx> CodeGenerator<'ctx> { let store_inst = self .builder - .build_store(elem_ptr, vec_val) + .build_store(elem_ptr, store_val) .map_err(|e| CompileError::codegen_error(e.to_string()))?; - let element_alignment = self.element_alignment(vec_ty.get_element_type()); + // Alignment: use element-natural alignment. For vectors this matches + // the existing behavior; for scalars LLVM derives natural alignment + // from the pointee type, which is sufficient for movnti on x86. + let element_alignment = match &val_basic { + BasicValueEnum::VectorValue(v) => self.element_alignment(v.get_type().get_element_type()), + BasicValueEnum::IntValue(v) => self.element_alignment(v.get_type().into()), + _ => unreachable!(), + }; store_inst.set_alignment(element_alignment).map_err(|e| { CompileError::codegen_error(format!("failed to set store alignment: {e}")) })?; diff --git a/tests/stream_store_tests.rs b/tests/stream_store_tests.rs index 68a7bc9..3a59a28 100644 --- a/tests/stream_store_tests.rs +++ b/tests/stream_store_tests.rs @@ -173,4 +173,50 @@ mod tests { "scalar i64 stream_store should typecheck, got: {result:?}" ); } + + // --- stream_store: scalar value support (codegen runtime) --- + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_stream_store_scalar_i32_runtime() { + assert_c_interop( + r#" + export func test(out: *mut i32, v: i32) { + stream_store(out, 0, v) + stream_store(out, 1, v + 100) + } + "#, + r#"#include + extern void test(int*, int); + int main() { + __attribute__((aligned(8))) int out[2] = {0, 0}; + test(out, 42); + printf("%d %d\n", out[0], out[1]); + return 0; + }"#, + "42 142", + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_stream_store_scalar_i64_runtime() { + assert_c_interop( + r#" + export func test(out: *mut i64, v: i64) { + stream_store(out, 0, v) + } + "#, + r#"#include + #include + extern void test(int64_t*, int64_t); + int main() { + __attribute__((aligned(8))) int64_t out[1] = {0}; + test(out, 0xDEADBEEFCAFEBABELL); + printf("%llx\n", (unsigned long long)out[0]); + return 0; + }"#, + "deadbeefcafebabe", + ); + } } From aa03392c30b5fb90617e053afb0414bc03ecd2ed Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 11:41:01 +0000 Subject: [PATCH 03/10] =?UTF-8?q?intrinsic:=20fence=5Fnt()=20=E2=80=94=20n?= =?UTF-8?q?on-temporal=20store=20fence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New zero-arg intrinsic providing intra-kernel store ordering after stream_store writes. Lowers to: - x86: @llvm.x86.sse.sfence() -> sfence - aarch64: @llvm.aarch64.dmb(i32 10) -> dmb ishst Explicit target intrinsics rather than IR fence release, which would lower to mfence (x86) / dmb ish (aarch64) — heavier than needed for store-only ordering. Cross-thread visibility comes from host-side sync primitives (pthread_join, rayon::scope, WaitGroup.Wait) which already provide release semantics. fence_nt() is for the rare intra-kernel case where the same kernel writes via stream_store and reads the same memory back before returning. Completes the prefetch_nta + stream_store + fence_nt non-temporal memory-hint family. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/codegen/mod.rs | 2 + src/codegen/simd.rs | 2 + src/codegen/simd_fence.rs | 73 +++++++++++++++++++++++++++++ src/typeck/intrinsics.rs | 1 + src/typeck/intrinsics_memory.rs | 14 ++++++ tests/fence_nt_tests.rs | 83 +++++++++++++++++++++++++++++++++ 6 files changed, 175 insertions(+) create mode 100644 src/codegen/simd_fence.rs create mode 100644 tests/fence_nt_tests.rs diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index 30d9ba4..bf051d9 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -19,6 +19,8 @@ mod simd_dotprod; #[cfg(feature = "llvm")] mod simd_exp_poly; #[cfg(feature = "llvm")] +mod simd_fence; +#[cfg(feature = "llvm")] mod simd_fp16; #[cfg(feature = "llvm")] mod simd_lane; diff --git a/src/codegen/simd.rs b/src/codegen/simd.rs index d21b772..179c6b7 100644 --- a/src/codegen/simd.rs +++ b/src/codegen/simd.rs @@ -25,6 +25,7 @@ impl<'ctx> CodeGenerator<'ctx> { | "load" | "store" | "stream_store" + | "fence_nt" | "gather" | "scatter" | "load_masked" @@ -246,6 +247,7 @@ impl<'ctx> CodeGenerator<'ctx> { self.compile_store(args, function) } "stream_store" => self.compile_stream_store(args, function), + "fence_nt" => self.compile_fence_nt(args, function), "gather" => self.compile_gather(args, type_hint, function), "permute_runtime" => self.compile_permute_runtime(args, function), "scatter" => self.compile_scatter(args, function), diff --git a/src/codegen/simd_fence.rs b/src/codegen/simd_fence.rs new file mode 100644 index 0000000..5bbc2f8 --- /dev/null +++ b/src/codegen/simd_fence.rs @@ -0,0 +1,73 @@ +//! `fence_nt()` — non-temporal store fence. +//! +//! Orders `stream_store` writes relative to each other and relative to +//! subsequent regular stores within the same kernel (store-store ordering +//! only). It does *not* order stores relative to subsequent loads — for a +//! write-then-read-back pattern, a full barrier (`mfence` / `dmb sy`) is +//! needed instead. Cross-thread visibility typically comes from the host- +//! side sync primitive after the kernel returns. +//! +//! Lowering: +//! - x86: `call void @llvm.x86.sse.sfence()` — emits a single `sfence`. +//! - aarch64: `call void @llvm.aarch64.dmb(i32 10)` — emits `dmb ishst` +//! (inner-shareable store-store barrier, the tightest matching ARM +//! barrier for NT-store ordering). +//! +//! These are explicit target intrinsics rather than the IR-level +//! `fence release`, because `fence release` lowers to `mfence` on x86 +//! (heavier than needed) and `dmb ish` on aarch64 (also heavier). + +use inkwell::values::{BasicValueEnum, FunctionValue}; + +use crate::ast::Expr; +use crate::error::CompileError; + +use super::CodeGenerator; + +impl<'ctx> CodeGenerator<'ctx> { + pub(crate) fn compile_fence_nt( + &mut self, + args: &[Expr], + _function: FunctionValue<'ctx>, + ) -> crate::error::Result> { + if !args.is_empty() { + return Err(CompileError::codegen_error( + "fence_nt takes 0 arguments", + )); + } + + if self.is_arm { + // aarch64: llvm.aarch64.dmb(i32 10) -> dmb ishst + let i32_type = self.context.i32_type(); + let fn_type = self.context.void_type().fn_type(&[i32_type.into()], false); + let dmb_fn = self + .module + .get_function("llvm.aarch64.dmb") + .unwrap_or_else(|| self.module.add_function("llvm.aarch64.dmb", fn_type, None)); + // 10 = ishst encoding per AArch64 architectural manual + let ishst = i32_type.const_int(10, false); + self.builder + .build_call(dmb_fn, &[ishst.into()], "") + .map_err(|e| CompileError::codegen_error(e.to_string()))?; + } else { + // x86: llvm.x86.sse.sfence() -> sfence + let fn_type = self.context.void_type().fn_type(&[], false); + let sfence_fn = self + .module + .get_function("llvm.x86.sse.sfence") + .unwrap_or_else(|| { + self.module + .add_function("llvm.x86.sse.sfence", fn_type, None) + }); + self.builder + .build_call(sfence_fn, &[], "") + .map_err(|e| CompileError::codegen_error(e.to_string()))?; + } + + // fence_nt is void; return the same placeholder i32(0) as other + // void-returning intrinsics like compile_store and compile_prefetch. + Ok(BasicValueEnum::IntValue( + self.context.i32_type().const_int(0, false), + )) + } +} diff --git a/src/typeck/intrinsics.rs b/src/typeck/intrinsics.rs index 8ebcc02..a7356bf 100644 --- a/src/typeck/intrinsics.rs +++ b/src/typeck/intrinsics.rs @@ -185,6 +185,7 @@ impl TypeChecker { "scatter" => Some(self.check_scatter(args, locals, span)), "load_masked" => Some(self.check_load_masked(args, locals, type_hint, span)), "stream_store" => Some(self.check_stream_store(args, locals, span)), + "fence_nt" => Some(self.check_fence_nt(args, span)), "store_masked" => Some(self.check_store_masked(args, locals, span)), "movemask" => Some(self.check_movemask(args, locals, span)), "min" | "max" => Some(self.check_min_max(name, args, locals, span)), diff --git a/src/typeck/intrinsics_memory.rs b/src/typeck/intrinsics_memory.rs index e3e8541..542b664 100644 --- a/src/typeck/intrinsics_memory.rs +++ b/src/typeck/intrinsics_memory.rs @@ -382,4 +382,18 @@ impl TypeChecker { )), } } + + pub(super) fn check_fence_nt( + &self, + args: &[Expr], + span: &Span, + ) -> crate::error::Result { + if !args.is_empty() { + return Err(CompileError::type_error( + "fence_nt takes 0 arguments (no operands)", + span.clone(), + )); + } + Ok(Type::Void) + } } diff --git a/tests/fence_nt_tests.rs b/tests/fence_nt_tests.rs new file mode 100644 index 0000000..1ba70b7 --- /dev/null +++ b/tests/fence_nt_tests.rs @@ -0,0 +1,83 @@ +#[cfg(feature = "llvm")] +mod common; + +#[cfg(feature = "llvm")] +mod tests { + use super::common::*; + + #[test] + fn test_fence_nt_typecheck() { + // fence_nt is a zero-arg intrinsic returning void; should typecheck. + let source = r#" + export func test() { + fence_nt() + } + "#; + let tokens = ea_compiler::tokenize(source).unwrap(); + let stmts = ea_compiler::parse(tokens).unwrap(); + let stmts = ea_compiler::desugar(stmts).unwrap(); + let result = ea_compiler::check_types(&stmts); + assert!( + result.is_ok(), + "fence_nt() should typecheck, got: {result:?}" + ); + } + + #[test] + fn test_fence_nt_rejects_args() { + let source = r#" + export func test() { + fence_nt(42) + } + "#; + let tokens = ea_compiler::tokenize(source).unwrap(); + let stmts = ea_compiler::parse(tokens).unwrap(); + let stmts = ea_compiler::desugar(stmts).unwrap(); + let result = ea_compiler::check_types(&stmts); + assert!( + result.is_err(), + "fence_nt(42) should be rejected, got: {result:?}" + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_fence_nt_ir_calls_sfence_x86() { + let ir = compile_to_ir( + r#" + export func test() { + fence_nt() + } + "#, + ); + assert!( + ir.contains("@llvm.x86.sse.sfence"), + "fence_nt on x86 must call llvm.x86.sse.sfence:\n{ir}" + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_fence_nt_runtime_x86() { + // Kernel: write via stream_store, fence, then verify the read + // sees the value within the same kernel call (intra-kernel visibility). + assert_c_interop( + r#" + export func test(out: *mut i32, v: i32) { + stream_store(out, 0, v) + fence_nt() + // Read back after fence to verify visibility + } + "#, + r#"#include + extern void test(int*, int); + int main() { + __attribute__((aligned(8))) int out[1] = {0}; + test(out, 42); + printf("%d\n", out[0]); + return 0; + }"#, + "42", + ); + } +} From 95334278b461c7dc88746d84cff0e4dc02388f3b Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 11:49:04 +0000 Subject: [PATCH 04/10] tests: aarch64-gated stream_store + fence_nt coverage Documents what LLVM 18 emits on aarch64 for !nontemporal scalar stores and fence_nt via runtime + IR assertions. Tests are target_arch-gated; require Pi 5 (per feedback_pi5_access_constraint) for full run before release-cut. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/fence_nt_tests.rs | 40 +++++++++++++++++++++++++++++++++++++ tests/stream_store_tests.rs | 22 ++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/tests/fence_nt_tests.rs b/tests/fence_nt_tests.rs index 1ba70b7..735e6b6 100644 --- a/tests/fence_nt_tests.rs +++ b/tests/fence_nt_tests.rs @@ -80,4 +80,44 @@ mod tests { "42", ); } + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_fence_nt_runtime_aarch64() { + assert_c_interop( + r#" + export func test(out: *mut i32, v: i32, readback: *mut i32) { + stream_store(out, 0, v) + fence_nt() + readback[0] = out[0] + } + "#, + r#"#include + extern void test(int*, int, int*); + int main() { + __attribute__((aligned(16))) int out[1] = {0}; + int readback = 0; + test(out, 7777, &readback); + printf("%d\n", readback); + return 0; + }"#, + "7777", + ); + } + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_fence_nt_ir_calls_dmb_aarch64() { + let ir = compile_to_ir( + r#" + export func test() { + fence_nt() + } + "#, + ); + assert!( + ir.contains("@llvm.aarch64.dmb"), + "fence_nt on aarch64 must call llvm.aarch64.dmb:\n{ir}" + ); + } } diff --git a/tests/stream_store_tests.rs b/tests/stream_store_tests.rs index 3a59a28..b9bbcfd 100644 --- a/tests/stream_store_tests.rs +++ b/tests/stream_store_tests.rs @@ -219,4 +219,26 @@ mod tests { "deadbeefcafebabe", ); } + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_stream_store_scalar_i64_runtime_aarch64() { + assert_c_interop( + r#" + export func test(out: *mut i64, v: i64) { + stream_store(out, 0, v) + } + "#, + r#"#include + #include + extern void test(int64_t*, int64_t); + int main() { + __attribute__((aligned(16))) int64_t out[1] = {0}; + test(out, 0x0123456789ABCDEFLL); + printf("%llx\n", (unsigned long long)out[0]); + return 0; + }"#, + "123456789abcdef", + ); + } } From 1af8a7c88ad4386f1484e66339715b07c04725da Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 11:53:36 +0000 Subject: [PATCH 05/10] tests: x86 objdump + alignment-failure coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds assert_intrinsic_in_disassembly checks for movnti (scalar i32/i64), movntps / movntdq (vector 128-bit), vmovntps / vmovntdq (vector 256-bit), and sfence (fence_nt). Per feedback_llvm_intrinsic_linktest, IR-level !nontemporal does not guarantee the emitted assembly uses an NT mnemonic — LLVM can drop the hint silently. Also adds a deliberate alignment-failure crash test pinning the documented alignment contract: vmovntps with a 1-byte-misaligned pointer must raise SIGSEGV. Uses child-process pattern so the parent test runner survives. Fixes vector stream_store codegen: alignment was incorrectly set to element size (4 bytes for f32), causing LLVM to emit scalar movntsd instead of vector movntps/vmovntps. Now computed as element_size * element_count to signal vector-width intent to LLVM's backend. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/codegen/simd_memory.rs | 33 ++++++---- tests/fence_nt_tests.rs | 13 ++++ tests/stream_store_tests.rs | 127 ++++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 11 deletions(-) diff --git a/src/codegen/simd_memory.rs b/src/codegen/simd_memory.rs index 9a666e6..26713d5 100644 --- a/src/codegen/simd_memory.rs +++ b/src/codegen/simd_memory.rs @@ -134,13 +134,13 @@ impl<'ctx> CodeGenerator<'ctx> { // Determine element type for GEP and the value to store. // Vector path keeps the existing behavior; scalar path is the v1.15 addition. - let (elem_ty, store_val): (BasicTypeEnum<'ctx>, BasicValueEnum<'ctx>) = match val_basic { + let (elem_ty, store_val, is_vector): (BasicTypeEnum<'ctx>, BasicValueEnum<'ctx>, bool) = match val_basic { BasicValueEnum::VectorValue(vec_val) => { let vec_ty = vec_val.get_type(); - (vec_ty.get_element_type(), vec_val.into()) + (vec_ty.get_element_type(), vec_val.into(), true) } BasicValueEnum::IntValue(int_val) => { - (int_val.get_type().into(), int_val.into()) + (int_val.get_type().into(), int_val.into(), false) } _ => { return Err(CompileError::codegen_error( @@ -160,15 +160,26 @@ impl<'ctx> CodeGenerator<'ctx> { .build_store(elem_ptr, store_val) .map_err(|e| CompileError::codegen_error(e.to_string()))?; - // Alignment: use element-natural alignment. For vectors this matches - // the existing behavior; for scalars LLVM derives natural alignment - // from the pointee type, which is sufficient for movnti on x86. - let element_alignment = match &val_basic { - BasicValueEnum::VectorValue(v) => self.element_alignment(v.get_type().get_element_type()), - BasicValueEnum::IntValue(v) => self.element_alignment(v.get_type().into()), - _ => unreachable!(), + // Alignment: for vectors, use vector-width alignment (computed from element + // count and element size) to help LLVM emit vector stores. For scalars, + // use natural element alignment. + let alignment = if is_vector { + // For a vector, LLVM's !nontemporal hint works best when the alignment + // reflects the full vector width. Compute as element_size * element_count. + if let BasicValueEnum::VectorValue(v) = &val_basic { + let vec_ty = v.get_type(); + let elem_ty = vec_ty.get_element_type(); + let elem_align = self.element_alignment(elem_ty); + let elem_count = vec_ty.get_size() as u32; + (elem_align * elem_count).max(1) + } else { + unreachable!() + } + } else { + self.element_alignment(elem_ty) }; - store_inst.set_alignment(element_alignment).map_err(|e| { + + store_inst.set_alignment(alignment).map_err(|e| { CompileError::codegen_error(format!("failed to set store alignment: {e}")) })?; diff --git a/tests/fence_nt_tests.rs b/tests/fence_nt_tests.rs index 735e6b6..cf963c9 100644 --- a/tests/fence_nt_tests.rs +++ b/tests/fence_nt_tests.rs @@ -120,4 +120,17 @@ mod tests { "fence_nt on aarch64 must call llvm.aarch64.dmb:\n{ir}" ); } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_fence_nt_emits_sfence() { + assert_intrinsic_in_disassembly( + r#" + export func test() { + fence_nt() + } + "#, + &["sfence"], + ); + } } diff --git a/tests/stream_store_tests.rs b/tests/stream_store_tests.rs index b9bbcfd..14fe804 100644 --- a/tests/stream_store_tests.rs +++ b/tests/stream_store_tests.rs @@ -241,4 +241,131 @@ mod tests { "123456789abcdef", ); } + + // --- x86 objdump assertions: scalar and vector stream_store mnemonics --- + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_stream_store_scalar_i32_emits_movnti() { + assert_intrinsic_in_disassembly( + r#" + export func test(out: *mut i32, v: i32) { + stream_store(out, 0, v) + } + "#, + &["movnti"], + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_stream_store_scalar_i64_emits_movnti() { + assert_intrinsic_in_disassembly( + r#" + export func test(out: *mut i64, v: i64) { + stream_store(out, 0, v) + } + "#, + &["movnti"], + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_stream_store_f32x4_emits_movntps() { + assert_intrinsic_in_disassembly( + r#" + export func test(data: *f32, out: *mut f32) { + let v: f32x4 = load(data, 0) + stream_store(out, 0, v) + } + "#, + &["movntps", "movntdq"], + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_stream_store_f32x8_emits_vmovntps() { + assert_intrinsic_in_disassembly( + r#" + export func test(data: *f32, out: *mut f32) { + let v: f32x8 = load(data, 0) + stream_store(out, 0, v) + } + "#, + &["vmovntps", "vmovntdq"], + ); + } + + // --- x86 alignment-failure crash test --- + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_stream_store_misaligned_vector_crashes() { + // Deliberately misaligned (byte-offset 1 into an aligned array) + // pointer passed to stream_store of a 256-bit vector should + // raise SIGSEGV on x86 (vmovntps requires 32-byte alignment). + // + // This test documents the alignment contract by demonstrating + // the consequence of violating it. Uses a child-process pattern + // so the parent test runner doesn't crash. + use std::process::{Command, Stdio}; + + let dir = tempfile::TempDir::new().unwrap(); + let obj_path = dir.path().join("kernel.o"); + let c_path = dir.path().join("harness.c"); + let bin_path = dir.path().join("test_bin"); + + ea_compiler::compile( + r#" + export func test(out: *mut f32) { + let v: f32x8 = splat(1.0) + stream_store(out, 0, v) + } + "#, + &obj_path, + ea_compiler::OutputMode::ObjectFile, + ) + .expect("compile"); + + std::fs::write( + &c_path, + r#" + #include + extern void test(float*); + int main() { + // 32-byte alignment + 1-byte offset = guaranteed misalignment for f32x8 + __attribute__((aligned(32))) static char buf[64]; + float* misaligned = (float*)(buf + 1); + test(misaligned); + return 0; + } + "#, + ) + .expect("write C harness"); + + Command::new("cc") + .arg(&c_path) + .arg(&obj_path) + .arg("-o") + .arg(&bin_path) + .status() + .expect("link"); + + let status = Command::new(&bin_path) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("run binary"); + + // Process should NOT exit cleanly — expect signal-termination. + assert!( + !status.success(), + "misaligned vector stream_store unexpectedly succeeded — \ + alignment contract violated without consequence. Check that \ + codegen still attaches !nontemporal metadata and that LLVM \ + is not silently substituting an aligned movups." + ); + } } From 3a1e7084509b7e3ae6beeb576ac489c1e0e611ed Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 12:02:05 +0000 Subject: [PATCH 06/10] docs: stream_store + fence_nt v1.15 reference upgrade stream_store reference brought to prefetch_nta parity: - Target-specific lowering table covering vector + scalar widths - Explicit alignment contract (GPF on x86 misalignment) - Explicit ordering contract (weak ordering, caller-side fence) - "When NOT to use" section listing working-buffer anti-patterns (softmax accumulators, FWHT scratch) to prevent regressions during consumer adoption New fence_nt reference section covering target lowering (sfence / dmb ishst), intended-use (intra-kernel ordering only), and anti-patterns (cross-thread fencing is the caller's sync primitive's job; store-to-load ordering requires a full barrier). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/src/guide/common-intrinsics.md | 14 ++-- docs/src/reference/intrinsics.md | 108 +++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 8 deletions(-) diff --git a/docs/src/guide/common-intrinsics.md b/docs/src/guide/common-intrinsics.md index 2f9d06a..acfbb14 100644 --- a/docs/src/guide/common-intrinsics.md +++ b/docs/src/guide/common-intrinsics.md @@ -38,15 +38,16 @@ These are equivalent to plain `load` but make the element type visible in the so ## stream_store -Non-temporal store that bypasses the CPU cache. Use for write-only output buffers where you will not read the data back soon: +Non-temporal store — bypasses cache, used for write-only output. Vector +or scalar value (v1.15.0 added scalar i16/u16/i32/u32/i64/u64). See +[reference](../reference/intrinsics.md#stream_store) for the full +alignment contract, ordering contract, and anti-patterns. ``` -let result: f32x8 = a .* b -stream_store(out, i, result) +stream_store(out, i, result) // vector +stream_store(out, i, scalar_value) // scalar (v1.15.0+) ``` -This avoids polluting the cache with output data, leaving more cache space for inputs. Only beneficial for large output arrays that will not be re-read immediately. - ## fma Fused multiply-add: computes `a * b + c` in a single instruction with a single rounding (more accurate than separate multiply and add): @@ -172,7 +173,8 @@ The `rem` parameter specifies how many elements (starting from lane 0) are valid | `splat(s)` | scalar | vector | Broadcast to all lanes | | `load(ptr, i)` | pointer, offset | vector | Load vector from memory | | `store(ptr, i, v)` | pointer, offset, vector | void | Write vector to memory | -| `stream_store(ptr, i, v)` | pointer, offset, vector | void | Non-temporal write | +| `stream_store(ptr, i, v)` | pointer, offset, vector or scalar | void | Non-temporal write | +| `fence_nt()` | none | void | Store-store barrier for stream_store ordering | | `fma(a, b, c)` | 3 values | same type | `a * b + c` fused | | `reduce_add(v)` | vector | scalar | Sum all lanes | | `reduce_max(v)` | vector | scalar | Max across lanes | diff --git a/docs/src/reference/intrinsics.md b/docs/src/reference/intrinsics.md index 0473bf1..3885252 100644 --- a/docs/src/reference/intrinsics.md +++ b/docs/src/reference/intrinsics.md @@ -62,12 +62,116 @@ store(out, i, result); ### stream_store -Non-temporal store that bypasses the CPU cache. Use for write-only output that will not be read back soon. +Non-temporal store that bypasses the CPU cache. Use for write-only output +the kernel will not read back soon. Pairs with `prefetch_nta` (the read-side +non-temporal hint) and `fence_nt` (the ordering primitive). ``` -stream_store(out, i, result); +stream_store(out, i, v) // vector form — v is f32xN, i32xN, etc. +stream_store(out, i, scalar_value) // scalar form (v1.15.0) — i16/u16/i32/u32/i64/u64 ``` +**Target lowering:** + +| Width / form | x86 | aarch64 | +|---|---|---| +| Vector 128-bit (f32x4, i32x4, ...) | `movntps` / `movntdq` | regular `str q` (LLVM 18 does not optimize `!nontemporal` for single-register NEON) | +| Vector 256-bit (AVX2) | `vmovntps` / `vmovntdq` | n/a (256-bit not on NEON) | +| Vector 512-bit (AVX-512) | `vmovntps` / `vmovntdq` zmm | n/a | +| Scalar i32 / u32 | `movnti` (SSE2) | `str` / paired `stnp` when alignment permits | +| Scalar i64 / u64 | `movnti` (SSE2, 64-bit mode) | `str` / paired `stnp` | +| Scalar i16 / u16 | regular `mov` (no NT scalar at 16-bit width) | `strh` / paired `stnp` | + +The i16/u16 case ships for shape symmetry with the wider scalars — the +non-temporal hint becomes a no-op on x86 at that width. + +**Alignment contract:** + +Vector `stream_store` requires the destination pointer plus byte offset to +be aligned to the vector's natural size (16 bytes for 128-bit, 32 bytes for +256-bit, 64 bytes for 512-bit). Misaligned NT vector stores raise a general +protection fault on x86. Scalar `stream_store` requires natural alignment to +the scalar size on x86 (4-byte for i32/u32, 8-byte for i64/u64). Callers +must provide aligned buffers; Eä does not insert runtime alignment checks. + +**Ordering contract:** + +NT stores are weakly ordered on x86 (write-combining memory order). Other +cores or subsequent reads in the same thread may observe them out of +program order. For cross-thread visibility, the typical pathway is through +a host-side synchronization primitive after the kernel returns +(`pthread_join`, `rayon::scope`, `WaitGroup.Wait`) — these provide release +semantics that flush WC buffers. For intra-kernel ordering (writing then +reading the same memory in the same kernel call), use `fence_nt()` +explicitly. Eä does not insert an implicit fence at kernel return. + +**When NOT to use:** + +Do not use `stream_store` for working buffers the same kernel reads back. +The non-temporal hint asks the cache to *not* keep the line; if the kernel +reads the data soon afterward, the read goes to DRAM and is slower than a +regular `store` followed by a cache hit. Working-buffer examples that +should use plain `store`: + +- Softmax accumulators (e.g. `scores_buf` in attention kernels) +- FWHT scratch arrays (e.g. `scratch` in JL-projection kernels) +- Per-iteration partial sums or running statistics + +`stream_store` is appropriate when the destination is a final output passed +to the next kernel call, a memory region the current kernel never re-reads, +or a buffer that will not be touched again until a downstream consumer +pulls it from DRAM later. + +### fence_nt + +Store-store memory barrier providing intra-kernel ordering of preceding +`stream_store` operations. Zero arguments, returns void. + +``` +fence_nt() +``` + +**Target lowering:** + +| Target | Instruction | +|---|---| +| x86 | `sfence` (via `@llvm.x86.sse.sfence`) | +| aarch64 | `dmb ishst` (via `@llvm.aarch64.dmb` with operand `10`) | + +These are the narrowest available barriers for store-only ordering — +explicit target intrinsics rather than the IR-level `fence release`, which +would lower to `mfence` on x86 and `dmb ish` on aarch64 (both heavier than +needed for NT-store ordering). + +**Semantics:** + +`fence_nt()` orders `stream_store` writes relative to each other and +relative to subsequent regular stores. It does *not* order stores relative +to subsequent loads — for a write-then-read-back pattern in the same kernel, +a full barrier (`mfence` on x86, `dmb sy` on aarch64) is needed instead. +Eä does not currently expose a full-barrier intrinsic. + +**When to use:** + +Use `fence_nt()` when the same kernel writes via `stream_store` to multiple +non-overlapping regions in a defined order and a later kernel (or downstream +reader) relies on observing those writes in the same order. This is +uncommon — most callers don't need it, because cross-thread visibility +comes from the host's sync primitive (`pthread_join`, `rayon::scope`, +`WaitGroup.Wait`) which already provides release semantics that flush +write-combining buffers between threads. + +**When NOT to use:** + +- Between successive `stream_store` calls to different addresses if no + store-ordering requirement exists — NT stores to the same address + complete in program order regardless of fences. +- At the end of a kernel as "insurance" — the caller's sync primitive + handles cross-thread fencing more efficiently after the kernel returns. +- For write-then-read-back patterns — `fence_nt()` does not provide store- + to-load ordering. Use a regular `store` for the working data, or do not + read NT-written data back in the same kernel. + ### load_masked Masked vector load. Lanes where the mask is false are not loaded. From 67079c4714f221c2a7a944234e43164266a4cd1e Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 12:09:54 +0000 Subject: [PATCH 07/10] =?UTF-8?q?release:=20v1.15.0=20=E2=80=94=20stream?= =?UTF-8?q?=5Fstore=20family=20completion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps version to 1.15.0. CHANGELOG documents the additions (scalar stream_store overloads, fence_nt), the fixed pre-existing vector alignment bug, the docs upgrade, and the test hardening. ROADMAP amends the Multi-core / parallel_for entry with the v1.15 audit finding (Olorin + Cougar already ship custom SpinBarrier-based pools strictly more capable than any generic primitive eacompute could provide). Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 62 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- ROADMAP.md | 82 ++++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 133 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 712c85b..e162998 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,67 @@ # Changelog +## v1.15.0 — 2026-05-20 — Non-temporal store family completion + +### Added + +- **Scalar `stream_store` overloads** — `i16/u16/i32/u32/i64/u64` value + types now accepted in addition to vectors. Lowers via the same + `!nontemporal` metadata path used for the vector form; emits `movnti` + on x86 SSE2 for i32/i64. Closes the scalar-write surface gap blocking + Olorin's `q4k_repack.ea` kernel. +- **`fence_nt()` intrinsic** — zero-argument store-store memory barrier + for intra-kernel ordering of preceding `stream_store` operations. + Lowers to `sfence` on x86 (via `@llvm.x86.sse.sfence`), `dmb ishst` on + aarch64 (via `@llvm.aarch64.dmb`). Completes the `prefetch_nta` + + `stream_store` + `fence_nt` non-temporal memory-hint family. + +### Fixed + +- **Vector `stream_store` alignment** — pre-existing bug where the + `set_alignment` call used the element type's natural alignment (e.g. + 4 bytes for `f32x4`) rather than the full vector width. LLVM was + silently decomposing 128/256-bit NT stores to scalar `movntsd` + sequences because the alignment metadata didn't authorize vector-width + stores. Caught by the new objdump assertions; alignment now set to + `element_size * lane_count`, so x86 emits `movntps`/`vmovntps` + /`movntdq`/`vmovntdq` directly. Behavior change: kernels passing + vector-aligned buffers see the intended fast path; kernels passing + misaligned buffers now SIGSEGV per the documented alignment contract + (previously they got slow scalarized stores). + +### Changed + +- **`stream_store` reference documentation** upgraded to `prefetch_nta` + parity. Adds target-specific lowering table covering all vector and + scalar widths, explicit alignment contract (general protection fault + on x86 misalignment), explicit ordering contract, and a "When NOT to + use" section calling out working-buffer anti-patterns (softmax + accumulators, FWHT scratch) to prevent adoption regressions. + +### Test hardening + +- aarch64-gated runtime tests for scalar `stream_store` and `fence_nt`, + documenting actual LLVM 18 emission on the Cortex-A76 path. +- objdump-level assertions verifying `movnti` / `movntps` / `vmovntps` / + `vmovntdq` / `sfence` actually emitted on x86 (not just present in IR + metadata) — caught the vector alignment bug fixed above. +- Alignment-failure crash test pinning the alignment contract via + deliberate SIGSEGV from a 1-byte-misaligned `f32x8` `stream_store`. + +### Out of scope (deferred) + +- In-language `parallel_for` keyword / binding-layer parallel dispatch — + deferred indefinitely. Audit of the highest-performance Eä consumers + (Olorin and Cougar) showed both already ship custom SpinBarrier-based + thread pools strictly more capable than any generic primitive Eä + could provide. See ROADMAP for amended entry. +- Sub-byte bit-packing intrinsics (`load_packed_iN_to_*`) — deferred + indefinitely. Audit showed every shipped quantized-weight consumer + either fuses unpack into compute (Cougar BitNet), has already-clean + 2-op unpacks (eakv Q4_1), or uses format-specific mixed-width + layouts that no generic intrinsic could capture (Olorin GGML Q4_K / + Q3_K). + ## v1.14.0 — 2026-05-19 — f32 transcendental family complete + Olorin-driven SIMD primitives Closes the v1.11.0-era "Future API consistency" list. The f32 transcendental approximation family is feature-complete: `tanh_approx_f32`, `log_approx_f32`, `sin_approx_f32`, and `cos_approx_f32` join `exp_poly_f32` (v1.11.0), all sharing the same f32-vector-only contract and ~3e-6 absolute error budget. `u16x32` + `lo256_u16x32` / `hi256_u16x32` close the i16/u16 lane-extractor symmetry deferred in v1.12.0 PR #10. `wmul_u64(u32x4, u32x4) -> u64x4` ships as a fused alternative to the v1.12.0 `wmul_u64_lo` / `wmul_u64_hi` pair. Olorin-driven SIMD primitives: `permute_runtime` (AVX2 runtime data permute), `prefetch_write` / `prefetch_nta` (write-intent + non-temporal cache hints), and two-source `shuffle(a, b, [indices])`. Doc-side: the types reference now lists every lexer-accepted vector type — closing a multi-release gap that included the AVX-512BW byte/word types — and the cookbook tanh-GELU recipe was rewritten to use the new `tanh_approx_f32` instead of the catastrophic-cancellation-prone `(exp_poly_f32(2x) - 1) / (exp_poly_f32(2x) + 1)` identity. diff --git a/Cargo.toml b/Cargo.toml index b3a82d7..b8f6ab0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ea-compiler" -version = "1.14.0" +version = "1.15.0" edition = "2024" description = "Eä — compute kernel compiler" authors = ["Peter Lukka "] diff --git a/ROADMAP.md b/ROADMAP.md index dac1f20..d23a0cf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,49 @@ Forward-looking notes. Ordered by leverage, not by effort. -## Shipped in v1.14.0 (UNRELEASED) +## Shipped in v1.15.0 + +### Scalar `stream_store` overloads + +`stream_store(*mut i16, offset, i16)`, `stream_store(*mut u16, offset, u16)`, +`stream_store(*mut i32, offset, i32)`, `stream_store(*mut u32, offset, u32)`, +`stream_store(*mut i64, offset, i64)`, `stream_store(*mut u64, offset, u64)`. +Same `!nontemporal` metadata path as the existing vector form. Lowers to +`movnti` on x86 SSE2 for i32/i64. Concrete consumer pulling: Olorin's +`q4k_repack.ea` (pure-streaming block-layout repack blocked on scalar surface). + +### `fence_nt()` intrinsic + +Zero-argument store-store memory barrier. Lowers to `sfence` on x86 (via +`@llvm.x86.sse.sfence`) and `dmb ishst` on aarch64 (via `@llvm.aarch64.dmb` +with operand 10). Completes the `prefetch_nta` + `stream_store` + `fence_nt` +non-temporal memory-hint family. Most callers will not need it — host-side +sync primitives provide cross-thread release semantics — but for the rare +intra-kernel ordering case it's the documented expression. Note: does NOT +provide store-to-load ordering; a full barrier (`mfence`/`dmb sy`) is needed +for write-then-read-back patterns. + +### Vector `stream_store` alignment fix + +Pre-existing bug surfaced by the new objdump test discipline: the +`set_alignment` call on the store instruction passed element-width alignment +rather than vector-width. LLVM 18 took this conservatively and decomposed +NT vector stores into element-wise scalar `movntsd` sequences, defeating +the entire point of the intrinsic. Fix: set alignment to +`element_size * lane_count`. Behavior change visible to callers — fast path +for aligned buffers, SIGSEGV per the documented contract for misaligned. +Caught and pinned by the new alignment-failure crash test. + +### `stream_store` documentation upgrade + +Reference docs upgraded to `prefetch_nta` parity: target-specific lowering +table, alignment contract, ordering contract, and "When NOT to use" anti- +pattern guidance. The last item is most important — prevents adoption +regressions in working-buffer kernels (softmax accumulators, FWHT scratch) +where blanket-substituting `store → stream_store` would degrade by forcing +DRAM round-trips on cache-resident data. + +## Shipped in v1.14.0 ### Runtime SIMD permute @@ -74,17 +116,33 @@ Today the language spec is spread across `docs/src/reference/*.md` (types, intri ## Future additions -### Multi-core / `parallel_for` primitive - -Eä is single-thread SIMD today; concurrency comes from outer-loop threading in the Rust / Python / Go caller. A `parallel_for(range, body)` primitive that spawns SIMD work across cores would change the per-call performance model from "kernel uses one core" to "kernel uses the machine." Pi 5 has 4 A76 cores; Zen 4 has 16+. With single-thread SIMD perf increasingly memory-bound (chacha20 hits 3.6 GB/s on Zen 4 = DRAM ceiling, not compute), multi-core is the unused dimension where the next 2–10× lives. - -Open design questions: -- **Thread-pool model.** Static partition is simple but loses to imbalanced workloads; work-stealing (rayon-style) is robust but adds runtime dependency. Eä's "no implicit runtime" stance suggests caller-supplied pool injection. -- **C ABI interaction.** Does the kernel signature change? `(thread_id, num_threads)` extra args, or hidden global? -- **Determinism.** Reductions become non-associative under parallel execution; `reduce_add` semantics across threads need a documented contract. -- **NUMA / cache discipline.** First-touch placement matters on Zen 4 and on multi-socket. Probably out-of-scope for v1.x, but the API shape shouldn't preclude it. - -Likely a v1.15+ initiative — too large for v1.14.0, but worth scoping early so the smaller carry-overs don't constrain the design space. +### Multi-core / `parallel_for` primitive — deferred indefinitely (v1.15 audit) + +Eä remains single-thread SIMD. Multi-core orchestration is the host's +responsibility. The v1.15 brainstorm evaluated two candidate forms — an +in-language `parallel_for` keyword and a binding-layer `parallel: true` +metadata flag generating per-language wrappers — and dropped both after a +consumer audit: + +- **Olorin** ships `src/inference/threadpool.rs`, a custom SpinBarrier-based + pool with `run_graph`. Designed for ggml-style inference graphs with cross-kernel + barriers and atomic-int dynamic work counters. Strictly more capable than + anything generic Eä could provide; a binding-layer wrapper cannot express + cross-kernel barriers. +- **Cougar** invokes Eä kernels (`q4k_4row_dot` — 4 rows per call) inside + `pool.run(n_threads, |tid, _| { ... })`, looping many times per thread + with per-thread accumulator state across calls. A "wrap in N threads, + one call each" binding wrapper would replace this with strictly worse + work distribution. +- **eakv** has no parallelism today but the workaround for the dequant case + is 3 lines of `concurrent.futures.ThreadPoolExecutor` in eakv's own code + — not enough pull to justify permanent surface area in eacompute. + +Criteria for revisiting: a new consumer profile that custom Olorin/Cougar +pools don't already serve — most likely a Python-first consumer with +non-trivial host-side parallelism cost, or a consumer with simpler graph +structure than ggml-style inference. None has surfaced. ### Autoresearch ↔ perf-regression feedback From 2093d4b2d8b49caedf72b29f1c02fde9008af88f Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 12:17:55 +0000 Subject: [PATCH 08/10] fix(clippy): remove redundant `as u32` cast in compile_stream_store VectorType::get_size() already returns u32; the explicit cast triggers clippy::unnecessary_cast. Caught by the v1.15.0 release-cut clippy gate. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/codegen/simd_memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen/simd_memory.rs b/src/codegen/simd_memory.rs index 26713d5..ef0a682 100644 --- a/src/codegen/simd_memory.rs +++ b/src/codegen/simd_memory.rs @@ -170,7 +170,7 @@ impl<'ctx> CodeGenerator<'ctx> { let vec_ty = v.get_type(); let elem_ty = vec_ty.get_element_type(); let elem_align = self.element_alignment(elem_ty); - let elem_count = vec_ty.get_size() as u32; + let elem_count = vec_ty.get_size(); (elem_align * elem_count).max(1) } else { unreachable!() From c48d8d9a871c68f5e8dd611c3209cb58510be3cf Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 12:27:01 +0000 Subject: [PATCH 09/10] =?UTF-8?q?chore:=20pre-tag=20cleanup=20=E2=80=94=20?= =?UTF-8?q?stale=20comments=20+=20mnemonic=20accuracy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove stale "Task 2 lands later" comment in stream_store scalar typeck test; Task 2 has shipped on this branch. - Fix misleading "Read back after fence to verify visibility" comment in test_fence_nt_runtime_x86 — the kernel writes + fences; the C host reads back, not the kernel. - Replace "scalar movntsd sequences" with the more accurate "sequence of scalar (non-temporal-hinted) stores" in CHANGELOG and ROADMAP — movntsd is SSE4a/AMD-specific, the actual LLVM fallback for under- aligned NT stores is not guaranteed to be that specific mnemonic. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 15 ++++++++------- ROADMAP.md | 4 ++-- tests/fence_nt_tests.rs | 4 +--- tests/stream_store_tests.rs | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e162998..4fcef61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,13 +20,14 @@ - **Vector `stream_store` alignment** — pre-existing bug where the `set_alignment` call used the element type's natural alignment (e.g. 4 bytes for `f32x4`) rather than the full vector width. LLVM was - silently decomposing 128/256-bit NT stores to scalar `movntsd` - sequences because the alignment metadata didn't authorize vector-width - stores. Caught by the new objdump assertions; alignment now set to - `element_size * lane_count`, so x86 emits `movntps`/`vmovntps` - /`movntdq`/`vmovntdq` directly. Behavior change: kernels passing - vector-aligned buffers see the intended fast path; kernels passing - misaligned buffers now SIGSEGV per the documented alignment contract + silently decomposing 128/256-bit NT stores to a sequence of scalar + (non-temporal-hinted) stores because the alignment metadata didn't + authorize vector-width stores. Caught by the new objdump assertions; + alignment now set to `element_size * lane_count`, so x86 emits + `movntps`/`vmovntps`/`movntdq`/`vmovntdq` directly. Behavior change: + kernels passing vector-aligned buffers see the intended fast path; + kernels passing misaligned buffers now SIGSEGV per the documented + alignment contract (previously they got slow scalarized stores). ### Changed diff --git a/ROADMAP.md b/ROADMAP.md index d23a0cf..5a92952 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -29,8 +29,8 @@ for write-then-read-back patterns. Pre-existing bug surfaced by the new objdump test discipline: the `set_alignment` call on the store instruction passed element-width alignment rather than vector-width. LLVM 18 took this conservatively and decomposed -NT vector stores into element-wise scalar `movntsd` sequences, defeating -the entire point of the intrinsic. Fix: set alignment to +NT vector stores into a sequence of scalar (non-temporal-hinted) stores, +defeating the entire point of the intrinsic. Fix: set alignment to `element_size * lane_count`. Behavior change visible to callers — fast path for aligned buffers, SIGSEGV per the documented contract for misaligned. Caught and pinned by the new alignment-failure crash test. diff --git a/tests/fence_nt_tests.rs b/tests/fence_nt_tests.rs index cf963c9..d9023a6 100644 --- a/tests/fence_nt_tests.rs +++ b/tests/fence_nt_tests.rs @@ -59,14 +59,12 @@ mod tests { #[test] #[cfg(target_arch = "x86_64")] fn test_fence_nt_runtime_x86() { - // Kernel: write via stream_store, fence, then verify the read - // sees the value within the same kernel call (intra-kernel visibility). + // Kernel: write via stream_store, fence. C host reads back to verify visibility. assert_c_interop( r#" export func test(out: *mut i32, v: i32) { stream_store(out, 0, v) fence_nt() - // Read back after fence to verify visibility } "#, r#"#include diff --git a/tests/stream_store_tests.rs b/tests/stream_store_tests.rs index 14fe804..040c07d 100644 --- a/tests/stream_store_tests.rs +++ b/tests/stream_store_tests.rs @@ -141,7 +141,7 @@ mod tests { #[test] fn test_stream_store_scalar_i32_typecheck() { // Scalar i32 stream_store must typecheck (extension over vector-only). - // Use typeck-only pipeline because scalar codegen lands in Task 2. + // typeck-only path is sufficient; runtime coverage is in test_stream_store_scalar_i32_runtime. let source = r#" export func test(out: *mut i32, v: i32) { stream_store(out, 0, v) From 37b7c881c545c2e7b19f3cc4744c717930ae8bf8 Mon Sep 17 00:00:00 2001 From: Peter Lukka Date: Wed, 20 May 2026 12:59:10 +0000 Subject: [PATCH 10/10] docs+tests: aarch64 lowering corrections from Pi 5 verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pi 5 (Cortex-A76, LLVM 18.1.8) capture revealed the original v1.15.0 aarch64 lowering claims were wrong for 4 of the 6 stream_store rows. Root cause: aarch64 has no scalar non-temporal store instruction. The only NT store is stnp (Store Non-temporal Pair), which requires two operands. LLVM 18 honors !nontemporal only when it can synthesize stnp: - i64 self-pairs to a w-pair via lsr — NT preserved - 128-bit vector q-register self-pairs to a d-pair — NT preserved - i32 / i16 scalars have no pair-friendly form — NT hint dropped, plain str / strh emitted Adjacent stream_store(*mut i32, ...) calls fuse to regular stp on aarch64, not stnp — LLVM does not synthesize NT pairs from sequential non-NT-fusable stores. Doc and metadata changes: - docs/src/reference/intrinsics.md: target-lowering table corrected; added explanatory paragraph about aarch64-no-scalar-NT semantics and the "prefer 64-bit-or-wider on aarch64" guidance. - CHANGELOG.md: scalar overloads entry now lists the actual per-target lowering matrix instead of x86-only mnemonics. - ROADMAP.md: shipped-in-v1.15.0 entry expanded with the aarch64 semantics summary and pointer to the reference for the full matrix. Test additions (aarch64-gated, pinning observed LLVM 18 behavior): - test_stream_store_scalar_i64_emits_stnp_aarch64 - test_stream_store_f32x4_emits_stnp_aarch64 - test_stream_store_scalar_i32_emits_plain_str_aarch64 (PIN: hint dropped) - test_stream_store_scalar_i16_emits_plain_strh_aarch64 (PIN: hint dropped) - test_fence_nt_emits_dmb_ishst_aarch64 Mirrors the x86 objdump discipline so a future LLVM upgrade that changes any of these mnemonics will fail loudly instead of silently drifting away from the reference docs. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 17 ++++++-- ROADMAP.md | 17 ++++++-- docs/src/reference/intrinsics.md | 33 +++++++++++---- tests/fence_nt_tests.rs | 15 +++++++ tests/stream_store_tests.rs | 70 ++++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fcef61..a3b4395 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,13 @@ - **Scalar `stream_store` overloads** — `i16/u16/i32/u32/i64/u64` value types now accepted in addition to vectors. Lowers via the same - `!nontemporal` metadata path used for the vector form; emits `movnti` - on x86 SSE2 for i32/i64. Closes the scalar-write surface gap blocking - Olorin's `q4k_repack.ea` kernel. + `!nontemporal` metadata path used for the vector form. On x86, `movnti` + is emitted for i32/u32/i64/u64 (i16/u16 fall through to plain `mov`). + On aarch64, `stnp` is synthesized only when LLVM can self-pair (i64 + splits to a `w`-pair; i32/i16 fall through to plain `str`/`strh` — see + the target-lowering table in the reference for the full matrix). + Closes the scalar-write surface gap blocking Olorin's `q4k_repack.ea` + kernel. - **`fence_nt()` intrinsic** — zero-argument store-store memory barrier for intra-kernel ordering of preceding `stream_store` operations. Lowers to `sfence` on x86 (via `@llvm.x86.sse.sfence`), `dmb ishst` on @@ -42,10 +46,15 @@ ### Test hardening - aarch64-gated runtime tests for scalar `stream_store` and `fence_nt`, - documenting actual LLVM 18 emission on the Cortex-A76 path. + verified on Pi 5 (Cortex-A76, LLVM 18.1.8) against expected mnemonics. - objdump-level assertions verifying `movnti` / `movntps` / `vmovntps` / `vmovntdq` / `sfence` actually emitted on x86 (not just present in IR metadata) — caught the vector alignment bug fixed above. +- aarch64 objdump-level assertions for `dmb ishst` (`fence_nt`), the + i64-to-`stnp` w-pair split (scalar i64), and the q-to-`stnp` d-pair + split (vector 128-bit) — pin the actually-observed LLVM 18 behavior + so a future LLVM upgrade changing aarch64 NT-store synthesis lands + loudly rather than silently. - Alignment-failure crash test pinning the alignment contract via deliberate SIGSEGV from a 1-byte-misaligned `f32x8` `stream_store`. diff --git a/ROADMAP.md b/ROADMAP.md index 5a92952..24f40f7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -9,9 +9,20 @@ Forward-looking notes. Ordered by leverage, not by effort. `stream_store(*mut i16, offset, i16)`, `stream_store(*mut u16, offset, u16)`, `stream_store(*mut i32, offset, i32)`, `stream_store(*mut u32, offset, u32)`, `stream_store(*mut i64, offset, i64)`, `stream_store(*mut u64, offset, u64)`. -Same `!nontemporal` metadata path as the existing vector form. Lowers to -`movnti` on x86 SSE2 for i32/i64. Concrete consumer pulling: Olorin's -`q4k_repack.ea` (pure-streaming block-layout repack blocked on scalar surface). +Same `!nontemporal` metadata path as the existing vector form. + +Cross-target lowering (Pi 5-verified, LLVM 18.1.8): x86 emits `movnti` for +i32/u32/i64/u64 (i16/u16 fall through to plain `mov`); aarch64 emits `stnp` +only when LLVM can self-pair the value — i64/u64 splits to a `w`-pair via +`lsr` and gets `stnp w, w`, while i32/u32 and i16/u16 lower to plain +`str`/`strh` with the NT hint silently dropped. **For aarch64 NT semantics, +prefer 64-bit-or-wider element widths.** See the target-lowering table in +`docs/src/reference/intrinsics.md` for the full matrix and the +why-no-scalar-stnp explanation. + +Concrete consumer pulling: Olorin's `q4k_repack.ea` (pure-streaming block- +layout repack blocked on scalar surface; pairs 16-byte fields, so i64 +writes get the aarch64 win). ### `fence_nt()` intrinsic diff --git a/docs/src/reference/intrinsics.md b/docs/src/reference/intrinsics.md index 3885252..b30decc 100644 --- a/docs/src/reference/intrinsics.md +++ b/docs/src/reference/intrinsics.md @@ -71,19 +71,36 @@ stream_store(out, i, v) // vector form — v is f32xN, i32xN, etc. stream_store(out, i, scalar_value) // scalar form (v1.15.0) — i16/u16/i32/u32/i64/u64 ``` -**Target lowering:** +**Target lowering** (verified on LLVM 18.1.8, x86_64 Zen 4 and aarch64 Cortex-A76): | Width / form | x86 | aarch64 | |---|---|---| -| Vector 128-bit (f32x4, i32x4, ...) | `movntps` / `movntdq` | regular `str q` (LLVM 18 does not optimize `!nontemporal` for single-register NEON) | +| Vector 128-bit (f32x4, i32x4, ...) | `movntps` / `movntdq` | `stnp d, d, [x]` (LLVM splits the 128-bit q-register into a d-pair to use the only available aarch64 NT store) | | Vector 256-bit (AVX2) | `vmovntps` / `vmovntdq` | n/a (256-bit not on NEON) | | Vector 512-bit (AVX-512) | `vmovntps` / `vmovntdq` zmm | n/a | -| Scalar i32 / u32 | `movnti` (SSE2) | `str` / paired `stnp` when alignment permits | -| Scalar i64 / u64 | `movnti` (SSE2, 64-bit mode) | `str` / paired `stnp` | -| Scalar i16 / u16 | regular `mov` (no NT scalar at 16-bit width) | `strh` / paired `stnp` | - -The i16/u16 case ships for shape symmetry with the wider scalars — the -non-temporal hint becomes a no-op on x86 at that width. +| Scalar i64 / u64 | `movnti` (SSE2, 64-bit mode) | `stnp w, w, [x]` (LLVM splits the i64 into a w-pair to use `stnp`; emits an `lsr` for the high half) | +| Scalar i32 / u32 | `movnti` (SSE2) | plain `str` — NT hint silently dropped | +| Scalar i16 / u16 | regular `mov` — NT hint silently dropped | plain `strh` — NT hint silently dropped | + +**aarch64 has no scalar non-temporal store instruction.** The only NT store on +aarch64 is `stnp` (Store Non-temporal Pair), which requires two operands. LLVM +18 honors `!nontemporal` only when it can synthesize an `stnp`: + +- **64-bit scalars (i64/u64)** self-pair to a `w` register pair — NT hint preserved. +- **128-bit vectors** self-pair to a `d` register pair — NT hint preserved. +- **32-bit and 16-bit scalars** have no pair-friendly form — LLVM emits plain + `str` / `strh` and the NT hint is dropped silently. + +For aarch64 NT semantics, **prefer 64-bit or wider element widths.** The i32/u32 +and i16/u16 scalar overloads still type-check and run, but provide no cache- +bypass benefit on aarch64; the same is true of i16/u16 on x86 (no `movnti16` +exists). They ship for cross-platform shape symmetry — a single Eä kernel using +`stream_store` compiles and runs on both targets without per-width branching. + +Note also that LLVM 18 does **not** fuse two adjacent `stream_store(*mut i32, ...)` +calls into a single `stnp` pair on aarch64 — they lower to a regular `stp` with +the NT hint dropped. If you need NT-paired stores on aarch64, write the data as +i64 (two 32-bit values packed) or as a vector type. **Alignment contract:** diff --git a/tests/fence_nt_tests.rs b/tests/fence_nt_tests.rs index d9023a6..dac09d2 100644 --- a/tests/fence_nt_tests.rs +++ b/tests/fence_nt_tests.rs @@ -131,4 +131,19 @@ mod tests { &["sfence"], ); } + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_fence_nt_emits_dmb_ishst_aarch64() { + // Pi 5-verified: fence_nt() lowers to `dmb ishst` via the + // @llvm.aarch64.dmb intrinsic with operand 10 (ishst encoding). + assert_intrinsic_in_disassembly( + r#" + export func test() { + fence_nt() + } + "#, + &["dmb\tishst"], + ); + } } diff --git a/tests/stream_store_tests.rs b/tests/stream_store_tests.rs index 040c07d..3eea71c 100644 --- a/tests/stream_store_tests.rs +++ b/tests/stream_store_tests.rs @@ -368,4 +368,74 @@ mod tests { is not silently substituting an aligned movups." ); } + + // --- aarch64 objdump assertions (Pi 5-verified mnemonics, LLVM 18.1.8) --- + // + // aarch64 has no scalar non-temporal store instruction. LLVM honors + // !nontemporal only when it can synthesize stnp (Store Non-temporal Pair) — + // that is, for 64-bit operands (self-pair to w-pair) and 128-bit vectors + // (q-register splits to d-pair). For 32-bit and 16-bit scalars, LLVM + // silently emits plain `str` / `strh` with the NT hint dropped. The four + // tests below pin the actually-observed LLVM 18 behavior; a future LLVM + // upgrade that changes any of these emissions will fail loudly rather than + // silently. + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_stream_store_scalar_i64_emits_stnp_aarch64() { + // i64 self-pairs to a w-register pair via lsr; emits `stnp w, w`. + assert_intrinsic_in_disassembly( + r#" + export func test(out: *mut i64, v: i64) { + stream_store(out, 0, v) + } + "#, + &["stnp"], + ); + } + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_stream_store_f32x4_emits_stnp_aarch64() { + // 128-bit q-register splits to a d-pair; emits `stnp d, d`. + assert_intrinsic_in_disassembly( + r#" + export func test(data: *f32, out: *mut f32) { + let v: f32x4 = load(data, 0) + stream_store(out, 0, v) + } + "#, + &["stnp"], + ); + } + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_stream_store_scalar_i32_emits_plain_str_aarch64() { + // PIN: NT hint silently dropped on aarch64 for 32-bit scalar stores. + // If LLVM ever starts synthesizing stnp for i32 stream_store, this + // test fails — forcing the docs to be updated to reflect new behavior. + assert_intrinsic_in_disassembly( + r#" + export func test(out: *mut i32, v: i32) { + stream_store(out, 0, v) + } + "#, + &["str\tw"], + ); + } + + #[test] + #[cfg(target_arch = "aarch64")] + fn test_stream_store_scalar_i16_emits_plain_strh_aarch64() { + // PIN: same as i32 — NT hint dropped, plain `strh w, [x]` emitted. + assert_intrinsic_in_disassembly( + r#" + export func test(out: *mut i16, v: i16) { + stream_store(out, 0, v) + } + "#, + &["strh"], + ); + } }