diff --git a/crates/cuda-intrinsics-gen/src/render/importer.rs b/crates/cuda-intrinsics-gen/src/render/importer.rs index 1fcc06c9c7..78478f8c59 100644 --- a/crates/cuda-intrinsics-gen/src/render/importer.rs +++ b/crates/cuda-intrinsics-gen/src/render/importer.rs @@ -38,6 +38,14 @@ use crate::render::reference::{ use std::fmt::Write as _; use std::path::PathBuf; +fn render_prepare_destination_write(output: &mut String, prev_op: &str) { + writeln!( + output, + " let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write(\n ctx, body, destination, value_map, block_ptr, {prev_op}, loc.clone(),\n )?;" + ) + .unwrap(); +} + fn render_importer_pure_value_dispatch( output: &mut String, catalog: &CatalogFile, @@ -79,12 +87,13 @@ fn render_importer_pure_value_dispatch( intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(output, "last_op"); output.push_str( - " helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n let result = intrinsic.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op);\n let result = intrinsic.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -121,13 +130,14 @@ fn render_importer_scalar_conversion_dispatch( intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(output, "last_op"); output.push_str( - " helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n\ + " helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op);\n\ let result = intrinsic.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -179,13 +189,14 @@ fn render_importer_scalar_arithmetic_dispatch( intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(output, "last_op"); output.push_str( - " helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n\ + " helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op);\n\ let result = intrinsic.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -228,13 +239,14 @@ fn render_importer_scalar_math_dispatch( intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(output, "last_op"); output.push_str( - " helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n\ + " helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op);\n\ let result = intrinsic.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -277,13 +289,14 @@ fn render_importer_extended_minmax_dispatch( intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(output, "last_op"); output.push_str( - " helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n\ + " helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op);\n\ let result = intrinsic.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -340,8 +353,9 @@ fn render_importer_elect_dispatch( intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(output, "last_op"); output.push_str( - " helpers::insert_op(ctx, elect, block_ptr, last_op);\n\ + " helpers::insert_op(ctx, elect, block_ptr, prepared_last_op);\n\ let leader = elect.deref(ctx).get_result(0);\n\ let elected = elect.deref(ctx).get_result(1);\n\ let tuple = Operation::new(\n\ @@ -355,7 +369,7 @@ fn render_importer_elect_dispatch( ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, tuple, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, tuple, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -657,7 +671,10 @@ fn render_importer_tcgen05_non_mma_dispatch( intrinsic_marker(catalog, record) ) .unwrap(); - output.push_str(" helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n"); + render_prepare_destination_write(output, "last_op"); + output.push_str( + " helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op);\n", + ); if count == 1 { output.push_str( " let result = intrinsic.deref(ctx).get_result(0);\n let value = intrinsic;\n", @@ -679,7 +696,7 @@ fn render_importer_tcgen05_non_mma_dispatch( } writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, value, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, value, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -774,7 +791,7 @@ fn sreg_arms(catalog: &CatalogFile) -> String { writeln!(output, " Ok(Some(helpers::{helper}(").unwrap(); writeln!( output, - " ctx, {}::get_concrete_op_info(), {:?}, destination, target, block_ptr,", + " ctx, body, {}::get_concrete_op_info(), {:?}, destination, target, block_ptr,", record.dialect.op_type, intrinsic_marker(catalog, record) ) @@ -808,12 +825,13 @@ fn active_mask_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "prev_op"); output.push_str( - " helpers::insert_op(ctx, active_mask, block_ptr, prev_op);\n let result = active_mask.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, active_mask, block_ptr, prepared_last_op);\n let result = active_mask.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, active_mask, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, active_mask, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -848,7 +866,9 @@ fn ldmatrix_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); - output.push_str(" helpers::insert_op(ctx, load, block_ptr, last_op);\n"); + render_prepare_destination_write(&mut output, "last_op"); + output + .push_str(" helpers::insert_op(ctx, load, block_ptr, prepared_last_op);\n"); if register_count == 1 { output.push_str(" let value = load.deref(ctx).get_result(0);\n"); output.push_str(" let last_op = load;\n"); @@ -861,7 +881,7 @@ fn ldmatrix_arms(catalog: &CatalogFile) -> String { } writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, value, target, block_ptr, last_op, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, value, target, block_ptr, last_op, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -992,12 +1012,13 @@ fn register_mma_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "Some(last_op)"); output.push_str( - " helpers::insert_op(ctx, mma, block_ptr, Some(last_op));\n let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone());\n", + " helpers::insert_op(ctx, mma, block_ptr, prepared_last_op);\n let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone());\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, last_op, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, last_op, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1056,12 +1077,13 @@ fn sparse_mma_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, mma, block_ptr, last_op);\n let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone());\n", + " helpers::insert_op(ctx, mma, block_ptr, prepared_last_op);\n let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone());\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, last_op, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, last_op, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1101,11 +1123,13 @@ fn packed_atomic_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); - output.push_str(" helpers::insert_op(ctx, atom, block_ptr, last_op);\n"); + render_prepare_destination_write(&mut output, "last_op"); + output + .push_str(" helpers::insert_op(ctx, atom, block_ptr, prepared_last_op);\n"); output.push_str(" let value = atom.deref(ctx).get_result(0);\n"); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, value, target, block_ptr, atom, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, value, target, block_ptr, atom, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1146,12 +1170,13 @@ fn redux_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, reduction, block_ptr, last_op);\n let result = reduction.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op);\n let result = reduction.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, reduction, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, reduction, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1192,12 +1217,13 @@ fn vote_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, vote, block_ptr, last_op);\n let result = vote.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, vote, block_ptr, prepared_last_op);\n let result = vote.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, vote, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, vote, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1234,12 +1260,13 @@ fn warp_match_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, warp_match, block_ptr, last_op);\n let result = warp_match.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, warp_match, block_ptr, prepared_last_op);\n let result = warp_match.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, warp_match, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, warp_match, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1335,12 +1362,13 @@ fn warp_shuffle_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, shuffle, block_ptr, last_op);\n let result = shuffle.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op);\n let result = shuffle.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, shuffle, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, shuffle, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1380,12 +1408,13 @@ fn dotprod_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, dot, block_ptr, last_op);\n let result = dot.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, dot, block_ptr, prepared_last_op);\n let result = dot.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, dot, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, dot, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1505,12 +1534,13 @@ fn prmt_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, prmt, block_ptr, last_op);\n let result = prmt.deref(ctx).get_result(0);\n", + " helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op);\n let result = prmt.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, prmt, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, prmt, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1623,13 +1653,14 @@ fn cluster_memory_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); + render_prepare_destination_write(&mut output, "last_op"); output.push_str( - " helpers::insert_op(ctx, cluster, block_ptr, last_op);\n\ + " helpers::insert_op(ctx, cluster, block_ptr, prepared_last_op);\n\ let result = cluster.deref(ctx).get_result(0);\n", ); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, cluster, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, cluster, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); @@ -1783,16 +1814,21 @@ fn clc_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); - output.push_str(" helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n"); if query { + render_prepare_destination_write(&mut output, "last_op"); + output.push_str( + " helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op);\n", + ); output.push_str(" let result = intrinsic.deref(ctx).get_result(0);\n"); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, intrinsic, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); } else { + output + .push_str(" helpers::insert_op(ctx, intrinsic, block_ptr, last_op);\n"); output.push_str(" if let Some(target_idx) = target {\n Ok(Some(helpers::emit_goto(ctx, *target_idx, intrinsic, block_map, loc)))\n } else {\n"); writeln!( output, @@ -2113,16 +2149,20 @@ fn mbarrier_basic_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); - output.push_str(" helpers::insert_op(ctx, mbarrier, block_ptr, last_op);\n"); if returns_value { + render_prepare_destination_write(&mut output, "last_op"); + output.push_str( + " helpers::insert_op(ctx, mbarrier, block_ptr, prepared_last_op);\n", + ); output.push_str(" let result = mbarrier.deref(ctx).get_result(0);\n"); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, mbarrier, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, mbarrier, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); } else { + output.push_str(" helpers::insert_op(ctx, mbarrier, block_ptr, last_op);\n"); output.push_str( " if let Some(target_idx) = target {\n Ok(Some(helpers::emit_goto(ctx, *target_idx, mbarrier, block_map, loc)))\n } else {\n", ); @@ -2199,16 +2239,20 @@ fn mbarrier_extended_arms(catalog: &CatalogFile) -> String { intrinsic_marker(catalog, record) ) .unwrap(); - output.push_str(" helpers::insert_op(ctx, extended, block_ptr, last_op);\n"); if returns_value { + render_prepare_destination_write(&mut output, "last_op"); + output.push_str( + " helpers::insert_op(ctx, extended, block_ptr, prepared_last_op);\n", + ); output.push_str(" let result = extended.deref(ctx).get_result(0);\n"); writeln!( output, - " Ok(Some(helpers::emit_store_result_and_goto(\n ctx, destination, result, target, block_ptr, extended, value_map, block_map, loc,\n {:?},\n )?))", + " Ok(Some(helpers::emit_prepared_result_and_goto(\n ctx, prepared_destination, result, target, block_ptr, extended, value_map, block_map, loc,\n {:?},\n )?))", format!("{} call without target block", record.rust.name) ) .unwrap(); } else { + output.push_str(" helpers::insert_op(ctx, extended, block_ptr, last_op);\n"); output.push_str( " if let Some(target_idx) = target {\n Ok(Some(helpers::emit_goto(ctx, *target_idx, extended, block_map, loc)))\n } else {\n", ); diff --git a/crates/cuda-intrinsics-gen/src/render/tests/scalar_misc.rs b/crates/cuda-intrinsics-gen/src/render/tests/scalar_misc.rs index 75e1e47783..b947253880 100644 --- a/crates/cuda-intrinsics-gen/src/render/tests/scalar_misc.rs +++ b/crates/cuda-intrinsics-gen/src/render/tests/scalar_misc.rs @@ -667,7 +667,10 @@ fn clc_rendering_preserves_api_and_uses_typed_llvm_routes() { assert!(importer.contains("cuda_device::clc::clc_try_cancel")); assert!(importer.contains("ClcTryCancelOp::get_concrete_op_info()")); assert!(importer.contains("ClcQueryIsCanceledOp::get_concrete_op_info()")); - assert!(importer.contains("helpers::emit_store_result_and_goto")); + // Destination is prepared before the intrinsic op is built, then the + // result is written through it: two typed helper routes, in that order. + assert!(importer.contains("helpers::prepare_destination_write(")); + assert!(importer.contains("helpers::emit_prepared_result_and_goto(")); let lowering = render_lowering(&catalog, "test-hash"); assert!(lowering.contains("clc::convert_generated_clc_query")); diff --git a/crates/mir-importer/src/translator/payload_store.rs b/crates/mir-importer/src/translator/payload_store.rs index e5521f2ea4..dc892a5f30 100644 --- a/crates/mir-importer/src/translator/payload_store.rs +++ b/crates/mir-importer/src/translator/payload_store.rs @@ -72,6 +72,13 @@ pub(crate) struct CanonicalPayloadStore { pub(crate) field: usize, } +/// A canonical enum-payload destination whose enclosing enum address was +/// materialized before a call. +pub(crate) struct PreparedCanonicalPayloadStore { + enum_ptr: Value, + store: CanonicalPayloadStore, +} + /// Classify an assignment destination. /// /// `None` for every destination the ordinary address path still owns: a @@ -118,6 +125,63 @@ pub(crate) fn classify( })) } +/// Materialize the enclosing enum address for a canonical payload call +/// destination. +/// +/// Keeping the typed payload metadata together with the address lets the +/// result write happen after the callee without re-evaluating the MIR place. +#[allow(clippy::too_many_arguments)] +pub(crate) fn prepare_call_store( + ctx: &mut Context, + body: &mir::Body, + value_map: &ValueMap, + store: CanonicalPayloadStore, + block_ptr: Ptr, + prev_op: Option>, + loc: Location, +) -> TranslationResult>)>> { + let Some((enum_ptr, prev)) = rvalue::translate_place_address( + ctx, + body, + value_map, + &store.enum_place, + /* is_mutable */ true, + block_ptr, + prev_op, + loc, + )? + else { + return Ok(None); + }; + + Ok(Some(( + PreparedCanonicalPayloadStore { enum_ptr, store }, + prev, + ))) +} + +/// Rebuild a canonical enum payload through an address prepared before a call. +pub(crate) fn rebuild_and_store_prepared( + ctx: &mut Context, + prepared: &PreparedCanonicalPayloadStore, + new_value: Value, + block_ptr: Ptr, + prev_op: Option>, + loc: Location, +) -> TranslationResult> { + let enum_ty = types::translate_type(ctx, &prepared.store.enum_rust_ty)?; + rebuild_and_store_at_ptr( + ctx, + &prepared.store, + prepared.enum_ptr, + enum_ty, + new_value, + block_ptr, + prev_op, + loc, + ) +} + /// Rebuild the enum around `new_value` and store it back. /// /// `Ok(None)` when the enum place has no address to load from and store to, @@ -149,7 +213,23 @@ pub(crate) fn rebuild_and_store( return Ok(None); }; - let (load_op, enum_value) = emit_load(ctx, enum_ptr, enum_ty, block_ptr, prev, loc.clone()); + Ok(Some(Some(rebuild_and_store_at_ptr( + ctx, store, enum_ptr, enum_ty, new_value, block_ptr, prev, loc, + )?))) +} + +#[allow(clippy::too_many_arguments)] +fn rebuild_and_store_at_ptr( + ctx: &mut Context, + store: &CanonicalPayloadStore, + enum_ptr: Value, + enum_ty: TypeHandle, + new_value: Value, + block_ptr: Ptr, + prev_op: Option>, + loc: Location, +) -> TranslationResult> { + let (load_op, enum_value) = emit_load(ctx, enum_ptr, enum_ty, block_ptr, prev_op, loc.clone()); let mut prev = Some(load_op); let field_count = { @@ -171,7 +251,6 @@ pub(crate) fn rebuild_and_store( } }; - // Every field of the variant, with the assigned one replaced. let mut payload_values = Vec::with_capacity(field_count); for field in 0..field_count { if field == store.field { @@ -213,8 +292,14 @@ pub(crate) fn rebuild_and_store( } let rebuilt = construct.deref(ctx).get_result(0); - let store_op = emit_store(ctx, rebuilt, enum_ptr, block_ptr, Some(construct), loc); - Ok(Some(Some(store_op))) + Ok(emit_store( + ctx, + rebuilt, + enum_ptr, + block_ptr, + Some(construct), + loc, + )) } /// The Rust type of one payload field, read from the enum's own definition. diff --git a/crates/mir-importer/src/translator/terminator/helpers.rs b/crates/mir-importer/src/translator/terminator/helpers.rs index e039b8a5f1..a85b623374 100644 --- a/crates/mir-importer/src/translator/terminator/helpers.rs +++ b/crates/mir-importer/src/translator/terminator/helpers.rs @@ -8,8 +8,10 @@ //! This module contains utility functions shared across terminator handlers: //! //! - [`emit_goto`]: Unconditional zero-operand branch to a target block. -//! - [`emit_store_result_and_goto`]: Write an intrinsic result to the -//! destination local's slot, then branch to the success target. +//! - [`prepare_destination_write`]: Materialize a call/intrinsic destination +//! before its result-producing operation executes. +//! - [`emit_prepared_result_and_goto`]: Write through a prepared destination +//! and branch to the success target. //! - [`emit_function_call`]: General function call emission. //! - [`emit_generated_nvvm_intrinsic`]: Zero-operand NVVM intrinsic emission //! for a catalog intrinsic, carrying its generated ABI marker. @@ -17,8 +19,8 @@ //! - [`insert_op`]: Common operation insertion pattern. use crate::error::{TranslationErr, TranslationResult}; -use crate::translator::rvalue; use crate::translator::values::{ValueMap, establish_declared_pointer_type}; +use crate::translator::{payload_store, rvalue}; use dialect_mir::{ attributes::MirPointerKindAuthorityAttr, ops::{MirCallOp, MirConstructArrayOp, MirGotoOp}, @@ -61,227 +63,154 @@ pub fn emit_goto( goto_op } -/// Writes `value` into `destination`, honouring a projection on it. +/// A call destination whose observable projection state has already been +/// evaluated. /// -/// A bare local goes to its slot, as before. A projected destination needs the -/// address of the place rather than of the local, because storing to the local -/// would overwrite the whole aggregate (or the pointer itself) instead of the -/// part the call names. -/// -/// Only the single-element projections a call destination is observed to carry -/// are modelled. Anything else is refused rather than written to the wrong -/// place, since a store aimed at the wrong address is a miscompile and an -/// unsupported-construct error is not. +/// Calls prepare this value before the callee executes and only perform the +/// final write afterward. This matches MIR call semantics even when the callee +/// mutates locals used by the destination projection. +pub(crate) enum PreparedDestinationWrite { + /// A bare local. Local storage remains responsible for pointer + /// representation coercions and ZST handling. + Local(mir::Local), + /// An ordinary projected place whose writable address is fixed already. + Address(Value), + /// An enum payload that must be rebuilt rather than written through a raw + /// payload pointer. + CanonicalPayload(payload_store::PreparedCanonicalPayloadStore), +} + +fn require_writable_destination_address( + translated: Option<(Value, Option>)>, + projection: &dyn std::fmt::Debug, + loc: Location, +) -> TranslationResult<(Value, Option>)> { + let Some(address) = translated else { + return input_err!( + loc, + TranslationErr::unsupported(format!( + "cannot prepare writable call destination {projection:?}" + )) + ); + }; + Ok(address) +} + +/// Evaluate a call destination before the callee executes. /// -/// Known fidelity gap: rustc evaluates the destination address *before* the -/// call, but this path materializes it *after* the call op. The difference is -/// observable only from custom MIR where the callee mutates the destination's -/// base local through a `&mut` argument. -pub fn store_result_to_place( +/// Bare locals require no address materialization. Canonical enum payloads +/// prepare the enclosing enum address, while every other projected place uses +/// the shared writable place-address walker. Unsupported destinations fail +/// before a call can be emitted with ambiguous store semantics. +#[allow(clippy::too_many_arguments)] +pub(crate) fn prepare_destination_write( ctx: &mut Context, body: &mir::Body, destination: &mir::Place, + value_map: &ValueMap, + block_ptr: Ptr, + prev_op: Option>, + loc: Location, +) -> TranslationResult<(PreparedDestinationWrite, Option>)> { + if destination.projection.is_empty() { + return Ok((PreparedDestinationWrite::Local(destination.local), prev_op)); + } + + if let Some(payload) = payload_store::classify(ctx, body, destination)? { + let Some((prepared, prepared_prev)) = payload_store::prepare_call_store( + ctx, + body, + value_map, + payload, + block_ptr, + prev_op, + loc.clone(), + )? + else { + return input_err!( + loc, + TranslationErr::unsupported(format!( + "cannot prepare canonical payload call destination {:?}", + destination.projection + )) + ); + }; + return Ok(( + PreparedDestinationWrite::CanonicalPayload(prepared), + prepared_prev.or(prev_op), + )); + } + + let translated_address = rvalue::translate_place_address( + ctx, + body, + value_map, + destination, + /* is_mutable */ true, + block_ptr, + prev_op, + loc.clone(), + )?; + let (address, address_prev) = + require_writable_destination_address(translated_address, &destination.projection, loc)?; + + Ok(( + PreparedDestinationWrite::Address(address), + address_prev.or(prev_op), + )) +} + +/// Store a call result through a destination prepared before the callee ran. +pub(crate) fn finish_destination_write( + ctx: &mut Context, + destination: PreparedDestinationWrite, value: Value, value_map: &mut ValueMap, block_ptr: Ptr, prev_op: Ptr, loc: Location, ) -> TranslationResult> { - use crate::translator::statement::{ - emit_array_element_store, pointer_address_space, pointer_is_mutable, - reject_raw_field_index_on_enum_pointee, slot_array_element_ty, - }; - use dialect_mir::ops::{MirFieldAddrOp, MirStoreOp}; + use dialect_mir::ops::MirStoreOp; - match destination.projection.as_slice() { - [] => Ok(value_map - .store_local(ctx, destination.local, value, block_ptr, Some(prev_op)) + match destination { + PreparedDestinationWrite::Local(local) => Ok(value_map + .store_local(ctx, local, value, block_ptr, Some(prev_op)) .unwrap_or(prev_op)), - - [mir::ProjectionElem::Deref] => { - let base_place = mir::Place { - local: destination.local, - projection: vec![], - }; - let (ptr_val, after_ptr) = rvalue::translate_place( - ctx, - body, - &base_place, - value_map, - block_ptr, - Some(prev_op), - loc.clone(), - )?; - let store_op = Operation::new( - ctx, - MirStoreOp::get_concrete_op_info(), - vec![], - vec![ptr_val, value], - vec![], - 0, - ); - store_op.deref_mut(ctx).set_loc(loc); - store_op.insert_after(ctx, after_ptr.unwrap_or(prev_op)); - Ok(store_op) - } - - [mir::ProjectionElem::Field(field_idx, field_ty)] => { - let Some(slot) = value_map.get_slot(destination.local) else { - return input_err!( - loc, - TranslationErr::unsupported(format!( - "Local {:?} has no alloca slot for a field destination", - destination.local - )) - ); - }; - reject_raw_field_index_on_enum_pointee(ctx, slot, &destination.projection, &loc)?; - - let field_type = crate::translator::types::translate_type(ctx, field_ty)?; - let field_ptr_ty = dialect_mir::types::MirPtrType::get( - ctx, - field_type, - pointer_is_mutable(ctx, slot), - pointer_address_space(ctx, slot), - ) - .into(); - let field_addr_op = MirFieldAddrOp::build(ctx, slot, field_ptr_ty, *field_idx as u32)?; - field_addr_op.deref_mut(ctx).set_loc(loc.clone()); - field_addr_op.insert_after(ctx, prev_op); - let field_ptr = field_addr_op.deref(ctx).get_result(0); - + PreparedDestinationWrite::Address(address) => { let store_op = Operation::new( ctx, MirStoreOp::get_concrete_op_info(), vec![], - vec![field_ptr, value], + vec![address, value], vec![], 0, ); store_op.deref_mut(ctx).set_loc(loc); - store_op.insert_after(ctx, field_addr_op); + store_op.insert_after(ctx, prev_op); Ok(store_op) } - - [mir::ProjectionElem::Index(index_local)] => { - let Some(arr_ptr) = value_map.get_slot(destination.local) else { - return input_err!( - loc, - TranslationErr::unsupported(format!( - "Local {:?} has no alloca slot for a runtime index destination", - destination.local - )) - ); - }; - let index_place = mir::Place { - local: *index_local, - projection: vec![], - }; - let (index_value, after_index) = rvalue::translate_place( - ctx, - body, - &index_place, - value_map, - block_ptr, - Some(prev_op), - loc.clone(), - )?; - let (element_ty, address_space) = slot_array_element_ty(ctx, arr_ptr, &loc)?; - Ok(emit_array_element_store( - ctx, - arr_ptr, - index_value, - value, - element_ty, - address_space, - block_ptr, - after_index, - loc, - )) - } - - [ - mir::ProjectionElem::ConstantIndex { - offset, - min_length: _, - from_end, - }, - ] => { - if *from_end { - return input_err!( - loc, - TranslationErr::unsupported( - "ConstantIndex with from_end=true is not supported for a call destination" - .to_string() - ) - ); - } - let Some(arr_ptr) = value_map.get_slot(destination.local) else { - return input_err!( - loc, - TranslationErr::unsupported(format!( - "Local {:?} has no alloca slot for an array element destination", - destination.local - )) - ); - }; - let (element_ty, address_space) = slot_array_element_ty(ctx, arr_ptr, &loc)?; - - let i64_ty = IntegerType::get(ctx, 64, Signedness::Signed); - let index_attr = pliron::builtin::attributes::IntegerAttr::new( - i64_ty, - pliron::utils::apint::APInt::from_i64( - *offset as i64, - std::num::NonZeroUsize::new(64).unwrap(), - ), - ); - let const_op = Operation::new( + PreparedDestinationWrite::CanonicalPayload(payload) => { + payload_store::rebuild_and_store_prepared( ctx, - dialect_mir::ops::MirConstantOp::get_concrete_op_info(), - vec![i64_ty.into()], - vec![], - vec![], - 0, - ); - const_op.deref_mut(ctx).set_loc(loc.clone()); - dialect_mir::ops::MirConstantOp::new(const_op).set_attr_value(ctx, index_attr); - const_op.insert_after(ctx, prev_op); - let index_value = const_op.deref(ctx).get_result(0); - - Ok(emit_array_element_store( - ctx, - arr_ptr, - index_value, + &payload, value, - element_ty, - address_space, block_ptr, - Some(const_op), + Some(prev_op), loc, - )) + ) } - - projection => input_err!( - loc, - TranslationErr::unsupported(format!( - "call destination with projection {:?} is not supported", - projection - )) - ), } } -/// Stores `result_value` into `destination`'s slot and emits a branch to -/// `target`. +/// Store a result through a destination prepared before the producer ran, +/// then branch to the success target. /// -/// Shared "write result + branch to success block" epilogue for intrinsic -/// handlers. The store is emitted after `prev_op`; the goto chains after the -/// store (or after `prev_op` directly if the destination is a ZST with no -/// backing slot). Returns the goto operation. +/// Generated intrinsic dispatch uses this epilogue after materializing the +/// destination before inserting the result-producing operation. #[allow(clippy::too_many_arguments)] -pub fn emit_store_result_and_goto( +pub(crate) fn emit_prepared_result_and_goto( ctx: &mut Context, - destination: &mir::Place, + destination: PreparedDestinationWrite, result_value: Value, target: &Option, block_ptr: Ptr, @@ -291,42 +220,20 @@ pub fn emit_store_result_and_goto( loc: Location, no_target_msg: &str, ) -> TranslationResult> { - // This epilogue has no `body`, so it cannot compute the address of a - // projected place the way [`store_result_to_place`] does. Refusing is the - // alternative to storing to `destination.local`, which for `x.0 = f()` or - // `(*p) = f()` writes the result over the whole aggregate or over the - // pointer. Most callers are generated, so the signature stays as it is. - if !destination.projection.is_empty() { - return input_err!( - loc, - TranslationErr::unsupported(format!( - "intrinsic result written to a projected destination {:?} is not supported", - destination.projection - )) - ); - } - - // Every pointer-producing intrinsic must establish its exact Rust result - // kind at the producer. This shared epilogue is intentionally ordinary - // local storage: it may preserve or erase provenance, but it must never - // manufacture a concrete pointer/reference kind for an emitter. - let goto_prev = value_map - .store_local( - ctx, - destination.local, - result_value, - block_ptr, - Some(prev_op), - ) - .unwrap_or(prev_op); + let goto_prev = finish_destination_write( + ctx, + destination, + result_value, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; if let Some(target_idx) = target { Ok(emit_goto(ctx, *target_idx, goto_prev, block_map, loc)) } else { - input_err!( - loc.clone(), - TranslationErr::unsupported(no_target_msg.to_string()) - ) + input_err!(loc, TranslationErr::unsupported(no_target_msg.to_string())) } } @@ -409,9 +316,10 @@ pub fn bundle_generated_u32_results_as_array( /// 1. Translate all MIR arguments to Pliron IR values /// 2. At a foreign ABI boundary, adapt pointer address spaces to the exact /// declared parameter types without changing pointer kind or mutability -/// 3. Create a `mir.call` operation carrying the callee's name attribute -/// 4. Store the result into the destination local's slot -/// 5. Emit a zero-operand goto to the call's success target +/// 3. Prepare the call destination using the current projection state +/// 4. Create a `mir.call` operation carrying the callee's name attribute +/// 5. Store the result through the prepared destination +/// 6. Emit a zero-operand goto to the call's success target /// /// Reference arguments (`&mut local`) are handed the local's alloca slot /// pointer directly, so callee writes through the reference are observed by @@ -475,6 +383,17 @@ pub fn emit_function_call( } } + let (prepared_destination, prepared_last_op) = prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + last_op = prepared_last_op; + use pliron::builtin::attributes::StringAttr; let call_op = Operation::new( @@ -506,10 +425,9 @@ pub fn emit_function_call( let result_value = call_op.deref(ctx).get_result(0); - let goto_prev = store_result_to_place( + let goto_prev = finish_destination_write( ctx, - body, - destination, + prepared_destination, result_value, value_map, block_ptr, @@ -578,6 +496,7 @@ fn normalize_foreign_pointer_argument( #[allow(clippy::too_many_arguments)] pub fn emit_generated_nvvm_intrinsic( ctx: &mut Context, + body: &mir::Body, opid: ( fn(pliron::context::Ptr) -> pliron::op::OpObj, std::any::TypeId, @@ -593,6 +512,7 @@ pub fn emit_generated_nvvm_intrinsic( ) -> TranslationResult> { emit_nvvm_integer_intrinsic( ctx, + body, opid, 32, Some(marker), @@ -611,6 +531,7 @@ pub fn emit_generated_nvvm_intrinsic( #[allow(clippy::too_many_arguments)] pub fn emit_generated_nvvm_intrinsic_u64( ctx: &mut Context, + body: &mir::Body, opid: ( fn(pliron::context::Ptr) -> pliron::op::OpObj, std::any::TypeId, @@ -626,6 +547,7 @@ pub fn emit_generated_nvvm_intrinsic_u64( ) -> TranslationResult> { emit_nvvm_integer_intrinsic( ctx, + body, opid, 64, Some(marker), @@ -642,6 +564,7 @@ pub fn emit_generated_nvvm_intrinsic_u64( #[allow(clippy::too_many_arguments)] fn emit_nvvm_integer_intrinsic( ctx: &mut Context, + body: &mir::Body, opid: ( fn(pliron::context::Ptr) -> pliron::op::OpObj, std::any::TypeId, @@ -658,13 +581,23 @@ fn emit_nvvm_integer_intrinsic( ) -> TranslationResult> { let result_type = IntegerType::get(ctx, result_width, Signedness::Unsigned); + let (prepared_destination, prepared_last_op) = prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + let nvvm_op = Operation::new(ctx, opid, vec![result_type.to_handle()], vec![], vec![], 0); nvvm_op.deref_mut(ctx).set_loc(loc.clone()); if let Some(marker) = generated_marker { set_generated_intrinsic_marker(ctx, nvvm_op, marker); } - let last_op = if let Some(prev) = prev_op { + let last_op = if let Some(prev) = prepared_last_op { nvvm_op.insert_after(ctx, prev); nvvm_op } else { @@ -674,15 +607,15 @@ fn emit_nvvm_integer_intrinsic( let result_value = nvvm_op.deref(ctx).get_result(0); - let goto_prev = value_map - .store_local( - ctx, - destination.local, - result_value, - block_ptr, - Some(last_op), - ) - .unwrap_or(last_op); + let goto_prev = finish_destination_write( + ctx, + prepared_destination, + result_value, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; if let Some(target_idx) = target { Ok(emit_goto(ctx, *target_idx, goto_prev, block_map, loc)) @@ -702,6 +635,7 @@ fn emit_nvvm_integer_intrinsic( #[allow(clippy::too_many_arguments)] pub fn emit_unit_noop_intrinsic( ctx: &mut Context, + body: &mir::Body, destination: &mir::Place, target: &Option, block_ptr: Ptr, @@ -711,6 +645,16 @@ pub fn emit_unit_noop_intrinsic( loc: Location, intrinsic_name: &str, ) -> TranslationResult> { + let (prepared_destination, prepared_last_op) = prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + let unit_ty = dialect_mir::types::MirTupleType::get(ctx, vec![]); let unit_op = Operation::new( ctx, @@ -721,24 +665,22 @@ pub fn emit_unit_noop_intrinsic( 0, ); unit_op.deref_mut(ctx).set_loc(loc.clone()); - insert_op(ctx, unit_op, block_ptr, prev_op); + insert_op(ctx, unit_op, block_ptr, prepared_last_op); let unit_val = unit_op.deref(ctx).get_result(0); - let goto_prev = value_map - .store_local(ctx, destination.local, unit_val, block_ptr, Some(unit_op)) - .unwrap_or(unit_op); - - if let Some(target_idx) = target { - Ok(emit_goto(ctx, *target_idx, goto_prev, block_map, loc)) - } else { - input_err!( - loc.clone(), - TranslationErr::unsupported(format!( - "{} call without target not supported", - intrinsic_name - )) - ) - } + let no_target_msg = format!("{} call without target not supported", intrinsic_name); + emit_prepared_result_and_goto( + ctx, + prepared_destination, + unit_val, + target, + block_ptr, + unit_op, + value_map, + block_map, + loc, + &no_target_msg, + ) } #[cfg(test)] @@ -763,6 +705,78 @@ mod tests { r#type::TypeHandle, }; + #[test] + fn opaque_cast_destination_walker_punt_is_an_unsupported_construct() { + use rustc_public_bridge::IndexedVal; + + let mut ctx = Context::new(); + dialect_mir::register(&mut ctx); + + let pointee: TypeHandle = IntegerType::get(&ctx, 32, Signedness::Unsigned).into(); + let slot_ty: TypeHandle = MirPtrType::get_generic(&mut ctx, pointee, true).into(); + let block = BasicBlock::new(&mut ctx, None, vec![slot_ty]); + let slot = block.deref(&ctx).get_argument(0); + + let local: mir::Local = 0; + let mut value_map = ValueMap::new(1); + value_map.set_slot(local, slot); + + // These synthetic rustc_public handles are never dereferenced by this + // fixture. The address walker punts as soon as it sees OpaqueCast. + let rust_ty = rustc_public::ty::Ty::to_val(0); + let rust_span = rustc_public::ty::Span::to_val(0); + let body = mir::Body::new( + vec![], + vec![mir::LocalDecl { + ty: rust_ty, + span: rust_span, + mutability: mir::Mutability::Mut, + }], + 0, + vec![], + None, + rust_span, + ); + let place = mir::Place { + local, + projection: vec![mir::ProjectionElem::OpaqueCast(rust_ty)], + }; + + let walked = rvalue::translate_place_address( + &mut ctx, + &body, + &value_map, + &place, + /* is_mutable */ true, + block, + None, + Location::Unknown, + ) + .expect("an unsupported projection must punt rather than fail internally"); + + assert!( + walked.is_none(), + "OpaqueCast must make the writable place-address walker punt" + ); + + // Avoid formatting the synthetic Ty in `place`: this half of the test + // checks the call-result boundary that converts a real walker punt + // into the importer's unsupported-construct category. + let projection = ["OpaqueCast"]; + let error = require_writable_destination_address(walked, &projection, Location::Unknown) + .expect_err("an address-walker punt must stop result emission"); + + let message = error.to_string(); + assert!( + message.contains("Unsupported construct"), + "walker punt must keep the unsupported-construct category: {message}" + ); + assert!( + message.contains("cannot prepare writable call destination"), + "walker punt must identify the call destination: {message}" + ); + } + #[test] fn generated_u32_result_array_is_marked_for_forwarding() { let mut ctx = Context::new(); diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/asm.rs b/crates/mir-importer/src/translator/terminator/intrinsics/asm.rs index ce3b739009..a9b24a6ab2 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/asm.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/asm.rs @@ -5,7 +5,7 @@ //! Inline PTX marker-call translation. -use super::super::helpers::{emit_goto, emit_store_result_and_goto}; +use super::super::helpers::{self, emit_goto}; use crate::error::{TranslationErr, TranslationResult}; use crate::translator::values::ValueMap; use crate::translator::{rvalue, types}; @@ -169,6 +169,16 @@ pub fn emit_inline_ptx( } } + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + if num_outputs <= 1 { // Single-output (backward-compatible path): the destination type is the result type. let result_tys = vec![types::translate_destination_type( @@ -207,9 +217,9 @@ pub fn emit_inline_ptx( }; let result_value = inline_ptx.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, @@ -285,9 +295,9 @@ pub fn emit_inline_ptx( super::super::helpers::set_compiler_result_bundle_marker(ctx, tuple_op); let tuple_val = tuple_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, tuple_val, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/atomic.rs b/crates/mir-importer/src/translator/terminator/intrinsics/atomic.rs index 94a6c43d5b..fdf2918608 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/atomic.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/atomic.rs @@ -62,7 +62,7 @@ //! All `core::sync::atomic` operations are lowered with **system scope** (`.sys`) //! for safe host-device coherence, matching CUDA C++ `cuda::atomic` defaults. -use super::super::helpers::emit_store_result_and_goto; +use super::super::helpers; use crate::error::{TranslationErr, TranslationResult}; use crate::translator::values::ValueMap; use crate::translator::{facts, rvalue, types}; @@ -402,6 +402,16 @@ fn emit_atomic_load( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicLoadOp::build(ctx, ptr_val, result_ty, ordering, type_info.scope.clone()); let op_ptr = nvvm_op.get_operation(); @@ -414,9 +424,9 @@ fn emit_atomic_load( } let result_value = op_ptr.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, @@ -479,6 +489,16 @@ fn emit_atomic_store( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicStoreOp::build(ctx, val, ptr_val, ordering, type_info.scope.clone()); let op_ptr = nvvm_op.get_operation(); op_ptr.deref_mut(ctx).set_loc(loc.clone()); @@ -502,9 +522,9 @@ fn emit_atomic_store( unit_op.deref_mut(ctx).set_loc(loc.clone()); unit_op.insert_after(ctx, op_ptr); let unit_val = unit_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, unit_val, target, block_ptr, @@ -593,6 +613,16 @@ fn emit_atomic_rmw( (val, last_op) }; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicRmwOp::build( ctx, ptr_val, @@ -612,9 +642,9 @@ fn emit_atomic_rmw( } let result_value = op_ptr.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, @@ -692,6 +722,16 @@ fn emit_atomic_compare_exchange( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicCmpxchgOp::build( ctx, ptr_val, @@ -712,9 +752,9 @@ fn emit_atomic_compare_exchange( } let result_value = op_ptr.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, @@ -967,6 +1007,7 @@ pub fn dispatch_core_intrinsic( let orderings = extract_core_intrinsic_orderings(func, &loc, 1)?; return emit_core_compiler_fence( ctx, + body, args, destination, target, @@ -982,6 +1023,7 @@ pub fn dispatch_core_intrinsic( let orderings = extract_core_intrinsic_orderings(func, &loc, 1)?; return emit_core_atomic_fence( ctx, + body, args, destination, target, @@ -1235,6 +1277,7 @@ fn build_compiler_fence_barrier( #[allow(clippy::too_many_arguments)] fn emit_core_compiler_fence( ctx: &mut Context, + body: &mir::Body, args: &[mir::Operand], destination: &mir::Place, target: &Option, @@ -1255,6 +1298,16 @@ fn emit_core_compiler_fence( ); } + let (prepared_destination, prev_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + let barrier = build_compiler_fence_barrier(ctx, &loc, &ordering)?; barrier.deref_mut(ctx).set_loc(loc.clone()); @@ -1278,9 +1331,9 @@ fn emit_core_compiler_fence( unit_op.insert_after(ctx, barrier); let unit_val = unit_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, unit_val, target, block_ptr, @@ -1299,6 +1352,7 @@ fn emit_core_compiler_fence( #[allow(clippy::too_many_arguments)] fn emit_core_atomic_fence( ctx: &mut Context, + body: &mir::Body, args: &[mir::Operand], destination: &mir::Place, target: &Option, @@ -1319,6 +1373,16 @@ fn emit_core_atomic_fence( ); } + let (prepared_destination, prev_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + let fence = NvvmAtomicFenceOp::build(ctx, ordering, AtomicScope::System); let fence_op = fence.get_operation(); fence_op.deref_mut(ctx).set_loc(loc.clone()); @@ -1343,9 +1407,9 @@ fn emit_core_atomic_fence( unit_op.insert_after(ctx, fence_op); let unit_val = unit_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, unit_val, target, block_ptr, @@ -1387,6 +1451,16 @@ fn emit_core_atomic_load( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicLoadOp::build(ctx, ptr_val, result_ty, ordering, type_info.scope.clone()); let op_ptr = nvvm_op.get_operation(); @@ -1399,9 +1473,9 @@ fn emit_core_atomic_load( } let result_value = op_ptr.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, @@ -1453,6 +1527,16 @@ fn emit_core_atomic_store( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicStoreOp::build(ctx, val, ptr_val, ordering, type_info.scope.clone()); let op_ptr = nvvm_op.get_operation(); op_ptr.deref_mut(ctx).set_loc(loc.clone()); @@ -1476,9 +1560,9 @@ fn emit_core_atomic_store( unit_op.deref_mut(ctx).set_loc(loc.clone()); unit_op.insert_after(ctx, op_ptr); let unit_val = unit_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, unit_val, target, block_ptr, @@ -1533,6 +1617,16 @@ fn emit_core_atomic_rmw( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicRmwOp::build( ctx, ptr_val, @@ -1552,9 +1646,9 @@ fn emit_core_atomic_rmw( } let result_value = op_ptr.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, @@ -1621,6 +1715,16 @@ fn emit_core_atomic_cmpxchg( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let nvvm_op = NvvmAtomicCmpxchgOp::build( ctx, ptr_val, @@ -1676,9 +1780,9 @@ fn emit_core_atomic_cmpxchg( tuple_op.insert_after(ctx, success_op); let tuple_value = tuple_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, tuple_value, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/debug.rs b/crates/mir-importer/src/translator/terminator/intrinsics/debug.rs index 029dfdf418..c51a4c1777 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/debug.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/debug.rs @@ -12,7 +12,7 @@ //! - `__gpu_assertfail()` - Device-side assertion diagnostics //! - `__gpu_vprintf()` - Formatted output to host console -use super::super::helpers::emit_store_result_and_goto; +use super::super::helpers; use crate::error::{TranslationErr, TranslationResult}; use crate::translator::values::ValueMap; use dialect_mir::ops::MirConstructTupleOp; @@ -74,6 +74,16 @@ pub fn emit_assertfail( last_op = next_op; } + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let assertfail_op = AssertFailOp::build( ctx, translated_args[0], @@ -106,9 +116,9 @@ pub fn emit_assertfail( unit_op.insert_after(ctx, assertfail_op); let unit_value = unit_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, unit_value, target, block_ptr, @@ -182,6 +192,16 @@ pub fn emit_vprintf( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + // Create the vprintf operation let vprintf_op = VprintfOp::build(ctx, format_ptr, args_ptr); vprintf_op.deref_mut(ctx).set_loc(loc.clone()); @@ -195,9 +215,9 @@ pub fn emit_vprintf( // Store the result (i32) in the destination let result_value = vprintf_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs b/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs index c33b3201a7..e77e0c21e1 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs @@ -558,6 +558,16 @@ pub fn emit_sincos( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let (sin_callee, cos_callee) = if is_f64 { ( rust_intrinsics::CALLEE_SIN_F64, @@ -623,37 +633,18 @@ pub fn emit_sincos( tuple_op.insert_after(ctx, cos_op); let tuple_val = tuple_op.deref(ctx).get_result(0); - // Store through the projected place, not the bare local: the tuple value - // above is typed from the projection, so a `(*p) = sincos(x)`-style - // destination written to the local's slot would be a type-mismatched - // store aimed at the wrong address. - let goto_prev = helpers::store_result_to_place( + helpers::emit_prepared_result_and_goto( ctx, - body, - destination, + prepared_destination, tuple_val, - value_map, + target, block_ptr, tuple_op, - loc.clone(), - )?; - - if let Some(target_idx) = target { - Ok(helpers::emit_goto( - ctx, - *target_idx, - goto_prev, - block_map, - loc, - )) - } else { - input_err!( - loc.clone(), - TranslationErr::unsupported( - "libm::sincos call without target not supported".to_string() - ) - ) - } + value_map, + block_map, + loc, + "libm::sincos call without target not supported", + ) } /// Emit a placeholder `mir.call` for a rustc float math intrinsic. diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/active_mask.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/active_mask.rs index 521f4ee605..03c070a49a 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/active_mask.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/active_mask.rs @@ -22,7 +22,7 @@ use rustc_public::mir; #[allow(clippy::too_many_arguments)] pub(super) fn try_dispatch( ctx: &mut Context, - _body: &mir::Body, + body: &mir::Body, name: &str, args: &[mir::Operand], destination: &mir::Place, @@ -40,11 +40,20 @@ pub(super) fn try_dispatch( let active_mask = ActiveMaskOp::build(ctx); active_mask.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, active_mask, "v1:i0044"); - helpers::insert_op(ctx, active_mask, block_ptr, prev_op); - let result = active_mask.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + helpers::insert_op(ctx, active_mask, block_ptr, prepared_last_op); + let result = active_mask.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/clc.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/clc.rs index 0b258f101d..6d9e89b258 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/clc.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/clc.rs @@ -77,11 +77,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0325"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -130,11 +139,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0326"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -183,11 +201,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0327"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -236,11 +263,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0324"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/cluster_memory.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/cluster_memory.rs index 77a67a0bd6..26496dcb86 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/cluster_memory.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/cluster_memory.rs @@ -58,11 +58,20 @@ pub(super) fn try_dispatch( let cluster = DsmemReadU32Op::build(ctx, source, rank); cluster.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, cluster, "v1:i0321"); - helpers::insert_op(ctx, cluster, block_ptr, last_op); - let result = cluster.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, cluster, block_ptr, prepared_last_op); + let result = cluster.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -98,11 +107,20 @@ pub(super) fn try_dispatch( let cluster = MapaSharedClusterOp::build(ctx, source, rank); cluster.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, cluster, "v1:i0320"); - helpers::insert_op(ctx, cluster, block_ptr, last_op); - let result = cluster.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, cluster, block_ptr, prepared_last_op); + let result = cluster.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/dotprod.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/dotprod.rs index cf20bc1f09..dc190c31e2 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/dotprod.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/dotprod.rs @@ -67,11 +67,20 @@ pub(super) fn try_dispatch( let dot = Dp2aS32Op::build(ctx, a, b, c); dot.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, dot, "v1:i0032"); - helpers::insert_op(ctx, dot, block_ptr, last_op); - let result = dot.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, dot, block_ptr, prepared_last_op); + let result = dot.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -115,11 +124,20 @@ pub(super) fn try_dispatch( let dot = Dp2aU32Op::build(ctx, a, b, c); dot.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, dot, "v1:i0033"); - helpers::insert_op(ctx, dot, block_ptr, last_op); - let result = dot.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, dot, block_ptr, prepared_last_op); + let result = dot.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -163,11 +181,20 @@ pub(super) fn try_dispatch( let dot = Dp4aS32Op::build(ctx, a, b, c); dot.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, dot, "v1:i0030"); - helpers::insert_op(ctx, dot, block_ptr, last_op); - let result = dot.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, dot, block_ptr, prepared_last_op); + let result = dot.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -211,11 +238,20 @@ pub(super) fn try_dispatch( let dot = Dp4aU32Op::build(ctx, a, b, c); dot.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, dot, "v1:i0031"); - helpers::insert_op(ctx, dot, block_ptr, last_op); - let result = dot.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, dot, block_ptr, prepared_last_op); + let result = dot.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/elect.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/elect.rs index 3b0a6e2440..d5d67d189f 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/elect.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/elect.rs @@ -79,7 +79,16 @@ pub(super) fn try_dispatch( ); elect.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, elect, "v1:i0367"); - helpers::insert_op(ctx, elect, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, elect, block_ptr, prepared_last_op); let leader = elect.deref(ctx).get_result(0); let elected = elect.deref(ctx).get_result(1); let tuple = Operation::new( @@ -93,9 +102,9 @@ pub(super) fn try_dispatch( tuple.deref_mut(ctx).set_loc(loc.clone()); tuple.insert_after(ctx, elect); let result = tuple.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/extended_minmax.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/extended_minmax.rs index d4866b198f..e8bd09a1f0 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/extended_minmax.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/extended_minmax.rs @@ -69,11 +69,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0850"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -116,11 +125,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0842"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -164,11 +182,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0843"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -212,11 +239,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0564"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -260,11 +296,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0845"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -308,11 +353,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0565"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -356,11 +410,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0849"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -404,11 +467,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0567"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -452,11 +524,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0566"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -500,11 +581,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0847"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -548,11 +638,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0569"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -596,11 +695,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0568"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -644,11 +752,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0851"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -692,11 +809,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0570"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -740,11 +866,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0844"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -788,11 +923,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0571"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -836,11 +980,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0853"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -884,11 +1037,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0572"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -932,11 +1094,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0848"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -980,11 +1151,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0574"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1028,11 +1208,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0573"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1076,11 +1265,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0852"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1124,11 +1322,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0575"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1172,11 +1379,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0846"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1220,11 +1436,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0577"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1268,11 +1493,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0576"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1315,11 +1549,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0838"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1362,11 +1605,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0830"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1410,11 +1662,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0831"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1458,11 +1719,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0550"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1506,11 +1776,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0833"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1554,11 +1833,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0551"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1602,11 +1890,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0837"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1650,11 +1947,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0553"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1698,11 +2004,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0552"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1746,11 +2061,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0835"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1794,11 +2118,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0555"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1842,11 +2175,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0554"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1890,11 +2232,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0839"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1938,11 +2289,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0556"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1986,11 +2346,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0832"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2034,11 +2403,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0557"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2082,11 +2460,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0841"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2130,11 +2517,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0558"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2178,11 +2574,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0836"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2226,11 +2631,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0560"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2274,11 +2688,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0559"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2322,11 +2745,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0840"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2370,11 +2802,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0561"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2418,11 +2859,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0834"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2466,11 +2916,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0563"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2514,11 +2973,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0562"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/integer_minmax.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/integer_minmax.rs index 061df42a98..c275f0f40c 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/integer_minmax.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/integer_minmax.rs @@ -61,11 +61,20 @@ pub(super) fn try_dispatch( let intrinsic = MaxReluS16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0994"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -100,11 +109,20 @@ pub(super) fn try_dispatch( let intrinsic = MaxReluS32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0988"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -139,11 +157,20 @@ pub(super) fn try_dispatch( let intrinsic = MaxS16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0990"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -178,11 +205,20 @@ pub(super) fn try_dispatch( let intrinsic = MaxU16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0992"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -217,11 +253,20 @@ pub(super) fn try_dispatch( let intrinsic = MinReluS16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0993"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -256,11 +301,20 @@ pub(super) fn try_dispatch( let intrinsic = MinReluS32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0987"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -295,11 +349,20 @@ pub(super) fn try_dispatch( let intrinsic = MinS16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0989"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -334,11 +397,20 @@ pub(super) fn try_dispatch( let intrinsic = MinU16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0991"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/ldmatrix.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/ldmatrix.rs index c65eb69ee4..f4741f7585 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/ldmatrix.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/ldmatrix.rs @@ -59,12 +59,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0378"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 2, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -97,12 +106,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0379"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 2, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -135,12 +153,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0380"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 2, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -173,12 +200,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0381"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 4, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -211,12 +247,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0382"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 4, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -249,12 +294,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0383"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 4, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -287,12 +341,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0384"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let value = load.deref(ctx).get_result(0); let last_op = load; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -325,12 +388,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0385"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let value = load.deref(ctx).get_result(0); let last_op = load; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -364,12 +436,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0386"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 2, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -402,12 +483,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0387"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 2, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -441,12 +531,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0388"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 4, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -479,12 +578,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0389"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 4, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -518,12 +626,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0025"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let value = load.deref(ctx).get_result(0); let last_op = load; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -557,12 +674,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0026"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let value = load.deref(ctx).get_result(0); let last_op = load; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -596,12 +722,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0027"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 2, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -635,12 +770,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0028"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 2, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -674,12 +818,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0013"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 4, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, @@ -713,12 +866,21 @@ pub(super) fn try_dispatch( ); load.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, load, "v1:i0029"); - helpers::insert_op(ctx, load, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, load, block_ptr, prepared_last_op); let (value, last_op) = helpers::bundle_generated_u32_results_as_array(ctx, load, 4, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, value, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_basic.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_basic.rs index 4ce8cf0a78..504773ee5b 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_basic.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_basic.rs @@ -53,11 +53,20 @@ pub(super) fn try_dispatch( let mbarrier = MbarrierArriveSharedOp::build(ctx, barrier); mbarrier.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, mbarrier, "v1:i0098"); - helpers::insert_op(ctx, mbarrier, block_ptr, last_op); - let result = mbarrier.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mbarrier, block_ptr, prepared_last_op); + let result = mbarrier.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -92,11 +101,20 @@ pub(super) fn try_dispatch( let mbarrier = MbarrierArriveNoCompleteSharedOp::build(ctx, barrier, count); mbarrier.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, mbarrier, "v1:i1017"); - helpers::insert_op(ctx, mbarrier, block_ptr, last_op); - let result = mbarrier.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mbarrier, block_ptr, prepared_last_op); + let result = mbarrier.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -206,11 +224,20 @@ pub(super) fn try_dispatch( let mbarrier = MbarrierTestWaitSharedOp::build(ctx, barrier, token); mbarrier.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, mbarrier, "v1:i0099"); - helpers::insert_op(ctx, mbarrier, block_ptr, last_op); - let result = mbarrier.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mbarrier, block_ptr, prepared_last_op); + let result = mbarrier.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_extended.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_extended.rs index 08c23edb33..55f8ff9ec7 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_extended.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/mbarrier_extended.rs @@ -188,11 +188,20 @@ pub(super) fn try_dispatch( let extended = MbarrierArriveExpectTxSharedOp::build(ctx, barrier, bytes); extended.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, extended, "v1:i0306"); - helpers::insert_op(ctx, extended, block_ptr, last_op); - let result = extended.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, extended, block_ptr, prepared_last_op); + let result = extended.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -227,11 +236,20 @@ pub(super) fn try_dispatch( let extended = MbarrierArriveExpectTxClusterOp::build(ctx, barrier, bytes); extended.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, extended, "v1:i0307"); - helpers::insert_op(ctx, extended, block_ptr, last_op); - let result = extended.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, extended, block_ptr, prepared_last_op); + let result = extended.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -266,11 +284,20 @@ pub(super) fn try_dispatch( let extended = MbarrierTryWaitSharedOp::build(ctx, barrier, token); extended.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, extended, "v1:i0309"); - helpers::insert_op(ctx, extended, block_ptr, last_op); - let result = extended.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, extended, block_ptr, prepared_last_op); + let result = extended.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -305,11 +332,20 @@ pub(super) fn try_dispatch( let extended = MbarrierTryWaitParitySharedOp::build(ctx, barrier, parity); extended.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, extended, "v1:i0310"); - helpers::insert_op(ctx, extended, block_ptr, last_op); - let result = extended.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, extended, block_ptr, prepared_last_op); + let result = extended.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -344,11 +380,20 @@ pub(super) fn try_dispatch( let extended = MbarrierTryWaitParityClusterOp::build(ctx, barrier, parity); extended.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, extended, "v1:i0311"); - helpers::insert_op(ctx, extended, block_ptr, last_op); - let result = extended.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, extended, block_ptr, prepared_last_op); + let result = extended.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/movmatrix.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/movmatrix.rs index c6547fda02..3eebb77287 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/movmatrix.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/movmatrix.rs @@ -49,11 +49,20 @@ pub(super) fn try_dispatch( let intrinsic = MovmatrixTransB16Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0305"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_alu.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_alu.rs index b4970b1e69..88aea564f9 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_alu.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_alu.rs @@ -55,11 +55,20 @@ pub(super) fn try_dispatch( let intrinsic = AbsBf16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0070"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -85,11 +94,20 @@ pub(super) fn try_dispatch( let intrinsic = AbsF16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0080"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -124,11 +142,20 @@ pub(super) fn try_dispatch( let intrinsic = AddBf16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0064"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -163,11 +190,20 @@ pub(super) fn try_dispatch( let intrinsic = AddF16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0074"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -202,11 +238,20 @@ pub(super) fn try_dispatch( let intrinsic = AddF32x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0995"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -241,11 +286,20 @@ pub(super) fn try_dispatch( let intrinsic = AddFtzF32x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0996"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -289,11 +343,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaBf16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0062"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -337,11 +400,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaF16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0072"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -385,11 +457,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaF32x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i1001"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -433,11 +514,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaFtzF16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0854"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -481,11 +571,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaFtzF32x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i1002"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -529,11 +628,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaFtzReluF16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0857"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -577,11 +685,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaFtzSatF16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0856"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -625,11 +742,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaReluBf16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0063"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -673,11 +799,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaReluF16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0073"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -721,11 +856,20 @@ pub(super) fn try_dispatch( let intrinsic = FmaSatF16x2Op::build(ctx, arg0, arg1, arg2); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0855"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -760,11 +904,20 @@ pub(super) fn try_dispatch( let intrinsic = MaxBf16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0068"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -799,11 +952,20 @@ pub(super) fn try_dispatch( let intrinsic = MaxF16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0078"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -838,11 +1000,20 @@ pub(super) fn try_dispatch( let intrinsic = MinBf16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0067"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -877,11 +1048,20 @@ pub(super) fn try_dispatch( let intrinsic = MinF16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0077"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -916,11 +1096,20 @@ pub(super) fn try_dispatch( let intrinsic = MulBf16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0066"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -955,11 +1144,20 @@ pub(super) fn try_dispatch( let intrinsic = MulF16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0076"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -994,11 +1192,20 @@ pub(super) fn try_dispatch( let intrinsic = MulF32x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0999"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1033,11 +1240,20 @@ pub(super) fn try_dispatch( let intrinsic = MulFtzF32x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i1000"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1063,11 +1279,20 @@ pub(super) fn try_dispatch( let intrinsic = NegBf16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0069"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1093,11 +1318,20 @@ pub(super) fn try_dispatch( let intrinsic = NegF16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0079"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1132,11 +1366,20 @@ pub(super) fn try_dispatch( let intrinsic = SubBf16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0065"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1171,11 +1414,20 @@ pub(super) fn try_dispatch( let intrinsic = SubF16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0075"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1210,11 +1462,20 @@ pub(super) fn try_dispatch( let intrinsic = SubF32x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0997"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1249,11 +1510,20 @@ pub(super) fn try_dispatch( let intrinsic = SubFtzF32x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0998"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_atomic.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_atomic.rs index 2d11bcb6b8..240e4a27e8 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_atomic.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_atomic.rs @@ -59,11 +59,20 @@ pub(super) fn try_dispatch( PackedAtomicAddOp::build(ctx, address, addend, PackedAtomicFormatAttr::Bf16x2); atom.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, atom, "v1:i0015"); - helpers::insert_op(ctx, atom, block_ptr, last_op); - let value = atom.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, atom, block_ptr, prepared_last_op); + let value = atom.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, value, target, block_ptr, @@ -99,11 +108,20 @@ pub(super) fn try_dispatch( PackedAtomicAddOp::build(ctx, address, addend, PackedAtomicFormatAttr::F16x2); atom.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, atom, "v1:i0014"); - helpers::insert_op(ctx, atom, block_ptr, last_op); - let value = atom.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, atom, block_ptr, prepared_last_op); + let value = atom.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, value, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_conversion.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_conversion.rs index d4d87efabf..0d0c6217cd 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_conversion.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/packed_conversion.rs @@ -65,11 +65,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtF16x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0081"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -104,11 +113,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtF32x2Bf16x2Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0071"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -134,11 +152,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnF16x2E4m3x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0826"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -164,11 +191,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnF16x2E5m2x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0828"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -203,11 +239,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnReluBf16x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0084"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -233,11 +278,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnReluF16x2E4m3x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0827"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -263,11 +317,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnReluF16x2E5m2x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0829"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -302,11 +365,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnReluF16x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0083"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -332,11 +404,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteE4m3x2F16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0822"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -371,11 +452,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteE4m3x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0259"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -401,11 +491,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteE5m2x2F16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0824"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -440,11 +539,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteE5m2x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0261"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -470,11 +578,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteReluE4m3x2F16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0823"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -509,11 +626,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteReluE4m3x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0260"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -539,11 +665,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteReluE5m2x2F16x2Op::build(ctx, arg0); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0825"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -578,11 +713,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRnSatfiniteReluE5m2x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0262"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -617,11 +761,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRzBf16x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0085"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -656,11 +809,20 @@ pub(super) fn try_dispatch( let intrinsic = CvtRzF16x2F32Op::build(ctx, arg0, arg1); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0082"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/prmt.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/prmt.rs index d1b9e3e03d..ee5c42a1ff 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/prmt.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/prmt.rs @@ -66,11 +66,20 @@ pub(super) fn try_dispatch( let prmt = PrmtOp::build(ctx, vec![arg0, arg1, arg2], PrmtModeAttr::Generic); prmt.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, prmt, "v1:i0252"); - helpers::insert_op(ctx, prmt, block_ptr, last_op); - let result = prmt.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op); + let result = prmt.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -113,11 +122,20 @@ pub(super) fn try_dispatch( let prmt = PrmtOp::build(ctx, vec![arg0, arg1, arg2], PrmtModeAttr::B4e); prmt.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, prmt, "v1:i0254"); - helpers::insert_op(ctx, prmt, block_ptr, last_op); - let result = prmt.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op); + let result = prmt.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -151,11 +169,20 @@ pub(super) fn try_dispatch( let prmt = PrmtOp::build(ctx, vec![arg0, arg1], PrmtModeAttr::Ecl); prmt.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, prmt, "v1:i0256"); - helpers::insert_op(ctx, prmt, block_ptr, last_op); - let result = prmt.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op); + let result = prmt.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -189,11 +216,20 @@ pub(super) fn try_dispatch( let prmt = PrmtOp::build(ctx, vec![arg0, arg1], PrmtModeAttr::Ecr); prmt.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, prmt, "v1:i0257"); - helpers::insert_op(ctx, prmt, block_ptr, last_op); - let result = prmt.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op); + let result = prmt.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -236,11 +272,20 @@ pub(super) fn try_dispatch( let prmt = PrmtOp::build(ctx, vec![arg0, arg1, arg2], PrmtModeAttr::F4e); prmt.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, prmt, "v1:i0253"); - helpers::insert_op(ctx, prmt, block_ptr, last_op); - let result = prmt.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op); + let result = prmt.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -275,11 +320,20 @@ pub(super) fn try_dispatch( let prmt = PrmtOp::build(ctx, vec![arg0, arg1], PrmtModeAttr::Rc16); prmt.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, prmt, "v1:i0258"); - helpers::insert_op(ctx, prmt, block_ptr, last_op); - let result = prmt.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op); + let result = prmt.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -313,11 +367,20 @@ pub(super) fn try_dispatch( let prmt = PrmtOp::build(ctx, vec![arg0, arg1], PrmtModeAttr::Rc8); prmt.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, prmt, "v1:i0255"); - helpers::insert_op(ctx, prmt, block_ptr, last_op); - let result = prmt.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, prmt, block_ptr, prepared_last_op); + let result = prmt.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/redux.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/redux.rs index 771031a6c4..fd228d84a1 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/redux.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/redux.rs @@ -63,11 +63,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncAddOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0017"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -102,11 +111,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncAndOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0022"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -141,11 +159,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFmaxAbsOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1009"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -180,11 +207,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFmaxAbsNanOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1010"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -219,11 +255,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFmaxOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1007"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -258,11 +303,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncMaxOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0021"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -297,11 +351,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFmaxNanOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1008"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -336,11 +399,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncUmaxOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0020"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -375,11 +447,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFminAbsOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1005"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -414,11 +495,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFminAbsNanOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1006"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -453,11 +543,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFminOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1003"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -492,11 +591,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncMinOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0019"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -531,11 +639,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncFminNanOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i1004"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -570,11 +687,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncUminOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0018"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -609,11 +735,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncOrOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0023"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -648,11 +783,20 @@ pub(super) fn try_dispatch( let reduction = ReduxSyncXorOp::build(ctx, member_mask, value); reduction.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, reduction, "v1:i0024"); - helpers::insert_op(ctx, reduction, block_ptr, last_op); - let result = reduction.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, reduction, block_ptr, prepared_last_op); + let result = reduction.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/register_mma.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/register_mma.rs index fee6bbecf5..f6db42deda 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/register_mma.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/register_mma.rs @@ -75,12 +75,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0161"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -125,12 +134,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0158"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -175,12 +193,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0524"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -225,12 +252,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0105"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -275,12 +311,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0106"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -325,12 +370,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0504"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -375,12 +429,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0505"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -425,12 +488,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0506"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -475,12 +547,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0507"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -525,12 +606,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0508"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -575,12 +665,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0509"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -625,12 +724,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0510"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -675,12 +783,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0511"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -725,12 +842,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0110"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -775,12 +901,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0117"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -825,12 +960,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0111"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -875,12 +1019,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0118"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -925,12 +1078,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0112"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -975,12 +1137,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0113"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1025,12 +1196,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0120"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1075,12 +1255,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0119"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1125,12 +1314,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0162"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1175,12 +1373,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0159"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1225,12 +1432,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0479"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1275,12 +1491,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0480"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1325,12 +1550,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0481"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1375,12 +1609,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0482"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1425,12 +1668,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0483"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1475,12 +1727,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0484"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1525,12 +1786,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0485"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1575,12 +1845,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0486"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1625,12 +1904,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0487"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1675,12 +1963,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0488"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1725,12 +2022,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0489"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1775,12 +2081,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0490"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1825,12 +2140,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0491"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1875,12 +2199,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0492"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1925,12 +2258,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0493"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1975,12 +2317,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0494"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2025,12 +2376,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0495"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2075,12 +2435,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0496"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2125,12 +2494,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0497"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2175,12 +2553,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0498"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2225,12 +2612,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0499"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2275,12 +2671,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0500"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2325,12 +2730,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0501"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2375,12 +2789,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0502"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2425,12 +2848,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0503"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2475,12 +2907,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0454"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2525,12 +2966,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0455"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2575,12 +3025,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0456"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2625,12 +3084,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0457"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2675,12 +3143,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0458"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2725,12 +3202,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0459"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2775,12 +3261,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0460"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2825,12 +3320,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0461"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2875,12 +3379,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0462"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2925,12 +3438,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0463"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2975,12 +3497,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0464"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3025,12 +3556,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0465"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3075,12 +3615,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0466"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3125,12 +3674,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0467"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3175,12 +3733,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0468"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3225,12 +3792,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0469"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3275,12 +3851,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0470"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3325,12 +3910,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0471"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3375,12 +3969,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0472"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3425,12 +4028,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0473"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3475,12 +4087,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0474"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3525,12 +4146,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0475"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3575,12 +4205,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0476"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3625,12 +4264,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0477"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3675,12 +4323,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0478"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3725,12 +4382,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0512"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3775,12 +4441,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0513"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3825,12 +4500,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0514"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3875,12 +4559,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0515"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3925,12 +4618,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0516"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3975,12 +4677,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0517"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4025,12 +4736,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0518"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4075,12 +4795,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0519"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4125,12 +4854,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0858"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4175,12 +4913,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0859"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4225,12 +4972,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0860"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4275,12 +5031,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0861"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4325,12 +5090,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0862"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4375,12 +5149,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0863"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4425,12 +5208,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0864"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4475,12 +5267,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0865"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4525,12 +5326,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0866"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4575,12 +5385,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0867"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4625,12 +5444,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0868"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4675,12 +5503,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0869"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4725,12 +5562,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0870"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4775,12 +5621,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0871"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4825,12 +5680,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0872"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4875,12 +5739,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0873"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4925,12 +5798,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0874"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4975,12 +5857,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0875"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5025,12 +5916,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0876"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5075,12 +5975,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0877"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5125,12 +6034,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0878"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5175,12 +6093,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0879"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5225,12 +6152,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0880"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5275,12 +6211,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0881"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5325,12 +6270,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0882"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5375,12 +6329,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0141"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5425,12 +6388,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0149"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5475,12 +6447,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0142"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5525,12 +6506,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0150"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5575,12 +6565,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0108"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5625,12 +6624,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0121"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5675,12 +6683,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0114"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5725,12 +6742,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0122"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5775,12 +6801,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0143"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5825,12 +6860,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0144"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5875,12 +6919,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0152"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5925,12 +6978,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0151"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5975,12 +7037,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0115"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6025,12 +7096,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0116"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6075,12 +7155,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0124"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6125,12 +7214,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0123"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6175,12 +7273,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0520"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6225,12 +7332,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0145"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6275,12 +7391,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0153"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6325,12 +7450,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0146"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6375,12 +7509,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0154"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6425,12 +7568,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0147"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6475,12 +7627,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0148"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6525,12 +7686,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0156"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6575,12 +7745,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0155"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6625,12 +7804,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0521"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6675,12 +7863,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0522"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6725,12 +7922,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0523"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6775,12 +7981,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0107"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6825,12 +8040,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0160"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6875,12 +8099,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0157"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6925,12 +8158,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0125"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6975,12 +8217,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0129"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7025,12 +8276,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0126"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7075,12 +8335,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0130"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7125,12 +8394,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0127"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7175,12 +8453,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0128"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7225,12 +8512,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0132"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7275,12 +8571,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0131"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7325,12 +8630,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0133"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7375,12 +8689,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0137"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7425,12 +8748,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0134"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7475,12 +8807,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0138"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7525,12 +8866,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0135"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7575,12 +8925,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Wrapping); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0136"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7625,12 +8984,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0140"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7675,12 +9043,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::Satfinite); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0139"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7725,12 +9102,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_register_mma_overflow(ctx, RegisterMmaOverflowAttr::NotApplicable); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0109"); - helpers::insert_op(ctx, mma, block_ptr, Some(last_op)); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + Some(last_op), + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_arithmetic.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_arithmetic.rs index b0cde6adfa..7d9545be9e 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_arithmetic.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_arithmetic.rs @@ -69,11 +69,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0446"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -116,11 +125,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0436"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -163,11 +181,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0447"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -210,11 +237,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0449"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -257,11 +293,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0448"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -304,11 +349,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0438"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -351,11 +405,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0434"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -398,11 +461,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0439"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -445,11 +517,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0441"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -492,11 +573,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0440"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -539,11 +629,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0450"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -586,11 +685,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0437"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -633,11 +741,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0451"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -680,11 +797,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0453"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -727,11 +853,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0452"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -774,11 +909,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0442"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -821,11 +965,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0435"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -868,11 +1021,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0443"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -915,11 +1077,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0445"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -962,11 +1133,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0444"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1009,11 +1189,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0414"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1056,11 +1245,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0396"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1103,11 +1301,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0415"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1150,11 +1357,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0410"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1197,11 +1413,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0394"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1244,11 +1469,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0411"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1291,11 +1525,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0416"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1338,11 +1581,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0397"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1385,11 +1637,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0417"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1432,11 +1693,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0412"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1479,11 +1749,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0395"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1526,11 +1805,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0413"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1582,11 +1870,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0426"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1638,11 +1935,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0400"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1694,11 +2000,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0427"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1750,11 +2065,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0429"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1806,11 +2130,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0428"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1862,11 +2195,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0418"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1918,11 +2260,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0398"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1974,11 +2325,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0419"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2030,11 +2390,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0421"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2086,11 +2455,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0420"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2142,11 +2520,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0430"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2198,11 +2585,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0401"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2254,11 +2650,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0431"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2310,11 +2715,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0433"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2366,11 +2780,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0432"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2422,11 +2845,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0422"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2478,11 +2910,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0399"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2534,11 +2975,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0423"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2590,11 +3040,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0425"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2646,11 +3105,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0424"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2693,11 +3161,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0406"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2740,11 +3217,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0392"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2787,11 +3273,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0407"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2834,11 +3329,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0402"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2881,11 +3385,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0390"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2928,11 +3441,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0403"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -2975,11 +3497,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0408"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -3022,11 +3553,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0393"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -3069,11 +3609,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0409"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -3116,11 +3665,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0404"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -3163,11 +3721,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0391"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -3210,11 +3777,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0405"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_conversion.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_conversion.rs index c9de33c3ff..3ac0df590a 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_conversion.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_conversion.rs @@ -56,11 +56,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0373"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -91,11 +100,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0371"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -126,11 +144,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0372"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -161,11 +188,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0370"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -196,11 +232,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0369"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -231,11 +276,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0368"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -266,11 +320,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0377"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -301,11 +364,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0375"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -336,11 +408,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0376"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -371,11 +452,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0374"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_math.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_math.rs index 170a7a2e49..e111d7c9e5 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_math.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/scalar_math.rs @@ -59,11 +59,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0784"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -96,11 +105,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0785"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -133,11 +151,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i1016"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -170,11 +197,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0819"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -207,11 +243,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0820"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -244,11 +289,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0786"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -281,11 +335,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0787"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -318,11 +381,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0788"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -355,11 +427,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0797"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -392,11 +473,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0793"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -429,11 +519,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0800"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -466,11 +565,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0794"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -503,11 +611,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0789"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -540,11 +657,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0798"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -577,11 +703,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0790"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -614,11 +749,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0795"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -651,11 +795,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0801"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -688,11 +841,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0796"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -725,11 +887,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0791"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -762,11 +933,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0799"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -799,11 +979,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0792"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -836,11 +1025,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0802"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -873,11 +1071,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0804"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -910,11 +1117,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0803"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -947,11 +1163,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0782"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -984,11 +1209,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0783"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1021,11 +1255,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0805"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1058,11 +1301,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0806"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1095,11 +1347,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0811"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1132,11 +1393,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0817"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1169,11 +1439,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0812"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1206,11 +1485,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0807"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1243,11 +1531,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0815"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1280,11 +1577,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0808"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1317,11 +1623,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0813"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1354,11 +1669,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0818"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1391,11 +1715,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0814"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1428,11 +1761,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0809"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1465,11 +1807,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0816"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1502,11 +1853,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0810"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -1539,11 +1899,20 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0821"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); - let result = intrinsic.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); + let result = intrinsic.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/sparse_mma.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/sparse_mma.rs index 6597e8a997..8066bc9502 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/sparse_mma.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/sparse_mma.rs @@ -104,12 +104,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0219"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -182,12 +191,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0223"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -260,12 +278,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0220"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -338,12 +365,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0224"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -416,12 +452,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0221"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -494,12 +539,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0222"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -572,12 +626,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0226"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -650,12 +713,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0225"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -728,12 +800,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0163"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -806,12 +887,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0167"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -884,12 +974,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0164"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -962,12 +1061,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0168"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1040,12 +1148,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0165"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1118,12 +1235,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0166"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1196,12 +1322,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0170"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1274,12 +1409,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0169"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1352,12 +1496,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0211"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1430,12 +1583,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0215"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1508,12 +1670,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0212"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1586,12 +1757,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0216"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1664,12 +1844,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0187"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1742,12 +1931,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0191"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1820,12 +2018,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0188"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1898,12 +2105,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0192"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1976,12 +2192,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0213"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2054,12 +2279,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0214"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2132,12 +2366,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0218"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2210,12 +2453,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0217"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2288,13 +2540,22 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0189"); - helpers::insert_op(ctx, mma, block_ptr, last_op); - let (result, last_op) = - bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, - result, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); + let (result, last_op) = + bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, + result, target, block_ptr, last_op, @@ -2366,12 +2627,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0190"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2444,12 +2714,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0194"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2522,12 +2801,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0193"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2600,12 +2888,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0203"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2678,12 +2975,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0207"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2756,12 +3062,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0204"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2834,12 +3149,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0208"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2912,12 +3236,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0205"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2990,12 +3323,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0206"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3068,12 +3410,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0210"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3146,12 +3497,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0209"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3228,12 +3588,21 @@ pub(super) fn try_dispatch( ); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1021"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3310,12 +3679,21 @@ pub(super) fn try_dispatch( ); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1020"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3392,12 +3770,21 @@ pub(super) fn try_dispatch( ); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1022"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3470,12 +3857,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1019"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3548,12 +3944,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1024"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3626,12 +4031,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1023"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3704,12 +4118,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1025"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3782,12 +4205,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0171"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3860,12 +4292,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0175"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3938,12 +4379,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0172"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4016,12 +4466,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0176"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4094,12 +4553,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0173"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4172,12 +4640,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0174"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4250,12 +4727,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0178"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4328,12 +4814,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0177"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4406,12 +4901,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0525"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4484,13 +4988,22 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0526"); - helpers::insert_op(ctx, mma, block_ptr, last_op); - let (result, last_op) = - bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, - result, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); + let (result, last_op) = + bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, + result, target, block_ptr, last_op, @@ -4562,12 +5075,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0527"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4640,12 +5162,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0528"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4718,12 +5249,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0529"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4796,12 +5336,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0530"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4874,12 +5423,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0531"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4952,12 +5510,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0532"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5030,12 +5597,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0533"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5108,12 +5684,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0534"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5186,12 +5771,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0535"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5264,12 +5858,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0536"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5342,12 +5945,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0537"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5420,12 +6032,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0538"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5498,12 +6119,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0539"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5576,12 +6206,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0540"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5654,12 +6293,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0541"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5732,12 +6380,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0542"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5810,12 +6467,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0543"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5888,12 +6554,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0544"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5966,12 +6641,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0545"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6044,12 +6728,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0546"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6122,12 +6815,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0547"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6200,12 +6902,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0548"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6278,12 +6989,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0549"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6356,12 +7076,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0227"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6434,12 +7163,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0228"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6512,12 +7250,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0229"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6590,12 +7337,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0230"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6668,12 +7424,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0231"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6746,12 +7511,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0232"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6824,12 +7598,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0233"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6902,12 +7685,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0234"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6980,12 +7772,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0235"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7058,12 +7859,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0236"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7136,12 +7946,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0237"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7214,12 +8033,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0238"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7292,12 +8120,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0239"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7370,12 +8207,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0240"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7448,12 +8294,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0241"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7526,12 +8381,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0242"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7604,12 +8468,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0243"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7682,12 +8555,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0244"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7760,12 +8642,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0245"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7838,12 +8729,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0246"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7916,12 +8816,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0247"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -7994,12 +8903,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0248"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8072,12 +8990,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0249"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8150,12 +9077,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0250"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8228,12 +9164,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0251"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8306,12 +9251,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0195"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8384,12 +9338,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0199"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8462,12 +9425,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0196"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8540,12 +9512,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0200"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8618,12 +9599,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0179"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8696,12 +9686,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0183"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8774,12 +9773,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0180"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8852,12 +9860,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0184"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -8930,12 +9947,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0197"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9008,12 +10034,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0198"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9086,12 +10121,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0202"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9164,12 +10208,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZeroOrOne); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0201"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9242,12 +10295,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0181"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9320,12 +10382,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0182"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9398,12 +10469,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0186"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9476,12 +10556,21 @@ pub(super) fn try_dispatch( mma.set_attr_nvvm_sparse_mma_selector(ctx, SparseMmaSelectorAttr::ImmediateZero); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i0185"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -9558,12 +10647,21 @@ pub(super) fn try_dispatch( ); let mma = mma.get_operation(); helpers::set_generated_intrinsic_marker(ctx, mma, "v1:i1018"); - helpers::insert_op(ctx, mma, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, mma, block_ptr, prepared_last_op); let (result, last_op) = bundle_generated_mma_results(ctx, mma, result_ty, result_count, loc.clone()); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/sreg.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/sreg.rs index b3ec374991..468092765c 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/sreg.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/sreg.rs @@ -37,7 +37,7 @@ use rustc_public::mir; #[allow(clippy::too_many_arguments)] pub(super) fn try_dispatch( ctx: &mut Context, - _body: &mir::Body, + body: &mir::Body, name: &str, args: &[mir::Operand], destination: &mir::Place, @@ -55,6 +55,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNtidXOp::get_concrete_op_info(), "v1:i0003", destination, @@ -72,6 +73,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNtidYOp::get_concrete_op_info(), "v1:i0007", destination, @@ -89,6 +91,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNtidZOp::get_concrete_op_info(), "v1:i0011", destination, @@ -106,6 +109,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregCtaidXOp::get_concrete_op_info(), "v1:i0002", destination, @@ -123,6 +127,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregCtaidYOp::get_concrete_op_info(), "v1:i0006", destination, @@ -140,6 +145,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregCtaidZOp::get_concrete_op_info(), "v1:i0010", destination, @@ -155,6 +161,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClockOp::get_concrete_op_info(), "v1:i0283", destination, @@ -170,6 +177,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic_u64( ctx, + body, ReadPtxSregClock64Op::get_concrete_op_info(), "v1:i0284", destination, @@ -185,6 +193,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterNctarankOp::get_concrete_op_info(), "v1:i0276", destination, @@ -201,6 +210,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterCtaidXOp::get_concrete_op_info(), "v1:i0263", destination, @@ -217,6 +227,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterCtaidYOp::get_concrete_op_info(), "v1:i0264", destination, @@ -233,6 +244,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterCtaidZOp::get_concrete_op_info(), "v1:i0265", destination, @@ -248,6 +260,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterCtarankOp::get_concrete_op_info(), "v1:i0275", destination, @@ -264,6 +277,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterNctaidXOp::get_concrete_op_info(), "v1:i0266", destination, @@ -280,6 +294,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterNctaidYOp::get_concrete_op_info(), "v1:i0267", destination, @@ -296,6 +311,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterNctaidZOp::get_concrete_op_info(), "v1:i0268", destination, @@ -312,6 +328,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNclusterIdXOp::get_concrete_op_info(), "v1:i0272", destination, @@ -328,6 +345,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNclusterIdYOp::get_concrete_op_info(), "v1:i0273", destination, @@ -344,6 +362,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNclusterIdZOp::get_concrete_op_info(), "v1:i0274", destination, @@ -360,6 +379,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterIdXOp::get_concrete_op_info(), "v1:i0269", destination, @@ -376,6 +396,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterIdYOp::get_concrete_op_info(), "v1:i0270", destination, @@ -392,6 +413,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregClusterIdZOp::get_concrete_op_info(), "v1:i0271", destination, @@ -408,6 +430,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregDynamicSmemSizeOp::get_concrete_op_info(), "v1:i0293", destination, @@ -423,6 +446,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregEnvReg1Op::get_concrete_op_info(), "v1:i0286", destination, @@ -438,6 +462,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregEnvReg2Op::get_concrete_op_info(), "v1:i0287", destination, @@ -454,6 +479,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic_u64( ctx, + body, ReadPtxSregGlobaltimerOp::get_concrete_op_info(), "v1:i0285", destination, @@ -471,6 +497,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNctaidXOp::get_concrete_op_info(), "v1:i0004", destination, @@ -488,6 +515,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNctaidYOp::get_concrete_op_info(), "v1:i0008", destination, @@ -505,6 +533,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNctaidZOp::get_concrete_op_info(), "v1:i0012", destination, @@ -522,6 +551,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic_u64( ctx, + body, ReadPtxSregGridIdOp::get_concrete_op_info(), "v1:i0290", destination, @@ -537,6 +567,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregLaneIdOp::get_concrete_op_info(), "v1:i0016", destination, @@ -553,6 +584,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregLanemaskEqOp::get_concrete_op_info(), "v1:i0037", destination, @@ -569,6 +601,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregLanemaskGeOp::get_concrete_op_info(), "v1:i0038", destination, @@ -585,6 +618,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregLanemaskGtOp::get_concrete_op_info(), "v1:i0039", destination, @@ -601,6 +635,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregLanemaskLeOp::get_concrete_op_info(), "v1:i0036", destination, @@ -617,6 +652,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregLanemaskLtOp::get_concrete_op_info(), "v1:i0035", destination, @@ -634,6 +670,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNsmIdOp::get_concrete_op_info(), "v1:i0289", destination, @@ -649,6 +686,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregNwarpIdOp::get_concrete_op_info(), "v1:i0292", destination, @@ -666,6 +704,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregSmIdOp::get_concrete_op_info(), "v1:i0288", destination, @@ -683,6 +722,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregTidXOp::get_concrete_op_info(), "v1:i0001", destination, @@ -700,6 +740,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregTidYOp::get_concrete_op_info(), "v1:i0005", destination, @@ -717,6 +758,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregTidZOp::get_concrete_op_info(), "v1:i0009", destination, @@ -733,6 +775,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregTotalSmemSizeOp::get_concrete_op_info(), "v1:i0294", destination, @@ -748,6 +791,7 @@ pub(super) fn try_dispatch( require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregWarpIdOp::get_concrete_op_info(), "v1:i0291", destination, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_ld.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_ld.rs index e313340cab..ac632c637b 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_ld.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_ld.rs @@ -99,7 +99,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0637"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -143,9 +152,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -186,7 +195,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0636"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -230,9 +248,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -273,7 +291,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0629"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -317,9 +344,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -360,7 +387,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0628"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -404,9 +440,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -447,7 +483,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0631"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -491,9 +536,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -534,7 +579,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0630"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -578,9 +632,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -621,7 +675,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0639"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -665,9 +728,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -708,7 +771,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0638"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -752,9 +824,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -795,7 +867,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0633"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -839,9 +920,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -882,7 +963,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0632"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -926,9 +1016,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -969,7 +1059,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0641"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1013,9 +1112,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1056,7 +1155,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0640"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1100,9 +1208,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1143,7 +1251,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0635"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1187,9 +1304,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1230,7 +1347,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0634"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1274,9 +1400,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1317,7 +1443,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0651"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1361,9 +1496,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1404,7 +1539,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0650"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1448,9 +1592,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1491,7 +1635,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0643"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1535,9 +1688,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1578,7 +1731,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0642"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1622,9 +1784,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1665,7 +1827,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0645"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1709,9 +1880,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1752,7 +1923,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0644"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1796,9 +1976,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1839,7 +2019,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0653"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1883,9 +2072,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -1926,7 +2115,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0652"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -1970,9 +2168,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2013,7 +2211,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0647"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2057,9 +2264,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2100,7 +2307,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0646"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2144,9 +2360,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2187,7 +2403,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0649"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2231,9 +2456,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2274,7 +2499,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0648"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2318,9 +2552,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2383,7 +2617,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0743"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2427,9 +2670,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2492,7 +2735,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0742"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2536,9 +2788,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2601,7 +2853,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0737"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2645,9 +2906,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2710,7 +2971,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0736"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2754,9 +3024,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2819,12 +3089,21 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0729"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let result = intrinsic.deref(ctx).get_result(0); let value = intrinsic; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2887,12 +3166,21 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0728"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let result = intrinsic.deref(ctx).get_result(0); let value = intrinsic; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -2955,7 +3243,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0731"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -2999,9 +3296,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3064,7 +3361,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0730"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3108,9 +3414,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3173,7 +3479,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0739"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3217,9 +3532,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3282,7 +3597,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0738"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3326,9 +3650,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3391,7 +3715,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0733"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3435,9 +3768,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3500,7 +3833,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0732"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3544,9 +3886,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3609,7 +3951,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0741"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3653,9 +4004,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3718,7 +4069,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0740"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3762,9 +4122,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3827,7 +4187,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0735"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3871,9 +4240,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -3936,7 +4305,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0734"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -3980,9 +4358,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4023,7 +4401,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0627"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4067,9 +4454,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4110,7 +4497,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0626"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4154,9 +4550,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4197,7 +4593,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0621"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4241,9 +4646,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4284,7 +4689,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0620"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4328,9 +4742,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4371,12 +4785,21 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0613"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let result = intrinsic.deref(ctx).get_result(0); let value = intrinsic; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4417,12 +4840,21 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0612"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let result = intrinsic.deref(ctx).get_result(0); let value = intrinsic; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4463,7 +4895,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0615"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4507,9 +4948,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4550,7 +4991,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0614"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4594,9 +5044,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4637,7 +5087,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0623"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4681,9 +5140,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4724,7 +5183,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0622"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4768,9 +5236,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4811,7 +5279,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0617"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4855,9 +5332,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4898,7 +5375,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0616"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -4942,9 +5428,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -4985,7 +5471,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0625"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5029,9 +5524,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5072,7 +5567,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0624"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5116,9 +5620,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5159,7 +5663,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0619"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5203,9 +5716,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5246,7 +5759,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0618"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5290,9 +5812,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5333,7 +5855,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0669"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5377,9 +5908,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5420,7 +5951,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0668"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..128) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5464,9 +6004,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5507,7 +6047,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0663"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5551,9 +6100,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5594,7 +6143,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0662"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..16) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5638,9 +6196,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5681,12 +6239,21 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0655"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let result = intrinsic.deref(ctx).get_result(0); let value = intrinsic; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5727,12 +6294,21 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0654"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let result = intrinsic.deref(ctx).get_result(0); let value = intrinsic; - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5773,7 +6349,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0657"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5817,9 +6402,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5860,7 +6445,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0656"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..2) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5904,9 +6498,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -5947,7 +6541,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0665"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -5991,9 +6594,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6034,7 +6637,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0664"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -6078,9 +6690,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6121,7 +6733,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0659"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -6165,9 +6786,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6208,7 +6829,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0658"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -6252,9 +6882,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6295,7 +6925,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0667"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -6339,9 +6978,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6382,7 +7021,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0666"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..64) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -6426,9 +7074,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6469,7 +7117,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0661"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -6513,9 +7170,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -6556,7 +7213,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0660"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..8) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -6600,9 +7266,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_other.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_other.rs index 0d64190fb9..4fd82cd93a 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_other.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/tcgen05_other.rs @@ -726,7 +726,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0356"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..4) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -770,9 +779,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -813,7 +822,16 @@ pub(super) fn try_dispatch( ); intrinsic.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, intrinsic, "v1:i0355"); - helpers::insert_op(ctx, intrinsic, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, intrinsic, block_ptr, prepared_last_op); let results: Vec = (0..32) .map(|index| intrinsic.deref(ctx).get_result(index)) .collect(); @@ -857,9 +875,9 @@ pub(super) fn try_dispatch( (value.deref(ctx).get_result(0), value) }; helpers::set_compiler_result_bundle_marker(ctx, value); - Ok(Some(helpers::emit_store_result_and_goto( + Ok(Some(helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/vote.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/vote.rs index a9795b56c8..d0f6aad24f 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/vote.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/vote.rs @@ -57,11 +57,20 @@ pub(super) fn try_dispatch( let vote = VoteSyncAllOp::build(ctx, member_mask, predicate); vote.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, vote, "v1:i0040"); - helpers::insert_op(ctx, vote, block_ptr, last_op); - let result = vote.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, vote, block_ptr, prepared_last_op); + let result = vote.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -95,11 +104,20 @@ pub(super) fn try_dispatch( let vote = VoteSyncAnyOp::build(ctx, member_mask, predicate); vote.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, vote, "v1:i0041"); - helpers::insert_op(ctx, vote, block_ptr, last_op); - let result = vote.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, vote, block_ptr, prepared_last_op); + let result = vote.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -134,11 +152,20 @@ pub(super) fn try_dispatch( let vote = VoteSyncBallotOp::build(ctx, member_mask, predicate); vote.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, vote, "v1:i0042"); - helpers::insert_op(ctx, vote, block_ptr, last_op); - let result = vote.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, vote, block_ptr, prepared_last_op); + let result = vote.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -172,11 +199,20 @@ pub(super) fn try_dispatch( let vote = VoteSyncUniOp::build(ctx, member_mask, predicate); vote.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, vote, "v1:i0043"); - helpers::insert_op(ctx, vote, block_ptr, last_op); - let result = vote.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, vote, block_ptr, prepared_last_op); + let result = vote.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_match.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_match.rs index 8e3c3b471e..016d23f012 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_match.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_match.rs @@ -60,11 +60,20 @@ pub(super) fn try_dispatch( let warp_match = MatchAllSyncI64Op::build(ctx, member_mask, value); warp_match.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, warp_match, "v1:i0048"); - helpers::insert_op(ctx, warp_match, block_ptr, last_op); - let result = warp_match.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, warp_match, block_ptr, prepared_last_op); + let result = warp_match.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -99,11 +108,20 @@ pub(super) fn try_dispatch( let warp_match = MatchAllSyncI32Op::build(ctx, member_mask, value); warp_match.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, warp_match, "v1:i0047"); - helpers::insert_op(ctx, warp_match, block_ptr, last_op); - let result = warp_match.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, warp_match, block_ptr, prepared_last_op); + let result = warp_match.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -138,11 +156,20 @@ pub(super) fn try_dispatch( let warp_match = MatchAnySyncI64Op::build(ctx, member_mask, value); warp_match.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, warp_match, "v1:i0046"); - helpers::insert_op(ctx, warp_match, block_ptr, last_op); - let result = warp_match.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, warp_match, block_ptr, prepared_last_op); + let result = warp_match.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -177,11 +204,20 @@ pub(super) fn try_dispatch( let warp_match = MatchAnySyncI32Op::build(ctx, member_mask, value); warp_match.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, warp_match, "v1:i0045"); - helpers::insert_op(ctx, warp_match, block_ptr, last_op); - let result = warp_match.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, warp_match, block_ptr, prepared_last_op); + let result = warp_match.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_shuffle.rs b/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_shuffle.rs index f1810e7a84..396b0c7dea 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_shuffle.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/generated/warp_shuffle.rs @@ -71,11 +71,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncDownF32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0056"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -119,11 +128,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncDownI32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0052"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -167,11 +185,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncDownI64Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0060"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -215,11 +242,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncIdxF32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0054"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -263,11 +299,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncIdxI32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0050"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -311,11 +356,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncIdxI64Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0058"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -359,11 +413,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncUpF32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0057"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -407,11 +470,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncUpI32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0053"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -455,11 +527,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncUpI64Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0061"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -503,11 +584,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncBflyF32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0055"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -551,11 +641,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncBflyI32Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0051"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, @@ -599,11 +698,20 @@ pub(super) fn try_dispatch( let shuffle = ShflSyncBflyI64Op::build(ctx, member_mask, value, lane_or_delta); shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0059"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); - let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( ctx, + body, destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); + let result = shuffle.deref(ctx).get_result(0); + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/iket.rs b/crates/mir-importer/src/translator/terminator/intrinsics/iket.rs index ac1567c2ec..78d0bcecbd 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/iket.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/iket.rs @@ -5,7 +5,7 @@ //! Translation of `cuda_device::iket` compiler markers. -use super::super::helpers::{emit_goto, emit_store_result_and_goto, insert_op}; +use super::super::helpers::{self, emit_goto, insert_op}; use crate::error::{TranslationErr, TranslationResult}; use crate::translator::{rvalue, values::ValueMap}; use dialect_iket::{ @@ -186,6 +186,22 @@ pub fn try_dispatch( None }; + let prepared_destination = if matches!(marker, Marker::RangeStart | Marker::RangeStartPayload) { + let (prepared_destination, next_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + last_op = next_op; + Some(prepared_destination) + } else { + None + }; + let operation = match marker { Marker::Mark | Marker::MarkPayload => IketMarkOp::new( ctx, @@ -224,9 +240,9 @@ pub fn try_dispatch( if matches!(marker, Marker::RangeStart | Marker::RangeStartPayload) { let result: Value = operation.deref(ctx).get_result(0); - return emit_store_result_and_goto( + return helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination.expect("range_start destination must be prepared"), result, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/indexing.rs b/crates/mir-importer/src/translator/terminator/intrinsics/indexing.rs index 2196ae60b8..6ca3b5e837 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/indexing.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/indexing.rs @@ -31,7 +31,7 @@ //! the witness packs `(row, col)` and the addressed slice resolves them //! against its own host-bound row width at the access site -use super::super::helpers::emit_store_result_and_goto; +use super::super::helpers; use crate::error::{TranslationErr, TranslationResult}; use crate::translator::rvalue; use crate::translator::types; @@ -159,7 +159,17 @@ pub fn emit_len( let (disjoint_slice_val, load_op) = load_disjoint_slice_receiver(ctx, disjoint_slice_val, block_ptr, last_op, loc.clone())?; - let mut last_op = Some(load_op); + let last_op = Some(load_op); + + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; // Extract len field (field 1) from DisjointSlice // DisjointSlice layout: { ptr: *mut T, len: usize, _marker: PhantomData } @@ -184,14 +194,11 @@ pub fn emit_len( } else { extract_len.get_operation().insert_at_front(block_ptr, ctx); } - last_op = Some(extract_len.get_operation()); - let len_val = extract_len.get_operation().deref(ctx).get_result(0); - - let prev = last_op.expect("should have at least extract_len op"); - emit_store_result_and_goto( + let prev = extract_len.get_operation(); + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, len_val, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/layout.rs b/crates/mir-importer/src/translator/terminator/intrinsics/layout.rs index bc52303f9d..cf9216d1a1 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/layout.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/layout.rs @@ -601,6 +601,16 @@ pub fn emit_rust_layout_intrinsic( } }; + let (prepared_destination, prev_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + let (result, result_op) = match intrinsic { RustLayoutIntrinsic::SizeOfVal => emit_size_of_val( ctx, @@ -617,10 +627,9 @@ pub fn emit_rust_layout_intrinsic( } }; - let store_op = helpers::store_result_to_place( + let store_op = helpers::finish_destination_write( ctx, - body, - destination, + prepared_destination, result, value_map, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs b/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs index da02d41fb7..ee48817227 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs @@ -5,7 +5,7 @@ //! Memory access and conversion intrinsics. -use super::super::helpers::{emit_goto, emit_store_result_and_goto}; +use super::super::helpers::{self, emit_goto}; use crate::error::{TranslationErr, TranslationResult}; use crate::translator::facts; use crate::translator::values::ValueMap; @@ -179,6 +179,16 @@ pub fn emit_volatile_load( } }; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let load_op = Operation::new( ctx, MirLoadOp::get_concrete_op_info(), @@ -197,9 +207,9 @@ pub fn emit_volatile_load( } let result = load_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -360,6 +370,16 @@ pub fn emit_arith_offset( op_after_ptr, loc.clone(), )?; + let (prepared_destination, op_after_count) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + op_after_count, + loc.clone(), + )?; + let offset = Operation::new( ctx, MirPtrOffsetOp::get_concrete_op_info(), @@ -377,9 +397,9 @@ pub fn emit_arith_offset( } let result = offset.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -524,29 +544,39 @@ fn emit_ptr_offset_from_with_result( prev_op, loc.clone(), )?; - let this_addr = emit_pointer_expose_address( + let (other_ptr, op_after_other) = rvalue::translate_operand( ctx, - this_ptr, - result_type, - op_after_this, + body, + &args[1], + value_map, block_ptr, + op_after_this, loc.clone(), - ); + )?; - let (other_ptr, op_after_other) = rvalue::translate_operand( + let (prepared_destination, op_after_destination) = helpers::prepare_destination_write( ctx, body, - &args[1], + destination, value_map, block_ptr, - Some(this_addr), + op_after_other, loc.clone(), )?; + + let this_addr = emit_pointer_expose_address( + ctx, + this_ptr, + result_type, + op_after_destination, + block_ptr, + loc.clone(), + ); let other_addr = emit_pointer_expose_address( ctx, other_ptr, result_type, - op_after_other, + Some(this_addr), block_ptr, loc.clone(), ); @@ -580,9 +610,9 @@ fn emit_ptr_offset_from_with_result( div_op.insert_after(ctx, size_const); let result = div_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result, target, block_ptr, @@ -776,7 +806,15 @@ pub fn emit_shared_array_index( last_op, loc.clone(), )?; - let mut last_op = last_op_after_index; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op_after_index, + loc.clone(), + )?; // The shared_array_val is a pointer to the shared memory array. // We need to compute ptr + index to get a pointer to the element. @@ -799,14 +837,11 @@ pub fn emit_shared_array_index( } else { offset_op.insert_at_front(block_ptr, ctx); } - last_op = Some(offset_op); - let result_ptr = offset_op.deref(ctx).get_result(0); - - let prev = last_op.expect("should have at least offset_op"); - emit_store_result_and_goto( + let prev = offset_op; + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_ptr, target, block_ptr, @@ -914,6 +949,16 @@ pub fn emit_shared_array_as_ptr( ); } + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let (result_ptr, cast_op) = establish_shared_array_raw_address( ctx, shared_ptr, @@ -922,9 +967,9 @@ pub fn emit_shared_array_as_ptr( last_op, loc.clone(), )?; - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_ptr, target, block_ptr, @@ -1002,6 +1047,16 @@ pub fn emit_dynamic_shared_get( // We use generic pointer type since MirExternSharedOp result will be cast let ptr_ty = MirPtrType::get_shared(ctx, elem_ty, true).into(); + let (prepared_destination, prev_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + // Create MirExternSharedOp let op = Operation::new( ctx, @@ -1037,9 +1092,9 @@ pub fn emit_dynamic_shared_get( Some(extern_shared.get_operation()), loc.clone(), )?; - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_ptr, target, block_ptr, @@ -1114,6 +1169,16 @@ pub fn emit_dynamic_shared_offset( // 1. If offset is a constant, store it as an attribute // 2. If offset is dynamic, we need to emit a GEP after the base pointer + let (prepared_destination, prev_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + prev_op, + loc.clone(), + )?; + // First create the base extern shared op let op = Operation::new( ctx, @@ -1213,9 +1278,9 @@ pub fn emit_dynamic_shared_offset( let (final_ptr, raw_mut_cast) = establish_dynamic_shared_raw_mut(ctx, final_ptr, block_ptr, Some(last_op), loc.clone())?; - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, final_ptr, target, block_ptr, @@ -1269,6 +1334,16 @@ pub fn emit_cvta_generic_to_shared_offset( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let u64_ty = IntegerType::get(ctx, 64, Signedness::Unsigned); let cvta_op = Operation::new( ctx, @@ -1287,9 +1362,9 @@ pub fn emit_cvta_generic_to_shared_offset( } let result_value = cvta_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/intrinsics/wgmma.rs b/crates/mir-importer/src/translator/terminator/intrinsics/wgmma.rs index daa4762c11..4d662072a4 100644 --- a/crates/mir-importer/src/translator/terminator/intrinsics/wgmma.rs +++ b/crates/mir-importer/src/translator/terminator/intrinsics/wgmma.rs @@ -7,7 +7,7 @@ //! //! Handles Hopper `sm_90a` asynchronous warpgroup matrix operations. -use super::super::helpers::{emit_goto, emit_store_result_and_goto}; +use super::super::helpers::{self, emit_goto}; use crate::error::{TranslationErr, TranslationResult}; use crate::translator::rvalue; use crate::translator::values::ValueMap; @@ -78,6 +78,16 @@ pub fn emit_wgmma_make_smem_desc( loc.clone(), )?; + let (prepared_destination, last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + let u64_ty = IntegerType::get(ctx, 64, Signedness::Unsigned); let desc_op = Operation::new( ctx, @@ -96,9 +106,9 @@ pub fn emit_wgmma_make_smem_desc( } let result_value = desc_op.deref(ctx).get_result(0); - emit_store_result_and_goto( + helpers::emit_prepared_result_and_goto( ctx, - destination, + prepared_destination, result_value, target, block_ptr, diff --git a/crates/mir-importer/src/translator/terminator/mod.rs b/crates/mir-importer/src/translator/terminator/mod.rs index 1f75500b20..e9aaa1f507 100644 --- a/crates/mir-importer/src/translator/terminator/mod.rs +++ b/crates/mir-importer/src/translator/terminator/mod.rs @@ -1236,6 +1236,7 @@ fn translate_call( } return helpers::emit_unit_noop_intrinsic( ctx, + body, destination, &target_usize, block_ptr, @@ -1579,6 +1580,22 @@ fn translate_function_item_call( } } + let prepared_destination = if target.is_some() { + let (prepared, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + last_op = prepared_last_op; + Some(prepared) + } else { + None + }; + let call_op = Operation::new( ctx, MirCallOp::get_concrete_op_info(), @@ -1604,15 +1621,10 @@ fn translate_function_item_call( return Ok(emit_unreachable_after(ctx, block_ptr, Some(call_op), loc)); } - // The result is typed from the projected destination above, so it must - // also be stored through the projection: a bare-local store would aim a - // field-typed value at the aggregate's slot (or a pointee-typed value at - // the pointer's). let result_value = call_op.deref(ctx).get_result(0); - let last_inserted = helpers::store_result_to_place( + let last_inserted = helpers::finish_destination_write( ctx, - body, - destination, + prepared_destination.expect("returning call prepared its destination"), result_value, value_map, block_ptr, @@ -1858,6 +1870,22 @@ fn translate_closure_call( unpacked_args.push(tuple_value); } + let prepared_destination = if target.is_some() { + let (prepared, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last_op, + loc.clone(), + )?; + last_op = prepared_last_op; + Some(prepared) + } else { + None + }; + // Now emit the call with unpacked arguments let call_op = Operation::new( ctx, @@ -1889,14 +1917,10 @@ fn translate_closure_call( return Ok(emit_unreachable_after(ctx, block_ptr, Some(call_op), loc)); } - // Store the call result into the destination place. The result is typed - // from the projected destination above, so it must also be stored - // through the projection, not the bare local's slot. let result_value = call_op.deref(ctx).get_result(0); - let last_inserted = helpers::store_result_to_place( + let last_inserted = helpers::finish_destination_write( ctx, - body, - destination, + prepared_destination.expect("returning call prepared its destination"), result_value, value_map, block_ptr, @@ -2418,16 +2442,27 @@ fn emit_ptr_memmove( let (count, last) = rvalue::translate_operand(ctx, body, &args[2], value_map, block_ptr, last, loc.clone())?; + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last, + loc.clone(), + )?; + // `mir.memmove` operand order is (dst, src, count). The typed builder // stamps the elem_type fact from dst for lowering's byte count. let xfer = MirMemmoveOp::build(ctx, dst, src, count)?; xfer.deref_mut(ctx).set_loc(loc.clone()); - match last { + match prepared_last_op { Some(p) => xfer.insert_after(ctx, p), None => xfer.insert_at_front(block_ptr, ctx), } - // The intrinsic yields `()`; materialize it for the destination local. + // The intrinsic yields `()`; materialize it after the memmove and write it + // through the destination prepared before the memory operation ran. let unit_ty = MirTupleType::get(ctx, vec![]); let unit_op = Operation::new( ctx, @@ -2441,26 +2476,18 @@ fn emit_ptr_memmove( unit_op.insert_after(ctx, xfer); let unit_val = unit_op.deref(ctx).get_result(0); - let goto_prev = value_map - .store_local(ctx, destination.local, unit_val, block_ptr, Some(unit_op)) - .unwrap_or(unit_op); - - if let Some(target_idx) = target { - Ok(helpers::emit_goto( - ctx, - *target_idx, - goto_prev, - block_map, - loc, - )) - } else { - input_err!( - loc, - TranslationErr::unsupported( - "ptr::copy intrinsic call without target not supported".to_string() - ) - ) - } + helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, + unit_val, + target, + block_ptr, + unit_op, + value_map, + block_map, + loc, + "ptr::copy intrinsic call without target not supported", + ) } /// Lower `core::intrinsics::typed_swap_nonoverlapping::(x, y)`, the @@ -2521,6 +2548,16 @@ fn emit_typed_swap( } }; + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + value_map, + block_ptr, + last, + loc.clone(), + )?; + // t0 = *x let load_x = Operation::new( ctx, @@ -2531,7 +2568,7 @@ fn emit_typed_swap( 0, ); load_x.deref_mut(ctx).set_loc(loc.clone()); - match last { + match prepared_last_op { Some(p) => load_x.insert_after(ctx, p), None => load_x.insert_at_front(block_ptr, ctx), } @@ -2588,26 +2625,18 @@ fn emit_typed_swap( unit_op.insert_after(ctx, store_y); let unit_val = unit_op.deref(ctx).get_result(0); - let goto_prev = value_map - .store_local(ctx, destination.local, unit_val, block_ptr, Some(unit_op)) - .unwrap_or(unit_op); - - if let Some(target_idx) = target { - Ok(helpers::emit_goto( - ctx, - *target_idx, - goto_prev, - block_map, - loc, - )) - } else { - input_err!( - loc, - TranslationErr::unsupported( - "typed_swap_nonoverlapping call without target not supported".to_string() - ) - ) - } + helpers::emit_prepared_result_and_goto( + ctx, + prepared_destination, + unit_val, + target, + block_ptr, + unit_op, + value_map, + block_map, + loc, + "typed_swap_nonoverlapping call without target not supported", + ) } /// Dispatches `cuda_device` intrinsic calls to their respective handlers. @@ -2838,6 +2867,7 @@ fn try_dispatch_intrinsic( "core::intrinsics::cold_path" | "std::intrinsics::cold_path" => { Ok(Some(helpers::emit_unit_noop_intrinsic( ctx, + body, destination, target, block_ptr, diff --git a/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs b/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs index 267be962f2..4a1ccfe81a 100644 --- a/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs +++ b/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs @@ -13,14 +13,13 @@ clippy::unit_arg )] -//! An intrinsic call writing through a projected destination. +//! Calls writing through prepared projected destinations. //! -//! rustc lowers an ordinary call whose destination carries a projection into a -//! call to a temporary followed by a store, so the projection never reaches -//! code generation. An intrinsic keeps its destination, which leaves three -//! shapes to translate: a dereferenced pointer, a struct field, and an array -//! element. Each is written here in custom MIR, since surface Rust cannot -//! produce them. +//! rustc normally lowers an ordinary surface-Rust call whose destination +//! carries a projection into a temporary followed by a store, so chained call +//! destinations rarely survive to code generation. Custom MIR keeps shapes +//! such as `RET.1[i]`, `(*ptr).field`, and `RET[i].field` intact so the importer +//! has to translate the actual destination chain. //! //! Three translation paths build or store the call result themselves and so //! have store sites of their own, exercised separately: a float-math @@ -34,9 +33,13 @@ //! the bodies are `inline(never)` and the sincos pointer arrives as an //! opaque argument. //! -//! Each case runs on the device and on the host from the same body, and the -//! two results must agree. A result that landed at the wrong address shows up -//! as a difference, since the host reads what the device wrote back. +//! The ordinary-call cases run on the device and on the host from the same +//! bodies and must agree. The generated-SREG case is device-only because the +//! host implementation intentionally traps; its observed value is checked +//! against the launch configuration. The `through_mutating_index` case is +//! deliberately adversarial: its callee mutates the runtime index after MIR +//! has fixed the destination, so #1164 requires the store to keep using the +//! pre-call address. //! //! Build and run with: //! cargo oxide run mir_projected_call_destination @@ -104,6 +107,39 @@ fn through_index(mut _1: usize) -> [i32; 3] { } } +/// Callee for the destination-ordering regression. +/// +/// The call mutates the runtime index local. MIR semantics require the call +/// destination to be evaluated before entering this callee. +#[inline(never)] +fn mutate_index(_1: *mut usize) -> i32 { + unsafe { *_1 = 2 }; + 7 +} + +/// `RET[i] = mutate_index(&mut i)`: destination evaluation must precede the call. +/// +/// With `_1 = 0`, rustc fixes the destination as `RET[0]` before the callee +/// mutates `_1` to 2, so the correct result is `[7, 0, 0]`. Before #1164 the +/// importer materialized the destination after the call and wrote to `RET[2]`, +/// producing `[0, 0, 7]`; this case pins the corrected ordering. +#[custom_mir(dialect = "runtime", phase = "initial")] +#[inline(never)] +fn through_mutating_index(mut _1: usize) -> [i32; 3] { + mir! { + type RET = [i32; 3]; + let _2: *mut usize; + { + RET = [0_i32, 0_i32, 0_i32]; + _2 = core::ptr::addr_of_mut!(_1); + Call(RET[_1] = mutate_index(_2), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + Return() + } + } +} + /// `RET.1 = sqrtf32(x)` on a tuple whose other field is eight bytes wide. /// /// Unlike `bswap`, a float-math intrinsic keeps a placeholder call in the @@ -201,10 +237,119 @@ fn through_deref_sincos(_1: *mut (f32, f32), _2: f32) { } } -/// Folds the seven results into one word per case, so the device can report -/// them through a `u64` slice and the host can compare without a layout -/// assumption. -fn case_results() -> [u64; 7] { +/// `RET.1[i] = double_it(x)`: Field -> Index chain. +/// +/// The first projection selects the array field and the second selects one +/// runtime element. Both projections must be materialized before the callee +/// executes, then the result is written through that prepared address. +#[custom_mir(dialect = "runtime", phase = "initial")] +#[inline(never)] +fn through_field_index(mut _1: usize, _2: u32) -> (u64, [u32; 3]) { + mir! { + type RET = (u64, [u32; 3]); + { + RET.1 = [3_u32, 5_u32, 7_u32]; + Call(RET.1[_1] = double_it(_2), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + RET.0 = 11_u64; + Return() + } + } +} + +/// `(*p).1 = double_it(x)`: Deref -> Field chain. +#[custom_mir(dialect = "runtime", phase = "initial")] +#[inline(never)] +fn through_deref_field(_1: *mut (u64, u32), _2: u32) { + mir! { + { + Call((*_1).1 = double_it(_2), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + Return() + } + } +} + +/// `RET[i].1 = double_it(x)`: Index -> Field chain. +#[custom_mir(dialect = "runtime", phase = "initial")] +#[inline(never)] +fn through_index_field(mut _1: usize, _2: u32, _3: [(u64, u32); 2]) -> [(u64, u32); 2] { + mir! { + type RET = [(u64, u32); 2]; + { + RET = _3; + Call(RET[_1].1 = double_it(_2), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + Return() + } + } +} + +/// Bare-local control: the prepared-destination abstraction must leave the +/// established `ValueMap::store_local` path unchanged. +#[custom_mir(dialect = "runtime", phase = "initial")] +#[inline(never)] +fn through_bare_local(_1: u32) -> u32 { + mir! { + type RET = u32; + { + Call(RET = double_it(_1), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + Return() + } + } +} + +/// Generated intrinsic acceptance case. +/// +/// `thread::blockDim_x()` is generated from the SREG catalog and is replaced +/// by the importer during device compilation. Its host body intentionally +/// traps, so this helper is called only from the kernel. The nested +/// `RET.1[i]` destination proves that generated dispatch consumes the same +/// prepared-destination abstraction as ordinary calls. +#[custom_mir(dialect = "runtime", phase = "initial")] +#[inline(never)] +fn through_generated_sreg(mut _1: usize) -> (u64, [u32; 3]) { + mir! { + type RET = (u64, [u32; 3]); + { + RET.1 = [0_u32, 0_u32, 0_u32]; + Call(RET.1[_1] = thread::blockDim_x(), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + RET.0 = 29_u64; + Return() + } + } +} + +fn fold_case_results(results: &[u64; 12]) -> u64 { + let mut folded = 0_u64; + let mut i = 0_usize; + while i < results.len() { + folded ^= results[i].rotate_left((i as u32 * 5) & 63); + i += 1; + } + folded +} + +fn fold_generated_sreg(result: (u64, [u32; 3])) -> u64 { + result.0 + ^ u64::from(result.1[0]).rotate_left(8) + ^ u64::from(result.1[1]).rotate_left(24) + ^ u64::from(result.1[2]).rotate_left(40) +} + +fn expected_generated_sreg(block_dim_x: u32) -> u64 { + fold_generated_sreg((29_u64, [0_u32, block_dim_x, 0_u32])) +} + +/// Folds twelve host/device-comparable results into one word per case. +fn case_results() -> [u64; 12] { let deref = through_deref(0) as u32 as u64; let field = through_field(); @@ -215,6 +360,11 @@ fn case_results() -> [u64; 7] { ^ ((indexed[1] as u32 as u64) << 8) ^ ((indexed[2] as u32 as u64) << 16); + let mutated_index = through_mutating_index(0); + let mutating_index = (mutated_index[0] as u32 as u64) + ^ ((mutated_index[1] as u32 as u64) << 16) + ^ ((mutated_index[2] as u32 as u64) << 32); + let field_float = through_field_float(2.0_f32); let field_float = field_float.0 ^ u64::from(field_float.1.to_bits()); @@ -230,14 +380,37 @@ fn case_results() -> [u64; 7] { through_deref_sincos(&raw mut pair, 0.0_f32); let sincos = u64::from(pair.0.to_bits()) ^ (u64::from(pair.1.to_bits()) << 32); + let field_index_value = through_field_index(2, 13); + let field_index = field_index_value.0 + ^ u64::from(field_index_value.1[0]).rotate_left(8) + ^ u64::from(field_index_value.1[1]).rotate_left(24) + ^ u64::from(field_index_value.1[2]).rotate_left(40); + + let mut deref_field_value = (23_u64, 3_u32); + through_deref_field(&raw mut deref_field_value, 17); + let deref_field = deref_field_value.0 ^ u64::from(deref_field_value.1).rotate_left(32); + + let index_field_value = through_index_field(1, 19, [(17_u64, 1_u32), (19_u64, 2_u32)]); + let index_field = index_field_value[0].0 + ^ u64::from(index_field_value[0].1).rotate_left(8) + ^ index_field_value[1].0.rotate_left(24) + ^ u64::from(index_field_value[1].1).rotate_left(48); + + let bare_local = u64::from(through_bare_local(23)); + [ deref, field, index, + mutating_index, field_float, index_float, field_fn, sincos, + field_index, + deref_field, + index_field, + bare_local, ] } @@ -247,37 +420,31 @@ mod kernels { #[kernel] pub fn projected_destinations(mut out: DisjointSlice) { - let results = case_results(); + let host_comparable = fold_case_results(&case_results()); + let generated = fold_generated_sreg(through_generated_sreg(1)); if let Some(slot) = out.get_mut(thread::index_1d()) { - *slot = results[0] - ^ results[1].rotate_left(8) - ^ results[2].rotate_left(16) - ^ results[3].rotate_left(24) - ^ results[4].rotate_left(32) - ^ results[5].rotate_left(40) - ^ results[6].rotate_left(48); + *slot = host_comparable ^ generated.rotate_left(17); } } } fn main() { let host = case_results(); - let host_folded = host[0] - ^ host[1].rotate_left(8) - ^ host[2].rotate_left(16) - ^ host[3].rotate_left(24) - ^ host[4].rotate_left(32) - ^ host[5].rotate_left(40) - ^ host[6].rotate_left(48); - - println!("=== intrinsic calls writing through a projected destination ===\n"); - println!("host deref case: 0x{:016x}", host[0]); - println!("host field case: 0x{:016x}", host[1]); - println!("host index case: 0x{:016x}", host[2]); - println!("host float field case: 0x{:016x}", host[3]); - println!("host float index case: 0x{:016x}", host[4]); - println!("host fn field case: 0x{:016x}", host[5]); - println!("host sincos case: 0x{:016x}", host[6]); + let host_comparable = fold_case_results(&host); + + println!("=== calls writing through prepared projected destinations ===\n"); + println!("host deref case: 0x{:016x}", host[0]); + println!("host field case: 0x{:016x}", host[1]); + println!("host index case: 0x{:016x}", host[2]); + println!("host mutating index case: 0x{:016x}", host[3]); + println!("host float field case: 0x{:016x}", host[4]); + println!("host float index case: 0x{:016x}", host[5]); + println!("host fn field case: 0x{:016x}", host[6]); + println!("host sincos case: 0x{:016x}", host[7]); + println!("host Field -> Index: 0x{:016x}", host[8]); + println!("host Deref -> Field: 0x{:016x}", host[9]); + println!("host Index -> Field: 0x{:016x}", host[10]); + println!("host bare-local control: 0x{:016x}", host[11]); let ctx = CudaContext::new(0).expect("failed to create CUDA context"); let stream = ctx.default_stream(); @@ -289,17 +456,22 @@ fn main() { block_dim: (1, 1, 1), shared_mem_bytes: 0, }; + let generated_expected = expected_generated_sreg(cfg.block_dim.0); + let host_folded = host_comparable ^ generated_expected.rotate_left(17); // SAFETY: the one argument matches `projected_destinations`' single slice // parameter, and `out` is a live DeviceBuffer allocated above. unsafe { module.projected_destinations(&stream, cfg, &mut out) }.expect("kernel launch failed"); let device_folded = out.to_host_vec(&stream).expect("readback")[0]; + println!("host generated SREG expected: 0x{generated_expected:016x}"); println!("\nhost folded: 0x{host_folded:016x}"); println!("device folded: 0x{device_folded:016x}"); if device_folded == host_folded { - println!("\nPASS: device and host agree on all seven projections"); + println!( + "\nPASS: host/device agree on twelve call-destination cases and the generated SREG projection" + ); } else { println!("\nFAIL: device and host disagree"); std::process::exit(1); diff --git a/cuda-oxide-book/compiler/adding-new-intrinsics.md b/cuda-oxide-book/compiler/adding-new-intrinsics.md index 6d09106a8e..934272abef 100644 --- a/cuda-oxide-book/compiler/adding-new-intrinsics.md +++ b/cuda-oxide-book/compiler/adding-new-intrinsics.md @@ -155,6 +155,7 @@ arm checks the FQDN: require_arity(name, args.len(), 0, &loc)?; Ok(Some(helpers::emit_generated_nvvm_intrinsic( ctx, + body, ReadPtxSregTidXOp::get_concrete_op_info(), "v1:i0001", destination, target, block_ptr, prev_op, @@ -170,10 +171,11 @@ The FQDN is used as-is for matching -- no `::` to `__` conversion happens before the intrinsic check. The `emit_generated_nvvm_intrinsic()` helper works for *any* zero-argument, -single-result NVVM intrinsic. It creates the operation, tags it with its -catalog ABI marker, stores the result in the value map, and emits a branch to -the next basic block. You never write this arm yourself: once the reviewed overlay, -ABI, and evidence inputs resolve into `intrinsics/catalog.json`, `cuda-intrinsics-gen` emits it. +single-result NVVM intrinsic. It prepares the MIR destination before inserting +the NVVM operation, tags the operation with its catalog ABI marker, writes the +result through that prepared destination, and emits a branch to the next basic +block. You never write this arm yourself: once the reviewed overlay, ABI, and +evidence inputs resolve into `intrinsics/catalog.json`, `cuda-intrinsics-gen` emits it. Hand-written arms in `terminator/mod.rs` exist only for what the catalog cannot describe. @@ -364,11 +366,16 @@ building the op. Here is the generated arm for `shuffle_xor_sync`, condensed: let (lane_or_delta, last_op) = rvalue::translate_operand(ctx, body, &args[2], /* ... */)?; let shuffle = ShflSyncBflyI32Op::build(ctx, member_mask, value, lane_or_delta); + shuffle.deref_mut(ctx).set_loc(loc.clone()); helpers::set_generated_intrinsic_marker(ctx, shuffle, "v1:i0051"); - helpers::insert_op(ctx, shuffle, block_ptr, last_op); + let (prepared_destination, prepared_last_op) = helpers::prepare_destination_write( + ctx, body, destination, value_map, block_ptr, last_op, loc.clone(), + )?; + helpers::insert_op(ctx, shuffle, block_ptr, prepared_last_op); let result = shuffle.deref(ctx).get_result(0); - Ok(Some(helpers::emit_store_result_and_goto( - ctx, destination, result, target, /* ... */ + Ok(Some(helpers::emit_prepared_result_and_goto( + ctx, prepared_destination, result, target, block_ptr, shuffle, + value_map, block_map, loc, "shuffle_xor_sync call without target block", )?)) } @@ -376,11 +383,13 @@ building the op. Here is the generated arm for `shuffle_xor_sync`, condensed: The arm translates the user's MIR operands into pliron values -- typically the results of `mir.load`s from each operand's alloca slot, or -`mir.constant`s for literal arguments -- and wires them into the NVVM -operation. (These are not SSA values yet; `pliron::opts::mem2reg` will -collapse the load/store chains once translation is complete.) For a catalog -intrinsic you never write any of this: `cuda-intrinsics-gen` derives it from -the resolved generated catalog. +`mir.constant`s for literal arguments -- then prepares the call destination +before inserting the NVVM operation. The result is written through that +prepared destination afterward, so dynamic projections are not re-evaluated +after the intrinsic executes. (These are not SSA values yet; +`pliron::opts::mem2reg` will collapse the load/store chains once translation +is complete.) For a catalog intrinsic you never write any of this: +`cuda-intrinsics-gen` derives it from the resolved generated catalog. ### Stage 4 -- Lower to the LLVM dialect @@ -563,8 +572,9 @@ describe), every file you need to touch, in order: 3. **`mir-importer/src/translator/terminator/mod.rs`** -- `match` arm in `try_dispatch_intrinsic()`, modelled on the hand-written `typed_swap` arm. Zero-operand intrinsics are catalog territory: the generated dispatcher - emits them through `helpers::emit_generated_nvvm_intrinsic()`, which also - stamps the ABI marker a hand-written arm would have to stamp itself. + emits them through `helpers::emit_generated_nvvm_intrinsic()`, which threads + the MIR body so the destination is prepared before the generated operation + and also stamps the ABI marker a hand-written arm would have to stamp itself. 4. **`mir-lower/src/convert/interface_impls.rs`** -- `MirToLlvmConversion` impl for the new op, dispatching to the converter function.