diff --git a/crates/rustc_codegen_spirv/src/linker/duplicates.rs b/crates/rustc_codegen_spirv/src/linker/duplicates.rs index 6b1b45d8cd..fe637e6fe9 100644 --- a/crates/rustc_codegen_spirv/src/linker/duplicates.rs +++ b/crates/rustc_codegen_spirv/src/linker/duplicates.rs @@ -117,7 +117,7 @@ fn gather_names(debug_names: &[Instruction]) -> FxHashMap { .collect() } -fn make_dedupe_key( +fn make_dedupe_key_with_array_context( inst: &Instruction, unresolved_forward_pointers: &FxHashSet, annotations: &FxHashMap>, @@ -169,6 +169,8 @@ fn make_dedupe_key( } } + // Array context feature removed - was never actually used + data } @@ -222,7 +224,12 @@ pub fn remove_duplicate_types(module: &mut Module) { // all_inst_iter_mut pass below. However, the code is a lil bit cleaner this way I guess. rewrite_inst_with_rules(inst, &rewrite_rules); - let key = make_dedupe_key(inst, &unresolved_forward_pointers, &annotations, &names); + let key = make_dedupe_key_with_array_context( + inst, + &unresolved_forward_pointers, + &annotations, + &names, + ); match key_to_result_id.entry(key) { hash_map::Entry::Vacant(entry) => { diff --git a/crates/rustc_codegen_spirv/src/linker/specializer.rs b/crates/rustc_codegen_spirv/src/linker/specializer.rs index 01c4c9f9b1..0db9fa32bb 100644 --- a/crates/rustc_codegen_spirv/src/linker/specializer.rs +++ b/crates/rustc_codegen_spirv/src/linker/specializer.rs @@ -57,6 +57,7 @@ use rspirv::spirv::{Op, StorageClass, Word}; use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use smallvec::SmallVec; +use std::cell::RefCell; use std::collections::{BTreeMap, VecDeque}; use std::ops::{Range, RangeTo}; use std::{fmt, io, iter, mem, slice}; @@ -128,11 +129,23 @@ pub fn specialize( .collect(); } + let spirv_version = module.header.as_ref().map_or((1, 0), |h| h.version()); + let has_workgroup_layout_capability = module.capabilities.iter().any(|inst| { + inst.class.opcode == Op::Capability + && inst.operands.first() + == Some(&Operand::Capability( + rspirv::spirv::Capability::WorkgroupMemoryExplicitLayoutKHR, + )) + }); + let mut specializer = Specializer { specialization, debug_names, generics: IndexMap::new(), int_consts: FxHashMap::default(), + array_layout_variants: RefCell::new(FxHashMap::default()), + spirv_version, + has_workgroup_layout_capability, }; specializer.collect_generics(&module); @@ -524,6 +537,29 @@ struct Generic { replacements: Replacements, } +/// For every `OpTypeArray` (identified by its defining instruction result ID), we may +/// need two physical variants that differ only by the presence of an `ArrayStride` +/// decoration. +/// +/// * `layout_required_id` – An array type *with* `ArrayStride` attached. This variant +/// must be used whenever the SPIR-V storage class context *requires* explicit +/// layout information (e.g. `StorageBuffer`). In most existing modules this will be +/// the original array type that rust-gpu produced. +/// * `layout_forbidden_id` – An array type *without* `ArrayStride`. This variant must +/// be used in storage classes where explicit layout is disallowed (e.g. +/// `Workgroup`, `Private`, `Function` in SPIR-V ≥ 1.4 without special +/// capabilities). +/// +/// Not every module needs both variants – e.g. an array only ever used in +/// `StorageBuffer` memory does not require the `layout_forbidden_id`. Therefore the +/// two fields are optional and are filled on demand by the storage-class inference +/// pass when it discovers the need for a particular variant. +#[derive(Clone, Debug, Default)] +struct ArrayLayoutVariants { + /// Variant that *omits* the `ArrayStride` decoration. + layout_forbidden_id: Option, +} + struct Specializer { specialization: S, @@ -536,6 +572,16 @@ struct Specializer { /// Integer `OpConstant`s (i.e. containing a `LiteralBit32`), to be used /// for interpreting `TyPat::IndexComposite` (such as for `OpAccessChain`). int_consts: FxHashMap, + + /// Lazily-populated map holding, for every original array type ID, the IDs of + /// its specialised layout-`required` / layout-`forbidden` clones (if any). + array_layout_variants: RefCell>, + + /// SPIR-V version of the current module (major, minor) + spirv_version: (u8, u8), + + /// Whether the module enables `WorkgroupMemoryExplicitLayoutKHR` + has_workgroup_layout_capability: bool, } impl Specializer { @@ -680,6 +726,17 @@ impl Specializer { Some(replacements) } } + + /// Return `true` if the given storage class allows explicit layout decorations + /// (i.e. `ArrayStride`). + fn allows_layout(&self, storage_class: StorageClass) -> bool { + match storage_class { + StorageClass::UniformConstant => false, + StorageClass::Workgroup => self.has_workgroup_layout_capability, + StorageClass::Function | StorageClass::Private => self.spirv_version < (1, 4), + _ => true, + } + } } /// Newtype'd inference variable index. @@ -2202,10 +2259,25 @@ struct Expander<'a, S: Specialization> { /// own `replacements` analyzed in order to fully collect all instances. // FIXME(eddyb) fine-tune the length of `SmallVec<[_; 4]>` here. propagate_instances_queue: VecDeque>>, + + /// Snapshot of all original type/global instructions keyed by their result ID. + /// This is necessary because during expansion we temporarily move the original + /// `types_global_values` out of the `Module`, making them inaccessible to helper + /// routines (such as `get_array_variant`). + original_types: FxHashMap, } impl<'a, S: Specialization> Expander<'a, S> { fn new(specializer: &'a Specializer, module: Module) -> Self { + // Build a lookup table of all original type/global instructions before moving + // `module` into the internal `Builder`. + let mut original_types = FxHashMap::default(); + for inst in &module.types_global_values { + if let Some(id) = inst.result_id { + original_types.insert(id, inst.clone()); + } + } + Expander { specializer, @@ -2213,6 +2285,8 @@ impl<'a, S: Specialization> Expander<'a, S> { instances: BTreeMap::new(), propagate_instances_queue: VecDeque::new(), + + original_types, } } @@ -2354,28 +2428,74 @@ impl<'a, S: Specialization> Expander<'a, S> { // Expand `Op(Member)Decorate* %target ...`, when `target` is "generic". let mut expanded_annotations = expand_debug_or_annotation(annotations); - // Expand "generic" globals (types, constants and module-scoped variables). + // Will store any additional type declarations (array variants, etc.) that we might + // need to generate while expanding pointer types. let mut expanded_types_global_values = Vec::with_capacity(types_global_values.len().next_power_of_two()); for inst in types_global_values { if let Some(result_id) = inst.result_id { if let Some(generic) = self.specializer.generics.get(&result_id) { - expanded_types_global_values.extend(self.all_instances_of(result_id).map( - |(instance, &instance_id)| { - let mut expanded_inst = inst.clone(); - expanded_inst.result_id = Some(instance_id); - for (loc, operand) in generic - .replacements - .to_concrete(&instance.generic_args, |i| self.instances[&i]) - { - expanded_inst.index_set(loc, operand.into()); + // Collect instances first to avoid borrowing `self` immutably and mutably at the same time. + let instance_info: Vec<_> = self + .all_instances_of(result_id) + .map(|(inst, &inst_id)| (inst.clone(), inst_id)) + .collect(); + + for (instance, instance_id) in instance_info { + let mut expanded_inst = inst.clone(); + expanded_inst.result_id = Some(instance_id); + for (loc, operand) in generic + .replacements + .to_concrete(&instance.generic_args, |i| self.instances[&i]) + { + expanded_inst.index_set(loc, operand.into()); + } + + // If this is a pointer type now specialized to a concrete storage + // class, ensure its pointee array type respects layout rules. + if expanded_inst.class.opcode == Op::TypePointer { + if let ( + Some(Operand::StorageClass(sc)), + Some(Operand::IdRef(pointee_id)), + ) = ( + expanded_inst.operands.first().cloned(), + expanded_inst.operands.get(1).cloned(), + ) { + let need_no_stride = !self.specializer.allows_layout(sc); + let variant_id = self.get_array_variant( + pointee_id, + need_no_stride, + &mut expanded_types_global_values, + ); + if variant_id != pointee_id { + expanded_inst.operands[1] = Operand::IdRef(variant_id); + } } - expanded_inst - }, - )); + } + + expanded_types_global_values.push(expanded_inst); + } continue; } } + let mut inst = inst; + // Ensure even non-generic `OpTypePointer`s use the proper array variant. + if inst.class.opcode == Op::TypePointer { + if let (Some(Operand::StorageClass(sc)), Some(Operand::IdRef(pointee_id))) = ( + inst.operands.first().cloned(), + inst.operands.get(1).cloned(), + ) { + let need_no_stride = !self.specializer.allows_layout(sc); + let variant_id = self.get_array_variant( + pointee_id, + need_no_stride, + &mut expanded_types_global_values, + ); + if variant_id != pointee_id { + inst.operands[1] = Operand::IdRef(variant_id); + } + } + } expanded_types_global_values.push(inst); } @@ -2463,9 +2583,52 @@ impl<'a, S: Specialization> Expander<'a, S> { module.entry_points = entry_points; module.debug_names = expanded_debug_names; module.annotations = expanded_annotations; - module.types_global_values = expanded_types_global_values; + module.types_global_values = expanded_types_global_values.clone(); module.functions = expanded_functions; + // `expanded_types_global_values` already contains any new type declarations (array variants). + module.types_global_values = expanded_types_global_values; + + // Pass 2: rewrite result types of value/composite instructions that still reference the + // *original* (layout-bearing) array types to their stride-less specialised variants. + if !self.specializer.array_layout_variants.borrow().is_empty() { + let mut variant_map = FxHashMap::::default(); + for (orig, variants) in self.specializer.array_layout_variants.borrow().iter() { + if let Some(forbidden) = variants.layout_forbidden_id { + variant_map.insert(*orig, forbidden); + } + } + + // Helper closure: given a mutable instruction, patch its result type if needed. + let patch_inst = |inst: &mut Instruction| { + if let Some(rt) = inst.result_type { + if let Some(&new_id) = variant_map.get(&rt) { + inst.result_type = Some(new_id); + } + } + }; + + // Patch globals (constants etc.). + for inst in &mut module.types_global_values { + patch_inst(inst); + } + + // Patch every instruction inside every function. + for func in &mut module.functions { + if let Some(def) = &mut func.def { + patch_inst(def); + } + for blk in &mut func.blocks { + if let Some(lbl) = &mut blk.label { + patch_inst(lbl); + } + for inst in &mut blk.instructions { + patch_inst(inst); + } + } + } + } + self.builder.module() } @@ -2551,4 +2714,126 @@ impl<'a, S: Specialization> Expander<'a, S> { } Ok(()) } + + /// Obtain (or lazily create) an array type variant that either keeps or removes + /// the `ArrayStride` decoration, according to `need_no_stride`. + /// + /// * `need_no_stride == false` => layout-required/allowed, return original ID. + /// * `need_no_stride == true` => layout-forbidden, create a stride-less clone if + /// one does not yet exist. + fn get_array_variant( + &mut self, + original_id: Word, + need_no_stride: bool, + new_types: &mut Vec, + ) -> Word { + // Look up the defining instruction from the cached map, as the original + // `types_global_values` list has been temporarily moved out of the `Module` + // during expansion. + let original_inst = match self.original_types.get(&original_id) { + Some(i) => i.clone(), + None => return original_id, + }; + + // Fast-path if we don't need to strip layout information. + if !need_no_stride { + return original_id; + } + + // If we already produced a stride-less variant for this type, just return it. + if let Some(id) = self + .specializer + .array_layout_variants + .borrow() + .get(&original_id) + .and_then(|v| v.layout_forbidden_id) + { + return id; + } + + match original_inst.class.opcode { + Op::TypeArray | Op::TypeRuntimeArray => { + // Handle arrays: remove the `ArrayStride` decoration (by simply cloning the + // instruction – the decoration lives in `OpDecorate`). + + let new_id = self.builder.id(); + let mut new_inst = original_inst.clone(); + new_inst.result_id = Some(new_id); + + // Recurse into the element type (arrays of arrays). + if let Some(&Operand::IdRef(elem_ty)) = original_inst.operands.first() { + let nested_variant = self.get_array_variant(elem_ty, need_no_stride, new_types); + if nested_variant != elem_ty { + new_inst.operands[0] = Operand::IdRef(nested_variant); + } + } + + new_types.push(new_inst.clone()); + // Cache definition for later lookups. + self.original_types.insert(new_id, new_inst); + + self.specializer + .array_layout_variants + .borrow_mut() + .entry(original_id) + .or_default() + .layout_forbidden_id = Some(new_id); + + new_id + } + + Op::TypeStruct => { + // For structs, we need to create a clone that references the stride-less + // variants of any member types that themselves changed. + + // Track whether any member type changed – if not, we can reuse the original. + let mut changed = false; + let mut new_operands = original_inst.operands.clone(); + + for (idx, op) in original_inst.operands.iter().enumerate() { + if let Operand::IdRef(member_ty) = op { + let new_member_ty = + self.get_array_variant(*member_ty, need_no_stride, new_types); + if new_member_ty != *member_ty { + new_operands[idx] = Operand::IdRef(new_member_ty); + changed = true; + } + } + } + + if !changed { + // Even if no member type changed, cache lookup to avoid redundant work. + self.specializer + .array_layout_variants + .borrow_mut() + .entry(original_id) + .or_default() + .layout_forbidden_id = Some(original_id); + return original_id; + } + + let new_id = self.builder.id(); + let mut new_inst = original_inst.clone(); + new_inst.result_id = Some(new_id); + new_inst.operands = new_operands; + + new_types.push(new_inst.clone()); + self.original_types.insert(new_id, new_inst); + + self.specializer + .array_layout_variants + .borrow_mut() + .entry(original_id) + .or_default() + .layout_forbidden_id = Some(new_id); + + new_id + } + + _ => { + // Not an array or struct – nothing to do. + original_id + } + } + } } diff --git a/tests/ui/linker/array_stride_fixer/function_storage_spirv13_kept.rs b/tests/ui/linker/array_stride_fixer/function_storage_spirv13_kept.rs new file mode 100644 index 0000000000..c7219048b2 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/function_storage_spirv13_kept.rs @@ -0,0 +1,21 @@ +// Test that ArrayStride decorations are kept for function storage in SPIR-V 1.3 + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +// only-spv1.3 +use spirv_std::spirv; + +#[spirv(compute(threads(1)))] +pub fn main( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] output: &mut [u32; 1], +) { + // Function storage in SPIR-V 1.3 should keep ArrayStride decorations + let mut function_var: [u32; 256] = [0; 256]; + function_var[0] = 42; + function_var[1] = function_var[0] + 1; + // Force the array to be used by writing to output + output[0] = function_var[1]; +} diff --git a/tests/ui/linker/array_stride_fixer/function_storage_spirv13_kept.stderr b/tests/ui/linker/array_stride_fixer/function_storage_spirv13_kept.stderr new file mode 100644 index 0000000000..5c2290c73a --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/function_storage_spirv13_kept.stderr @@ -0,0 +1,30 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpExtension "SPV_KHR_shader_clock" +OpMemoryModel Logical Simple +OpEntryPoint GLCompute %1 "main" +OpExecutionMode %1 LocalSize 1 1 1 +%2 = OpString "$OPSTRING_FILENAME/function_storage_spirv13_kept.rs" +OpDecorate %4 ArrayStride 4 +OpDecorate %5 Block +OpMemberDecorate %5 0 Offset 0 +OpDecorate %3 Binding 0 +OpDecorate %3 DescriptorSet 0 +%6 = OpTypeVoid +%7 = OpTypeFunction %6 +%8 = OpTypeInt 32 0 +%9 = OpConstant %8 1 +%4 = OpTypeArray %8 %9 +%10 = OpTypePointer StorageBuffer %4 +%5 = OpTypeStruct %4 +%11 = OpTypePointer StorageBuffer %5 +%3 = OpVariable %11 StorageBuffer +%12 = OpConstant %8 0 +%13 = OpTypeBool +%14 = OpConstant %8 256 +%15 = OpConstant %8 42 +%16 = OpTypePointer StorageBuffer %8 diff --git a/tests/ui/linker/array_stride_fixer/mixed_storage_classes.rs b/tests/ui/linker/array_stride_fixer/mixed_storage_classes.rs new file mode 100644 index 0000000000..9947ce3426 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/mixed_storage_classes.rs @@ -0,0 +1,21 @@ +// Test that mixed storage class usage results in proper ArrayStride handling + +// compile-flags: -C llvm-args=--disassemble-globals +// only-vulkan1.1 +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +use spirv_std::spirv; + +#[spirv(compute(threads(64)))] +pub fn main( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] storage_data: &mut [u32; 256], + #[spirv(workgroup)] workgroup_data: &mut [u32; 256], +) { + // Both variables use the same array type [u32; 256] but in different storage classes: + // - storage_data is in StorageBuffer (requires ArrayStride) + // - workgroup_data is in Workgroup (forbids ArrayStride in SPIR-V 1.4+) + + storage_data[0] = 42; + workgroup_data[0] = storage_data[0]; +} diff --git a/tests/ui/linker/array_stride_fixer/mixed_storage_classes.stderr b/tests/ui/linker/array_stride_fixer/mixed_storage_classes.stderr new file mode 100644 index 0000000000..a6d066e4ae --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/mixed_storage_classes.stderr @@ -0,0 +1,36 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_shader_clock" +OpExtension "SPV_KHR_vulkan_memory_model" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %1 "main" +OpExecutionMode %1 LocalSize 64 1 1 +%2 = OpString "$OPSTRING_FILENAME/mixed_storage_classes.rs" +OpName %4 "workgroup_data" +OpDecorate %5 ArrayStride 4 +OpDecorate %6 Block +OpMemberDecorate %6 0 Offset 0 +OpDecorate %3 Binding 0 +OpDecorate %3 DescriptorSet 0 +%7 = OpTypeVoid +%8 = OpTypeFunction %7 +%9 = OpTypeInt 32 0 +%10 = OpConstant %9 256 +%5 = OpTypeArray %9 %10 +%11 = OpTypePointer StorageBuffer %5 +%6 = OpTypeStruct %5 +%12 = OpTypePointer StorageBuffer %6 +%3 = OpVariable %12 StorageBuffer +%13 = OpConstant %9 0 +%14 = OpTypeBool +%15 = OpTypePointer StorageBuffer %9 +%16 = OpConstant %9 42 +%17 = OpTypePointer Workgroup %9 +%18 = OpTypeArray %9 %10 +%19 = OpTypePointer Workgroup %18 +%4 = OpVariable %19 Workgroup diff --git a/tests/ui/linker/array_stride_fixer/nested_structs_function_storage.rs b/tests/ui/linker/array_stride_fixer/nested_structs_function_storage.rs new file mode 100644 index 0000000000..3407f66b90 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/nested_structs_function_storage.rs @@ -0,0 +1,34 @@ +// Test that ArrayStride decorations are removed from nested structs with arrays in Function storage class + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals +// only-vulkan1.2 +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +use spirv_std::spirv; + +#[derive(Copy, Clone)] +struct InnerStruct { + data: [f32; 4], +} + +#[derive(Copy, Clone)] +struct OuterStruct { + inner: InnerStruct, +} + +#[spirv(compute(threads(1)))] +pub fn main( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] output: &mut [f32; 1], +) { + // Function-local variables with nested structs containing arrays + // Should have ArrayStride removed in SPIR-V 1.4+ + let mut function_var = OuterStruct { + inner: InnerStruct { data: [0.0; 4] }, + }; + function_var.inner.data[0] = 42.0; + function_var.inner.data[1] = function_var.inner.data[0] + 1.0; + // Force usage to prevent optimization + output[0] = function_var.inner.data[1]; +} diff --git a/tests/ui/linker/array_stride_fixer/nested_structs_function_storage.stderr b/tests/ui/linker/array_stride_fixer/nested_structs_function_storage.stderr new file mode 100644 index 0000000000..340badc371 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/nested_structs_function_storage.stderr @@ -0,0 +1,40 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_shader_clock" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %1 "main" %2 +OpExecutionMode %1 LocalSize 1 1 1 +%3 = OpString "$OPSTRING_FILENAME/nested_structs_function_storage.rs" +OpDecorate %4 ArrayStride 4 +OpDecorate %5 Block +OpMemberDecorate %5 0 Offset 0 +OpDecorate %2 Binding 0 +OpDecorate %2 DescriptorSet 0 +%6 = OpTypeFloat 32 +%7 = OpTypeInt 32 0 +%8 = OpConstant %7 1 +%4 = OpTypeArray %6 %8 +%5 = OpTypeStruct %4 +%9 = OpTypePointer StorageBuffer %5 +%10 = OpTypeVoid +%11 = OpTypeFunction %10 +%12 = OpTypePointer StorageBuffer %4 +%2 = OpVariable %9 StorageBuffer +%13 = OpConstant %7 0 +%14 = OpConstant %7 4 +%15 = OpTypeArray %6 %14 +%16 = OpTypeStruct %15 +%17 = OpTypeStruct %16 +%18 = OpConstant %6 0 +%19 = OpConstantComposite %15 %18 %18 %18 %18 +%20 = OpUndef %17 +%21 = OpTypeBool +%22 = OpConstant %6 1109917696 +%23 = OpConstant %6 1065353216 +%24 = OpTypePointer StorageBuffer %6 + diff --git a/tests/ui/linker/array_stride_fixer/private_storage_spirv14_removed.rs b/tests/ui/linker/array_stride_fixer/private_storage_spirv14_removed.rs new file mode 100644 index 0000000000..19de8523be --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/private_storage_spirv14_removed.rs @@ -0,0 +1,22 @@ +// Test that ArrayStride decorations are removed from private storage in SPIR-V 1.4 + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +// only-spv1.4 +use spirv_std::spirv; + +// Helper function to create an array in private storage +fn create_private_array() -> [u32; 4] { + [0, 1, 2, 3] +} + +#[spirv(compute(threads(1)))] +pub fn main() { + // This creates a private storage array in SPIR-V 1.4+ + // ArrayStride decorations should be removed + let mut private_array = create_private_array(); + private_array[0] = 42; +} diff --git a/tests/ui/linker/array_stride_fixer/private_storage_spirv14_removed.stderr b/tests/ui/linker/array_stride_fixer/private_storage_spirv14_removed.stderr new file mode 100644 index 0000000000..2eff9c5bee --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/private_storage_spirv14_removed.stderr @@ -0,0 +1,22 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpExtension "SPV_KHR_shader_clock" +OpMemoryModel Logical Simple +OpEntryPoint GLCompute %1 "main" +OpExecutionMode %1 LocalSize 1 1 1 +%2 = OpString "$OPSTRING_FILENAME/private_storage_spirv14_removed.rs" +%4 = OpTypeVoid +%5 = OpTypeFunction %4 +%6 = OpTypeInt 32 0 +%7 = OpConstant %6 4 +%8 = OpTypeArray %6 %7 +%9 = OpTypeFunction %8 +%10 = OpConstant %6 0 +%11 = OpConstant %6 1 +%12 = OpConstant %6 2 +%13 = OpConstant %6 3 +%14 = OpTypeBool diff --git a/tests/ui/linker/array_stride_fixer/runtime_arrays_in_workgroup.rs b/tests/ui/linker/array_stride_fixer/runtime_arrays_in_workgroup.rs new file mode 100644 index 0000000000..9c486544ab --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/runtime_arrays_in_workgroup.rs @@ -0,0 +1,22 @@ +// Test that ArrayStride decorations are removed from runtime arrays in Workgroup storage class + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals +// only-vulkan1.1 +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +use spirv_std::RuntimeArray; +use spirv_std::spirv; + +#[spirv(compute(threads(64)))] +pub fn main( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] output: &mut [u32; 1], + #[spirv(workgroup)] shared_array: &mut [u32; 256], +) { + // Workgroup arrays should have ArrayStride removed + shared_array[0] = 42; + shared_array[1] = shared_array[0] + 1; + // Force usage to prevent optimization + output[0] = shared_array[1]; +} diff --git a/tests/ui/linker/array_stride_fixer/runtime_arrays_in_workgroup.stderr b/tests/ui/linker/array_stride_fixer/runtime_arrays_in_workgroup.stderr new file mode 100644 index 0000000000..42ca0186aa --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/runtime_arrays_in_workgroup.stderr @@ -0,0 +1,37 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_shader_clock" +OpExtension "SPV_KHR_vulkan_memory_model" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %1 "main" +OpExecutionMode %1 LocalSize 64 1 1 +%2 = OpString "$OPSTRING_FILENAME/runtime_arrays_in_workgroup.rs" +OpName %4 "shared_array" +OpDecorate %5 ArrayStride 4 +OpDecorate %6 Block +OpMemberDecorate %6 0 Offset 0 +OpDecorate %3 Binding 0 +OpDecorate %3 DescriptorSet 0 +%7 = OpTypeVoid +%8 = OpTypeFunction %7 +%9 = OpTypeInt 32 0 +%10 = OpConstant %9 1 +%5 = OpTypeArray %9 %10 +%11 = OpTypePointer StorageBuffer %5 +%6 = OpTypeStruct %5 +%12 = OpTypePointer StorageBuffer %6 +%3 = OpVariable %12 StorageBuffer +%13 = OpConstant %9 0 +%14 = OpTypeBool +%15 = OpConstant %9 256 +%16 = OpTypePointer Workgroup %9 +%17 = OpTypeArray %9 %15 +%18 = OpTypePointer Workgroup %17 +%4 = OpVariable %18 Workgroup +%19 = OpConstant %9 42 +%20 = OpTypePointer StorageBuffer %9 diff --git a/tests/ui/linker/array_stride_fixer/storage_buffer_arrays_kept.rs b/tests/ui/linker/array_stride_fixer/storage_buffer_arrays_kept.rs new file mode 100644 index 0000000000..5b9261a8b0 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/storage_buffer_arrays_kept.rs @@ -0,0 +1,17 @@ +// Test that ArrayStride decorations are kept for arrays in StorageBuffer storage class + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals +// only-vulkan1.1 +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +use spirv_std::spirv; + +#[spirv(compute(threads(1)))] +pub fn main( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] storage_buffer_var: &mut [u32; 256], +) { + // StorageBuffer storage class should keep ArrayStride decorations + storage_buffer_var[0] = 42; +} diff --git a/tests/ui/linker/array_stride_fixer/storage_buffer_arrays_kept.stderr b/tests/ui/linker/array_stride_fixer/storage_buffer_arrays_kept.stderr new file mode 100644 index 0000000000..5f0007f120 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/storage_buffer_arrays_kept.stderr @@ -0,0 +1,31 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_shader_clock" +OpExtension "SPV_KHR_vulkan_memory_model" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %1 "main" +OpExecutionMode %1 LocalSize 1 1 1 +%2 = OpString "$OPSTRING_FILENAME/storage_buffer_arrays_kept.rs" +OpDecorate %4 ArrayStride 4 +OpDecorate %5 Block +OpMemberDecorate %5 0 Offset 0 +OpDecorate %3 Binding 0 +OpDecorate %3 DescriptorSet 0 +%6 = OpTypeVoid +%7 = OpTypeFunction %6 +%8 = OpTypeInt 32 0 +%9 = OpConstant %8 256 +%4 = OpTypeArray %8 %9 +%10 = OpTypePointer StorageBuffer %4 +%5 = OpTypeStruct %4 +%11 = OpTypePointer StorageBuffer %5 +%3 = OpVariable %11 StorageBuffer +%12 = OpConstant %8 0 +%13 = OpTypeBool +%14 = OpTypePointer StorageBuffer %8 +%15 = OpConstant %8 42 diff --git a/tests/ui/linker/array_stride_fixer/workgroup_2d_arrays_issue.rs b/tests/ui/linker/array_stride_fixer/workgroup_2d_arrays_issue.rs new file mode 100644 index 0000000000..c01c4108c2 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/workgroup_2d_arrays_issue.rs @@ -0,0 +1,30 @@ +// Test that reproduces the OpInBoundsAccessChain type mismatch issue +// with workgroup 2D arrays after array_stride_fixer changes + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals +// only-vulkan1.1 +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" + +use spirv_std::spirv; + +const TILE_SIZE: u32 = 32; + +#[spirv(compute(threads(32, 32)))] +pub fn transpose_2d_workgroup( + #[spirv(local_invocation_id)] lid: spirv_std::glam::UVec3, + #[spirv(workgroup)] shared_real: &mut [[f32; TILE_SIZE as usize]; TILE_SIZE as usize], + #[spirv(workgroup)] shared_imag: &mut [[f32; TILE_SIZE as usize]; TILE_SIZE as usize], +) { + let lx = lid.x as usize; + let ly = lid.y as usize; + + // This should trigger the OpInBoundsAccessChain issue + shared_real[ly][lx] = 1.0; + shared_imag[ly][lx] = 2.0; + + // Read back to ensure usage + let _val = shared_real[lx][ly] + shared_imag[lx][ly]; +} diff --git a/tests/ui/linker/array_stride_fixer/workgroup_2d_arrays_issue.stderr b/tests/ui/linker/array_stride_fixer/workgroup_2d_arrays_issue.stderr new file mode 100644 index 0000000000..95bac54330 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/workgroup_2d_arrays_issue.stderr @@ -0,0 +1,34 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_shader_clock" +OpExtension "SPV_KHR_vulkan_memory_model" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %1 "transpose_2d_workgroup" %2 +OpExecutionMode %1 LocalSize 32 32 1 +%3 = OpString "$OPSTRING_FILENAME/workgroup_2d_arrays_issue.rs" +OpName %4 "shared_real" +OpName %5 "shared_imag" +OpDecorate %2 BuiltIn LocalInvocationId +%6 = OpTypeInt 32 0 +%7 = OpTypeVector %6 3 +%8 = OpTypePointer Input %7 +%9 = OpTypeVoid +%10 = OpTypeFunction %9 +%2 = OpVariable %8 Input +%11 = OpTypeBool +%12 = OpConstant %6 32 +%13 = OpTypeFloat 32 +%14 = OpTypeArray %13 %12 +%15 = OpTypePointer Workgroup %14 +%16 = OpTypeArray %14 %12 +%17 = OpTypePointer Workgroup %16 +%4 = OpVariable %17 Workgroup +%18 = OpTypePointer Workgroup %13 +%19 = OpConstant %13 1065353216 +%5 = OpVariable %17 Workgroup +%20 = OpConstant %13 1073741824 diff --git a/tests/ui/linker/array_stride_fixer/workgroup_arrays_removed.rs b/tests/ui/linker/array_stride_fixer/workgroup_arrays_removed.rs new file mode 100644 index 0000000000..967113c70f --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/workgroup_arrays_removed.rs @@ -0,0 +1,21 @@ +// Test that ArrayStride decorations are removed from arrays in Function storage class (SPIR-V 1.4+) + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals +// only-vulkan1.2 +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +use spirv_std::spirv; + +#[spirv(compute(threads(64)))] +pub fn main( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] output: &mut [u32; 1], + #[spirv(workgroup)] shared_data: &mut [u32; 256], +) { + // Workgroup storage arrays should have ArrayStride removed + shared_data[0] = 42; + shared_data[1] = shared_data[0] + 1; + // Force usage to prevent optimization + output[0] = shared_data[1]; +} diff --git a/tests/ui/linker/array_stride_fixer/workgroup_arrays_removed.stderr b/tests/ui/linker/array_stride_fixer/workgroup_arrays_removed.stderr new file mode 100644 index 0000000000..3f769cba39 --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/workgroup_arrays_removed.stderr @@ -0,0 +1,36 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability ShaderClockKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_shader_clock" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %1 "main" %2 %3 +OpExecutionMode %1 LocalSize 64 1 1 +%4 = OpString "$OPSTRING_FILENAME/workgroup_arrays_removed.rs" +OpName %3 "shared_data" +OpDecorate %5 ArrayStride 4 +OpDecorate %6 Block +OpMemberDecorate %6 0 Offset 0 +OpDecorate %2 Binding 0 +OpDecorate %2 DescriptorSet 0 +%7 = OpTypeInt 32 0 +%8 = OpConstant %7 1 +%5 = OpTypeArray %7 %8 +%6 = OpTypeStruct %5 +%9 = OpTypePointer StorageBuffer %6 +%10 = OpConstant %7 256 +%11 = OpTypeArray %7 %10 +%12 = OpTypePointer Workgroup %11 +%13 = OpTypeVoid +%14 = OpTypeFunction %13 +%15 = OpTypePointer StorageBuffer %5 +%2 = OpVariable %9 StorageBuffer +%16 = OpConstant %7 0 +%17 = OpTypeBool +%18 = OpTypePointer Workgroup %7 +%3 = OpVariable %12 Workgroup +%19 = OpConstant %7 42 +%20 = OpTypePointer StorageBuffer %7 diff --git a/tests/ui/linker/array_stride_fixer/workgroup_arrays_with_capability.rs b/tests/ui/linker/array_stride_fixer/workgroup_arrays_with_capability.rs new file mode 100644 index 0000000000..505809003e --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/workgroup_arrays_with_capability.rs @@ -0,0 +1,22 @@ +// Test that ArrayStride decorations are kept for arrays in Workgroup storage class with WorkgroupMemoryExplicitLayoutKHR capability + +// build-pass +// compile-flags: -C llvm-args=--disassemble-globals -Ctarget-feature=+WorkgroupMemoryExplicitLayoutKHR,+ext:SPV_KHR_workgroup_memory_explicit_layout +// normalize-stderr-test "OpLine .*\n" -> "" +// normalize-stderr-test "OpSource .*\n" -> "" +// normalize-stderr-test "\S*/lib/rustlib/" -> "$SYSROOT/lib/rustlib/" +// only-vulkan1.2 + +use spirv_std::spirv; + +#[spirv(compute(threads(64)))] +pub fn main( + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] output: &mut [u32; 1], + #[spirv(workgroup)] shared_data: &mut [u32; 256], +) { + // With WorkgroupMemoryExplicitLayoutKHR capability, ArrayStride should be kept + shared_data[0] = 42; + shared_data[1] = shared_data[0] + 1; + // Force usage to prevent optimization + output[0] = shared_data[1]; +} diff --git a/tests/ui/linker/array_stride_fixer/workgroup_arrays_with_capability.stderr b/tests/ui/linker/array_stride_fixer/workgroup_arrays_with_capability.stderr new file mode 100644 index 0000000000..a645d0bcbd --- /dev/null +++ b/tests/ui/linker/array_stride_fixer/workgroup_arrays_with_capability.stderr @@ -0,0 +1,39 @@ +OpCapability Shader +OpCapability Float64 +OpCapability Int64 +OpCapability Int16 +OpCapability Int8 +OpCapability WorkgroupMemoryExplicitLayoutKHR +OpCapability ShaderClockKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_shader_clock" +OpExtension "SPV_KHR_workgroup_memory_explicit_layout" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %1 "main" %2 %3 +OpExecutionMode %1 LocalSize 64 1 1 +%4 = OpString "$OPSTRING_FILENAME/workgroup_arrays_with_capability.rs" +OpName %3 "shared_data" +OpDecorate %5 ArrayStride 4 +OpDecorate %6 Block +OpMemberDecorate %6 0 Offset 0 +OpDecorate %7 ArrayStride 4 +OpDecorate %2 Binding 0 +OpDecorate %2 DescriptorSet 0 +%8 = OpTypeInt 32 0 +%9 = OpConstant %8 1 +%5 = OpTypeArray %8 %9 +%6 = OpTypeStruct %5 +%10 = OpTypePointer StorageBuffer %6 +%11 = OpConstant %8 256 +%7 = OpTypeArray %8 %11 +%12 = OpTypePointer Workgroup %7 +%13 = OpTypeVoid +%14 = OpTypeFunction %13 +%15 = OpTypePointer StorageBuffer %5 +%2 = OpVariable %10 StorageBuffer +%16 = OpConstant %8 0 +%17 = OpTypeBool +%18 = OpTypePointer Workgroup %8 +%3 = OpVariable %12 Workgroup +%19 = OpConstant %8 42 +%20 = OpTypePointer StorageBuffer %8