fix(spv-out): fix large source test failed with utf8 problem

naga/src/back/spv/helpers.rs

@@ -24,9 +24,17 @@ pub(super) fn str_bytes_to_words(bytes: &[u8]) -> Vec<Word> {
     words
 }
 
+/// split a string into chunks and keep utf8 valid
+#[allow(unstable_name_collisions)]
 pub(super) fn string_to_byte_chunks(input: &str, limit: usize) -> Vec<&[u8]> {
-    let bytes = input.as_bytes();
-    let words = bytes.chunks(limit).collect();
+    let mut offset: usize = 0;
+    let mut start: usize = 0;
+    let mut words = vec![];
+    while offset < input.len() {
+        offset = input.floor_char_boundary(offset + limit);
+        words.push(input[start..offset].as_bytes());
+        start = offset;
+    }
 
     words
 }
@@ -118,3 +126,35 @@ pub fn global_needs_wrapper(ir_module: &crate::Module, var: &crate::GlobalVariab
         _ => true,
     }
 }
+
+///HACK: this is taken from std unstable, remove it when std's floor_char_boundary is stable
+trait U8Internal {
+    fn is_utf8_char_boundary(self) -> bool;
+}
+
+impl U8Internal for u8 {
+    fn is_utf8_char_boundary(self) -> bool {
+        // This is bit magic equivalent to: b < 128 || b >= 192
+        (self as i8) >= -0x40
+    }
+}
+
+trait StrUnstable {
+    fn floor_char_boundary(&self, index: usize) -> usize;
+}
+
+impl StrUnstable for str {
+    fn floor_char_boundary(&self, index: usize) -> usize {
+        if index >= self.len() {
+            self.len()
+        } else {
+            let lower_bound = index.saturating_sub(3);
+            let new_index = self.as_bytes()[lower_bound..=index]
+                .iter()
+                .rposition(|b| b.is_utf8_char_boundary());
+
+            // SAFETY: we know that the character boundary will be within four bytes
+            unsafe { lower_bound + new_index.unwrap_unchecked() }
+        }
+    }
+}

naga/tests/in/debug-symbol-large-source.wgsl

@@ -5720,6 +5720,34 @@
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
+
+
+// Taken from https://github.com/sotrh/learn-wgpu/blob/11820796f5e1dbce42fb1119f04ddeb4b167d2a0/code/intermediate/tutorial13-terrain/src/terrain.wgsl
+// ============================
+// Terrain Generation
+// ============================
+
+// https://gist.github.com/munrocket/236ed5ba7e409b8bdf1ff6eca5dcdc39
+//  MIT License. © Ian McEwan, Stefan Gustavson, Munrocket
+// - Less condensed glsl implementation with comments can be found at https://weber.itn.liu.se/~stegu/jgt2012/article.pdf
+
+fn permute3(x: vec3<f32>) -> vec3<f32> { return (((x * 34.) + 1.) * x) % vec3<f32>(289.); }
+
+fn snoise2(v: vec2<f32>) -> f32 {
+    let C = vec4<f32>(0.211324865405187, 0.366025403784439, -0.577350269189626, 0.024390243902439);
+    var i: vec2<f32> = floor(v + dot(v, C.yy));
+    let x0 = v - i + dot(i, C.xx);
+    // I flipped the condition here from > to < as it fixed some artifacting I was observing
+    var i1: vec2<f32> = select(vec2<f32>(1., 0.), vec2<f32>(0., 1.), (x0.x < x0.y));
+    var x12: vec4<f32> = x0.xyxy + C.xxzz - vec4<f32>(i1, 0., 0.);
+    i = i % vec2<f32>(289.);
+    let p = permute3(permute3(i.y + vec3<f32>(0., i1.y, 1.)) + i.x + vec3<f32>(0., i1.x, 1.));
+    var m: vec3<f32> = max(0.5 - vec3<f32>(dot(x0, x0), dot(x12.xy, x12.xy), dot(x12.zw, x12.zw)), vec3<f32>(0.));
+    m = m * m;
+    m = m * m;
+    let x = 2. * fract(p * C.www) - 1.;
+    let h = abs(x) - 0.5;
+    //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
@@ -7138,34 +7166,6 @@
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
 //This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
-//This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다 This is a test 这是测试 これはテストだ 테스트입니다
-
-
-// Taken from https://github.com/sotrh/learn-wgpu/blob/11820796f5e1dbce42fb1119f04ddeb4b167d2a0/code/intermediate/tutorial13-terrain/src/terrain.wgsl
-// ============================
-// Terrain Generation
-// ============================
-
-// https://gist.github.com/munrocket/236ed5ba7e409b8bdf1ff6eca5dcdc39
-//  MIT License. © Ian McEwan, Stefan Gustavson, Munrocket
-// - Less condensed glsl implementation with comments can be found at https://weber.itn.liu.se/~stegu/jgt2012/article.pdf
-
-fn permute3(x: vec3<f32>) -> vec3<f32> { return (((x * 34.) + 1.) * x) % vec3<f32>(289.); }
-
-fn snoise2(v: vec2<f32>) -> f32 {
-    let C = vec4<f32>(0.211324865405187, 0.366025403784439, -0.577350269189626, 0.024390243902439);
-    var i: vec2<f32> = floor(v + dot(v, C.yy));
-    let x0 = v - i + dot(i, C.xx);
-    // I flipped the condition here from > to < as it fixed some artifacting I was observing
-    var i1: vec2<f32> = select(vec2<f32>(1., 0.), vec2<f32>(0., 1.), (x0.x < x0.y));
-    var x12: vec4<f32> = x0.xyxy + C.xxzz - vec4<f32>(i1, 0., 0.);
-    i = i % vec2<f32>(289.);
-    let p = permute3(permute3(i.y + vec3<f32>(0., i1.y, 1.)) + i.x + vec3<f32>(0., i1.x, 1.));
-    var m: vec3<f32> = max(0.5 - vec3<f32>(dot(x0, x0), dot(x12.xy, x12.xy), dot(x12.zw, x12.zw)), vec3<f32>(0.));
-    m = m * m;
-    m = m * m;
-    let x = 2. * fract(p * C.www) - 1.;
-    let h = abs(x) - 0.5;
     let ox = floor(x + 0.5);
     let a0 = x - ox;
     m = m * (1.79284291400159 - 0.85373472095314 * (a0 * a0 + h * h));