Fix non-deterministic encoding of syntax contexts

compiler/rustc_span/src/hygiene.rs

@@ -1305,9 +1305,49 @@ pub struct HygieneEncodeContext {
 
     serialized_expns: FxHashSet<ExpnId>,
     latest_expns: FxHashSet<ExpnId>,
+
+    // Maps every `SyntaxContext` into its encoding index.
+    /// Earlier the `ctxt.0` was used when writing metadata, however,
+    /// this results into non-deterministic metadata (see #129094).
+    /// The non-determinism is encountered when decoding syntax contexts
+    /// in `decode_syntax_context` function below. The syntax contexts from
+    /// other crate metadata can be decoded in different order, which results
+    /// into different ids assigned to decoded syntax contexts.
+    /// First invocation:
+    /// (ALLOC - syntax context id, ORIG - original id of decoded syntax context:
+    /// `raw_id` in `decode_syntax_context`)
+    /// ALLOC: #3, ORIG: 1
+    /// ALLOC: #9, ORIG: 18769
+    /// ALLOC: #10, ORIG: 25868
+    /// ALLOC: #11, ORIG: 18822
+    /// ALLOC: #12, ORIG: 23092
+    ///
+    /// Second invocation:
+    /// ALLOC: #3, ORIG: 1
+    /// ALLOC: #9, ORIG: 25868
+    /// ALLOC: #10, ORIG: 18769
+    /// ALLOC: #11, ORIG: 18822
+    /// ALLOC: #12, ORIG: 23092
+    ///
+    /// We see that `18769` and `25868` assigned different syntax context ids,
+    /// however, the order of encoding is deterministic, so we can remap allocated
+    /// syntax context ids into encoding indices and use them, thus outputting
+    /// same metadata.
+    encoding_indices: FxHashMap<SyntaxContext, u32>,
 }
 
 impl HygieneEncodeContext {
+    fn get_encoding_index(&mut self, ctxt: SyntaxContext) -> u32 {
+        // Zero is taken by root syntax context.
+        if ctxt.is_root() {
+            return 0;
+        }
+
+        let map = &mut self.encoding_indices;
+        let encoding_index = map.len() + 1;
+        *map.entry(ctxt).or_insert(encoding_index as u32)
+    }
+
     /// Record the fact that we need to serialize the corresponding `ExpnData`.
     #[inline]
     pub fn schedule_expn_data_for_encoding(&mut self, expn: ExpnId) {
@@ -1351,12 +1391,17 @@ impl HygieneEncodeContext {
                         continue;
                     }
 
-                    all_ctxt_data.push((ctxt.0, data.syntax_context_data[ctxt.0 as usize].key()));
+                    all_ctxt_data.push((
+                        mut_hctxt.get_encoding_index(ctxt),
+                        data.syntax_context_data[ctxt.0 as usize].key(),
+                    ));
                 }
             });
 
             drop(mut_hctxt);
 
+            all_ctxt_data.sort_by_key(|(idx, _)| *idx);
+
             for (idx, ctxt_key) in all_ctxt_data.drain(..) {
                 encode_ctxt(encoder, idx, &ctxt_key);
             }
@@ -1515,8 +1560,10 @@ pub fn raw_encode_syntax_context(
     context: Rc<RefCell<HygieneEncodeContext>>,
     e: &mut impl Encoder,
 ) {
-    context.borrow_mut().latest_ctxts.insert(ctxt);
-    ctxt.0.encode(e);
+    let mut mut_ctxt = context.borrow_mut();
+
+    mut_ctxt.latest_ctxts.insert(ctxt);
+    mut_ctxt.get_encoding_index(ctxt).encode(e);
 }
 
 /// Updates the `disambiguator` field of the corresponding `ExpnData`

tests/run-make/parallel-reproducible-build/derives-issue-129094.rs

@@ -0,0 +1,5 @@
+#![crate_type = "lib"]
+#[derive(Clone, Copy, Hash, PartialEq, PartialOrd)]
+struct PackedPoint {
+    x: u32,
+}

tests/run-make/parallel-reproducible-build/rmake.rs

@@ -7,29 +7,38 @@ use std::rc::Rc;
 
 use run_make_support::{bin_name, is_windows_msvc, rfs, run_in_tmpdir, rustc};
 
-/// Test that parallel compiler produces identical binaries.
+/// Test that parallel compiler produces identical artifacts (binaries, metadata).
 fn main() {
-    const FILE_NAME: &str = "static-muts-issue-140413";
-    let bin_name = bin_name(FILE_NAME);
+    const TESTS: &[(&str, &[&str])] = &[
+        ("static-muts-issue-140413", &["-Zthreads=50"]),
+        ("derives-issue-129094", &["-Zthreads=16", "-Copt-level=3"]),
+    ];
 
-    let mut reference = None;
+    for (file, args) in TESTS {
+        let mut reference = None;
+        let bin_name = bin_name(file);
 
-    for _ in 0..10 {
-        // Tmp dir as previous runs affect output binary on windows.
-        run_in_tmpdir(|| {
-            let mut rustc = rustc();
-            rustc.input(format!("{FILE_NAME}.rs")).arg("-Zthreads=50").output(&bin_name);
+        for _ in 0..10 {
+            // Tmp dir as previous runs affect output binary on windows.
+            run_in_tmpdir(|| {
+                let mut rustc = rustc();
+                rustc.input(format!("{file}.rs")).output(&bin_name);
 
-            if is_windows_msvc() {
-                rustc.arg("-Clink-arg=/Brepro");
-            }
+                for arg in *args {
+                    rustc.arg(arg);
+                }
 
-            rustc.run();
+                if is_windows_msvc() {
+                    rustc.arg("-Clink-arg=/Brepro");
+                }
 
-            let current = Rc::new(rfs::read(&bin_name));
-            reference.get_or_insert(Rc::clone(&current));
+                rustc.run();
 
-            assert_eq!(Some(current), reference);
-        });
+                let current = Rc::new(rfs::read(&bin_name));
+                reference.get_or_insert(Rc::clone(&current));
+
+                assert_eq!(Some(current), reference);
+            });
+        }
     }
 }