Use horde's SyncTable for default query caches and CtxtInterners

Cargo.lock

@@ -1709,6 +1709,15 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "horde"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3d7c629771c2e116e71d8e7bbc6e6e0450a8817766a7230bb0c98f81311df34"
+dependencies = [
+ "parking_lot",
+]
+
 [[package]]
 name = "html-checker"
 version = "0.1.0"
@@ -3803,6 +3812,7 @@ dependencies = [
  "elsa",
  "ena",
  "hashbrown 0.16.1",
+ "horde",
  "indexmap",
  "jobserver",
  "libc",

compiler/rustc_data_structures/Cargo.toml

@@ -10,6 +10,7 @@ bitflags = "2.4.1"
 either = "1.0"
 elsa = "1.11.0"
 ena = "0.14.4"
+horde = { version = "0.1.2", features = ["nightly"] }
 indexmap = "2.12.1"
 jobserver_crate = { version = "0.1.28", package = "jobserver" }
 measureme = "12.0.1"

compiler/rustc_data_structures/src/sharded.rs

@@ -201,60 +201,6 @@ impl<K: Eq + Hash, V> ShardedHashMap<K, V> {
     }
 }
 
-impl<K: Eq + Hash + Copy> ShardedHashMap<K, ()> {
-    #[inline]
-    pub fn intern_ref<Q: ?Sized>(&self, value: &Q, make: impl FnOnce() -> K) -> K
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq,
-    {
-        let hash = make_hash(value);
-        let mut shard = self.lock_shard_by_hash(hash);
-
-        match table_entry(&mut shard, hash, value) {
-            Entry::Occupied(e) => e.get().0,
-            Entry::Vacant(e) => {
-                let v = make();
-                e.insert((v, ()));
-                v
-            }
-        }
-    }
-
-    #[inline]
-    pub fn intern<Q>(&self, value: Q, make: impl FnOnce(Q) -> K) -> K
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq,
-    {
-        let hash = make_hash(&value);
-        let mut shard = self.lock_shard_by_hash(hash);
-
-        match table_entry(&mut shard, hash, &value) {
-            Entry::Occupied(e) => e.get().0,
-            Entry::Vacant(e) => {
-                let v = make(value);
-                e.insert((v, ()));
-                v
-            }
-        }
-    }
-}
-
-pub trait IntoPointer {
-    /// Returns a pointer which outlives `self`.
-    fn into_pointer(&self) -> *const ();
-}
-
-impl<K: Eq + Hash + Copy + IntoPointer> ShardedHashMap<K, ()> {
-    pub fn contains_pointer_to<T: Hash + IntoPointer>(&self, value: &T) -> bool {
-        let hash = make_hash(&value);
-        let shard = self.lock_shard_by_hash(hash);
-        let value = value.into_pointer();
-        shard.find(hash, |(k, ())| k.into_pointer() == value).is_some()
-    }
-}
-
 #[inline]
 pub fn make_hash<K: Hash + ?Sized>(val: &K) -> u64 {
     let mut state = FxHasher::default();

compiler/rustc_data_structures/src/sync.rs

@@ -25,6 +25,7 @@
 use std::collections::HashMap;
 use std::hash::{BuildHasher, Hash};
 
+pub use horde::collect;
 pub use parking_lot::{
     MappedRwLockReadGuard as MappedReadGuard, MappedRwLockWriteGuard as MappedWriteGuard,
     RwLockReadGuard as ReadGuard, RwLockWriteGuard as WriteGuard,
@@ -39,13 +40,15 @@ pub use self::parallel::{
     broadcast, par_fns, par_for_each_in, par_join, par_map, parallel_guard, spawn,
     try_par_for_each_in,
 };
+pub use self::sync_table::{IntoPointer, LockedWrite, Read, SyncTable};
 pub use self::vec::{AppendOnlyIndexVec, AppendOnlyVec};
 pub use self::worker_local::{Registry, WorkerLocal};
 pub use crate::marker::*;
 
 mod freeze;
 mod lock;
 mod parallel;
+mod sync_table;
 mod vec;
 mod worker_local;
 

compiler/rustc_data_structures/src/sync/lock.rs

@@ -112,6 +112,11 @@ impl<T> Lock<T> {
         self.data.get_mut()
     }
 
+    #[inline(always)]
+    pub fn mode(&self) -> Mode {
+        self.mode
+    }
+
     #[inline(always)]
     pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
         let mode = self.mode;

compiler/rustc_data_structures/src/sync/sync_table.rs

@@ -0,0 +1,215 @@
+use std::borrow::Borrow;
+use std::hash::{BuildHasher, Hash, Hasher};
+use std::hint::cold_path;
+use std::ops::{Deref, DerefMut};
+
+use horde::collect::{Pin, pin};
+pub use horde::sync_table::Read;
+use horde::sync_table::Write;
+use rustc_hash::FxBuildHasher;
+
+use crate::sync::{DynSync, Lock, LockGuard, Mode};
+
+pub struct SyncTable<K, V> {
+    // We use this lock to protect `table` instead of the internal mutex in `horde::SyncTable`
+    // as it's faster when synchronization is disabled.
+    lock: Lock<()>,
+
+    table: horde::SyncTable<K, V, FxBuildHasher>,
+}
+
+// Memory reclamation can move elements to other threads for dropping,
+// so we require `Sync` instead of `DynSync` here
+unsafe impl<K: Sync, V: Sync> DynSync for SyncTable<K, V> where FxBuildHasher: Sync {}
+
+impl<K, V> Default for SyncTable<K, V> {
+    fn default() -> Self {
+        Self { lock: Lock::default(), table: horde::SyncTable::default() }
+    }
+}
+
+impl<K, V> SyncTable<K, V> {
+    /// Creates a [Read] handle from a pinned region.
+    ///
+    /// Use [horde::collect::pin] to get a `Pin` instance.
+    #[inline]
+    pub fn read<'a>(&'a self, pin: Pin<'a>) -> Read<'a, K, V, FxBuildHasher> {
+        self.table.read(pin)
+    }
+
+    /// Creates a [LockedWrite] handle by taking the underlying mutex that protects writes.
+    #[inline]
+    pub fn lock(&self) -> LockedWrite<'_, K, V> {
+        LockedWrite {
+            _guard: self.lock.lock(),
+            table: {
+                // SAFETY: We ensure there's only 1 writer at a time using our own lock
+                unsafe { self.table.unsafe_write() }
+            },
+        }
+    }
+
+    /// Hashes a key with the table's hasher.
+    #[inline]
+    pub fn hash_key<Q>(&self, key: &Q) -> u64
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + Hash,
+    {
+        self.table.hash_key::<Q>(key)
+    }
+
+    pub fn len(&self) -> usize {
+        pin(|pin| self.read(pin).len())
+    }
+
+    pub fn with_capacity(cap: usize) -> Self {
+        Self { lock: Lock::new(()), table: horde::SyncTable::new_with(FxBuildHasher, cap) }
+    }
+}
+
+/// A handle to a [SyncTable] with write access protected by a lock.
+pub struct LockedWrite<'a, K, V> {
+    table: Write<'a, K, V, FxBuildHasher>,
+    _guard: LockGuard<'a, ()>,
+}
+
+impl<'a, K, V> Deref for LockedWrite<'a, K, V> {
+    type Target = Write<'a, K, V, FxBuildHasher>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        &self.table
+    }
+}
+
+impl<'a, K, V> DerefMut for LockedWrite<'a, K, V> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.table
+    }
+}
+
+pub trait IntoPointer {
+    /// Returns a pointer which outlives `self`.
+    fn into_pointer(&self) -> *const ();
+}
+
+impl<K: Eq + Hash + Copy + Send> SyncTable<K, ()> {
+    pub fn contains_pointer_to<T: Hash + IntoPointer>(&self, value: &T) -> bool
+    where
+        K: IntoPointer,
+    {
+        pin(|pin| {
+            let mut state = FxBuildHasher.build_hasher();
+            value.hash(&mut state);
+            let hash = state.finish();
+            let value = value.into_pointer();
+            self.read(pin).get_from_hash(hash, |entry| entry.into_pointer() == value).is_some()
+        })
+    }
+
+    #[inline]
+    pub fn intern_ref<Q: ?Sized>(&self, value: &Q, make: impl FnOnce() -> K) -> K
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq,
+    {
+        if self.lock.mode() == Mode::Sync {
+            pin(|pin| {
+                let hash = self.hash_key(value);
+
+                let potential = match self.read(pin).get_potential(&value, Some(hash)) {
+                    Ok(entry) => return *entry.0,
+                    Err(potential) => {
+                        cold_path();
+                        potential
+                    }
+                };
+
+                let mut write = self.lock();
+
+                let potential = match potential.refresh(self.read(pin), &value, Some(hash)) {
+                    Ok(entry) => {
+                        cold_path();
+                        return *entry.0;
+                    }
+                    Err(potential) => potential,
+                };
+
+                let result = make();
+
+                potential.insert_new(&mut write, result, (), Some(hash));
+
+                result
+            })
+        } else {
+            let mut write = self.lock();
+
+            let hash = self.hash_key(&value);
+
+            let entry = write.read().get(&value, Some(hash));
+            if let Some(entry) = entry {
+                return *entry.0;
+            }
+
+            let result = make();
+
+            write.insert_new(result, (), Some(hash));
+
+            result
+        }
+    }
+
+    #[inline]
+    pub fn intern<Q>(&self, value: Q, make: impl FnOnce(Q) -> K) -> K
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq,
+    {
+        if self.lock.mode() == Mode::Sync {
+            pin(|pin| {
+                let hash = self.hash_key(&value);
+
+                let potential = match self.read(pin).get_potential(&value, Some(hash)) {
+                    Ok(entry) => return *entry.0,
+                    Err(potential) => {
+                        cold_path();
+                        potential
+                    }
+                };
+
+                let mut write = self.lock();
+
+                let potential = match potential.refresh(self.read(pin), &value, Some(hash)) {
+                    Ok(entry) => {
+                        cold_path();
+                        return *entry.0;
+                    }
+                    Err(potential) => potential,
+                };
+
+                let result = make(value);
+
+                potential.insert_new(&mut write, result, (), Some(hash));
+
+                result
+            })
+        } else {
+            let mut write = self.lock();
+
+            let hash = self.hash_key(&value);
+
+            let entry = write.read().get(&value, Some(hash));
+            if let Some(entry) = entry {
+                return *entry.0;
+            }
+
+            let result = make(value);
+
+            write.insert_new(result, (), Some(hash));
+
+            result
+        }
+    }
+}

compiler/rustc_interface/src/util.rs

@@ -14,7 +14,7 @@ use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::jobserver::Proxy;
-use rustc_data_structures::sync;
+use rustc_data_structures::sync::{self, collect};
 use rustc_metadata::{DylibError, EncodedMetadata, load_symbol_from_dylib};
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::{CurrentGcx, TyCtxt};
@@ -216,7 +216,10 @@ pub(crate) fn run_in_thread_pool_with_globals<
     let builder = rustc_thread_pool::ThreadPoolBuilder::new()
         .thread_name(|_| "rustc".to_string())
         .acquire_thread_handler(move || proxy_.acquire_thread())
-        .release_thread_handler(move || proxy__.release_thread())
+        .release_thread_handler(move || {
+            collect::release();
+            proxy__.release_thread()
+        })
         .num_threads(threads)
         .deadlock_handler(move || {
             // On deadlock, creates a new thread and forwards information in thread