Skip to content

Commit d8c559f

Browse files
bors[bot]bertptrs
andauthored
Merge #32
32: Capture backtraces for mutex dependencies r=bertptrs a=bertptrs Builds on top of #28. This PR adds backtrace data to the dependency graph, so you can figure out what series of events might have introduced the cycle in dependencies. Only the first backtrace These changes do have a performance penalty, with a worst case of 20-50% degradation over previous results. This applies to the worst case scenario where every dependency between mutexes is new and thus is unlikely to be as severe. Below is an example of what this can look like, generated with `examples/mutex_cycle.rs`. The formatting is decidedly suboptimal but backtraces cannot be formatted very well in stable rust at the moment. The exact performance hit depends on a lot of things, such as the level of backtraces captured (off, 1, or full), and how many dependencies are involved. ``` thread 'main' panicked at 'Found cycle in mutex dependency graph: 0: tracing_mutex::MutexDep::capture at ./src/lib.rs:278:23 1: core::ops::function::FnOnce::call_once at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/ops/function.rs:250:5 2: tracing_mutex::graph::DiGraph<V,E>::add_edge at ./src/graph.rs:131:50 3: tracing_mutex::MutexId::mark_held::{{closure}} at ./src/lib.rs:146:17 4: std::thread::local::LocalKey<T>::try_with at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/thread/local.rs:270:16 5: std::thread::local::LocalKey<T>::with at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/thread/local.rs:246:9 6: tracing_mutex::MutexId::mark_held at ./src/lib.rs:142:25 7: tracing_mutex::MutexId::get_borrowed at ./src/lib.rs:129:9 8: tracing_mutex::stdsync::tracing::Mutex<T>::lock at ./src/stdsync.rs:110:25 9: mutex_cycle::main at ./examples/mutex_cycle.rs:20:18 10: core::ops::function::FnOnce::call_once at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/ops/function.rs:250:5 11: std::sys_common::backtrace::__rust_begin_short_backtrace at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/sys_common/backtrace.rs:135:18 12: std::rt::lang_start::{{closure}} at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:166:18 13: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/ops/function.rs:284:13 14: std::panicking::try::do_call at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:500:40 15: std::panicking::try at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:464:19 16: std::panic::catch_unwind at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panic.rs:142:14 17: std::rt::lang_start_internal::{{closure}} at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:148:48 18: std::panicking::try::do_call at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:500:40 19: std::panicking::try at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:464:19 20: std::panic::catch_unwind at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panic.rs:142:14 21: std::rt::lang_start_internal at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:148:20 22: std::rt::lang_start at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:165:17 23: main 24: <unknown> 25: __libc_start_main 26: _start 0: tracing_mutex::MutexDep::capture at ./src/lib.rs:278:23 1: core::ops::function::FnOnce::call_once at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/ops/function.rs:250:5 2: tracing_mutex::graph::DiGraph<V,E>::add_edge at ./src/graph.rs:131:50 3: tracing_mutex::MutexId::mark_held::{{closure}} at ./src/lib.rs:146:17 4: std::thread::local::LocalKey<T>::try_with at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/thread/local.rs:270:16 5: std::thread::local::LocalKey<T>::with at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/thread/local.rs:246:9 6: tracing_mutex::MutexId::mark_held at ./src/lib.rs:142:25 7: tracing_mutex::MutexId::get_borrowed at ./src/lib.rs:129:9 8: tracing_mutex::stdsync::tracing::Mutex<T>::lock at ./src/stdsync.rs:110:25 9: mutex_cycle::main at ./examples/mutex_cycle.rs:14:18 10: core::ops::function::FnOnce::call_once at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/ops/function.rs:250:5 11: std::sys_common::backtrace::__rust_begin_short_backtrace at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/sys_common/backtrace.rs:135:18 12: std::rt::lang_start::{{closure}} at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:166:18 13: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/ops/function.rs:284:13 14: std::panicking::try::do_call at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:500:40 15: std::panicking::try at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:464:19 16: std::panic::catch_unwind at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panic.rs:142:14 17: std::rt::lang_start_internal::{{closure}} at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:148:48 18: std::panicking::try::do_call at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:500:40 19: std::panicking::try at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:464:19 20: std::panic::catch_unwind at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panic.rs:142:14 21: std::rt::lang_start_internal at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:148:20 22: std::rt::lang_start at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/rt.rs:165:17 23: main 24: <unknown> 25: __libc_start_main 26: _start ', src/lib.rs:163:13 stack backtrace: 0: rust_begin_unwind at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/std/src/panicking.rs:593:5 1: core::panicking::panic_fmt at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/panicking.rs:67:14 2: tracing_mutex::MutexId::mark_held at ./src/lib.rs:163:13 3: tracing_mutex::MutexId::get_borrowed at ./src/lib.rs:129:9 4: tracing_mutex::stdsync::tracing::Mutex<T>::lock at ./src/stdsync.rs:110:25 5: mutex_cycle::main at ./examples/mutex_cycle.rs:25:14 6: core::ops::function::FnOnce::call_once at /rustc/eb26296b556cef10fb713a38f3d16b9886080f26/library/core/src/ops/function.rs:250:5 ``` Importantly, the error shows all the dependencies that are already part of the graph, not the one that was just added, since that is already visible from the immediate panic. Co-authored-by: Bert Peters <[email protected]>
2 parents 0ae544a + a8e8af6 commit d8c559f

File tree

7 files changed

+198
-59
lines changed

7 files changed

+198
-59
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
1111
- The minimum supported Rust version is now defined as 1.70. Previously it was undefined.
1212
- Wrappers for `std::sync` primitives can now be `const` constructed.
1313
- Add support for `std::sync::OnceLock`
14+
- Added backtraces of mutex allocations to the cycle report.
1415

1516
### Breaking
1617

Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ name = "mutex"
3131
harness = false
3232

3333
[features]
34+
default = ["backtraces"]
35+
backtraces = []
3436
# Feature names do not match crate names pending namespaced features.
3537
lockapi = ["lock_api"]
3638
parkinglot = ["parking_lot", "lockapi"]

examples/mutex_cycle.rs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
//! Show what a crash looks like
2+
//!
3+
//! This shows what a traceback of a cycle detection looks like. It is expected to crash.
4+
use tracing_mutex::stdsync::Mutex;
5+
6+
fn main() {
7+
let a = Mutex::new(());
8+
let b = Mutex::new(());
9+
let c = Mutex::new(());
10+
11+
// Create an edge from a to b
12+
{
13+
let _a = a.lock();
14+
let _b = b.lock();
15+
}
16+
17+
// Create an edge from b to c
18+
{
19+
let _b = b.lock();
20+
let _c = c.lock();
21+
}
22+
23+
// Now crash by trying to add an edge from c to a
24+
let _c = c.lock();
25+
let _a = a.lock(); // This line will crash
26+
}

src/graph.rs

Lines changed: 88 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use std::cell::Cell;
2+
use std::collections::hash_map::Entry;
23
use std::collections::HashMap;
34
use std::collections::HashSet;
45
use std::hash::Hash;
@@ -19,31 +20,32 @@ type Order = usize;
1920
/// visibly changed.
2021
///
2122
/// [paper]: https://whileydave.com/publications/pk07_jea/
22-
#[derive(Default, Debug)]
23-
pub struct DiGraph<V>
23+
#[derive(Debug)]
24+
pub struct DiGraph<V, E>
2425
where
2526
V: Eq + Hash + Copy,
2627
{
27-
nodes: HashMap<V, Node<V>>,
28-
/// Next topological sort order
29-
next_ord: Order,
28+
nodes: HashMap<V, Node<V, E>>,
29+
// Instead of reordering the orders in the graph whenever a node is deleted, we maintain a list
30+
// of unused ids that can be handed out later again.
31+
unused_order: Vec<Order>,
3032
}
3133

3234
#[derive(Debug)]
33-
struct Node<V>
35+
struct Node<V, E>
3436
where
3537
V: Eq + Hash + Clone,
3638
{
3739
in_edges: HashSet<V>,
38-
out_edges: HashSet<V>,
40+
out_edges: HashMap<V, E>,
3941
// The "Ord" field is a Cell to ensure we can update it in an immutable context.
4042
// `std::collections::HashMap` doesn't let you have multiple mutable references to elements, but
4143
// this way we can use immutable references and still update `ord`. This saves quite a few
4244
// hashmap lookups in the final reorder function.
4345
ord: Cell<Order>,
4446
}
4547

46-
impl<V> DiGraph<V>
48+
impl<V, E> DiGraph<V, E>
4749
where
4850
V: Eq + Hash + Copy,
4951
{
@@ -54,12 +56,18 @@ where
5456
/// the node in the topological order.
5557
///
5658
/// New nodes are appended to the end of the topological order when added.
57-
fn add_node(&mut self, n: V) -> (&mut HashSet<V>, &mut HashSet<V>, Order) {
58-
let next_ord = &mut self.next_ord;
59+
fn add_node(&mut self, n: V) -> (&mut HashSet<V>, &mut HashMap<V, E>, Order) {
60+
// need to compute next id before the call to entry() to avoid duplicate borrow of nodes
61+
let fallback_id = self.nodes.len();
5962

6063
let node = self.nodes.entry(n).or_insert_with(|| {
61-
let order = *next_ord;
62-
*next_ord = next_ord.checked_add(1).expect("Topological order overflow");
64+
let order = if let Some(id) = self.unused_order.pop() {
65+
// Reuse discarded ordering entry
66+
id
67+
} else {
68+
// Allocate new order id
69+
fallback_id
70+
};
6371

6472
Node {
6573
ord: Cell::new(order),
@@ -77,9 +85,12 @@ where
7785
Some(Node {
7886
out_edges,
7987
in_edges,
80-
..
88+
ord,
8189
}) => {
82-
out_edges.into_iter().for_each(|m| {
90+
// Return ordering to the pool of unused ones
91+
self.unused_order.push(ord.get());
92+
93+
out_edges.into_keys().for_each(|m| {
8394
self.nodes.get_mut(&m).unwrap().in_edges.remove(&n);
8495
});
8596

@@ -96,18 +107,29 @@ where
96107
///
97108
/// Nodes, both from and to, are created as needed when creating new edges. If the new edge
98109
/// would introduce a cycle, the edge is rejected and `false` is returned.
99-
pub(crate) fn add_edge(&mut self, x: V, y: V) -> bool {
110+
///
111+
/// # Errors
112+
///
113+
/// If the edge would introduce the cycle, the underlying graph is not modified and a list of
114+
/// all the edge data in the would-be cycle is returned instead.
115+
pub(crate) fn add_edge(&mut self, x: V, y: V, e: impl FnOnce() -> E) -> Result<(), Vec<E>>
116+
where
117+
E: Clone,
118+
{
100119
if x == y {
101120
// self-edges are always considered cycles
102-
return false;
121+
return Err(Vec::new());
103122
}
104123

105124
let (_, out_edges, ub) = self.add_node(x);
106125

107-
if !out_edges.insert(y) {
108-
// Edge already exists, nothing to be done
109-
return true;
110-
}
126+
match out_edges.entry(y) {
127+
Entry::Occupied(_) => {
128+
// Edge already exists, nothing to be done
129+
return Ok(());
130+
}
131+
Entry::Vacant(entry) => entry.insert(e()),
132+
};
111133

112134
let (in_edges, _, lb) = self.add_node(y);
113135

@@ -119,7 +141,7 @@ where
119141
let mut delta_f = Vec::new();
120142
let mut delta_b = Vec::new();
121143

122-
if !self.dfs_f(&self.nodes[&y], ub, &mut visited, &mut delta_f) {
144+
if let Err(cycle) = self.dfs_f(&self.nodes[&y], ub, &mut visited, &mut delta_f) {
123145
// This edge introduces a cycle, so we want to reject it and remove it from the
124146
// graph again to keep the "does not contain cycles" invariant.
125147

@@ -129,7 +151,7 @@ where
129151
self.nodes.get_mut(&x).map(|node| node.out_edges.remove(&y));
130152

131153
// No edge was added
132-
return false;
154+
return Err(cycle);
133155
}
134156

135157
// No need to check as we should've found the cycle on the forward pass
@@ -141,44 +163,49 @@ where
141163
self.reorder(delta_f, delta_b);
142164
}
143165

144-
true
166+
Ok(())
145167
}
146168

147169
/// Forwards depth-first-search
148170
fn dfs_f<'a>(
149171
&'a self,
150-
n: &'a Node<V>,
172+
n: &'a Node<V, E>,
151173
ub: Order,
152174
visited: &mut HashSet<V>,
153-
delta_f: &mut Vec<&'a Node<V>>,
154-
) -> bool {
175+
delta_f: &mut Vec<&'a Node<V, E>>,
176+
) -> Result<(), Vec<E>>
177+
where
178+
E: Clone,
179+
{
155180
delta_f.push(n);
156181

157-
n.out_edges.iter().all(|w| {
182+
for (w, e) in &n.out_edges {
158183
let node = &self.nodes[w];
159184
let ord = node.ord.get();
160185

161186
if ord == ub {
162187
// Found a cycle
163-
false
188+
return Err(vec![e.clone()]);
164189
} else if !visited.contains(w) && ord < ub {
165190
// Need to check recursively
166191
visited.insert(*w);
167-
self.dfs_f(node, ub, visited, delta_f)
168-
} else {
169-
// Already seen this one or not interesting
170-
true
192+
if let Err(mut chain) = self.dfs_f(node, ub, visited, delta_f) {
193+
chain.push(e.clone());
194+
return Err(chain);
195+
}
171196
}
172-
})
197+
}
198+
199+
Ok(())
173200
}
174201

175202
/// Backwards depth-first-search
176203
fn dfs_b<'a>(
177204
&'a self,
178-
n: &'a Node<V>,
205+
n: &'a Node<V, E>,
179206
lb: Order,
180207
visited: &mut HashSet<V>,
181-
delta_b: &mut Vec<&'a Node<V>>,
208+
delta_b: &mut Vec<&'a Node<V, E>>,
182209
) {
183210
delta_b.push(n);
184211

@@ -192,7 +219,7 @@ where
192219
}
193220
}
194221

195-
fn reorder(&self, mut delta_f: Vec<&Node<V>>, mut delta_b: Vec<&Node<V>>) {
222+
fn reorder(&self, mut delta_f: Vec<&Node<V, E>>, mut delta_b: Vec<&Node<V, E>>) {
196223
self.sort(&mut delta_f);
197224
self.sort(&mut delta_b);
198225

@@ -213,50 +240,65 @@ where
213240
}
214241
}
215242

216-
fn sort(&self, ids: &mut [&Node<V>]) {
243+
fn sort(&self, ids: &mut [&Node<V, E>]) {
217244
// Can use unstable sort because mutex ids should not be equal
218245
ids.sort_unstable_by_key(|v| &v.ord);
219246
}
220247
}
221248

249+
// Manual `Default` impl as derive causes unnecessarily strong bounds.
250+
impl<V, E> Default for DiGraph<V, E>
251+
where
252+
V: Eq + Hash + Copy,
253+
{
254+
fn default() -> Self {
255+
Self {
256+
nodes: Default::default(),
257+
unused_order: Default::default(),
258+
}
259+
}
260+
}
261+
222262
#[cfg(test)]
223263
mod tests {
224264
use rand::seq::SliceRandom;
225265
use rand::thread_rng;
226266

227267
use super::*;
228268

269+
fn nop() {}
270+
229271
#[test]
230272
fn test_no_self_cycle() {
231273
// Regression test for https://github.com/bertptrs/tracing-mutex/issues/7
232274
let mut graph = DiGraph::default();
233275

234-
assert!(!graph.add_edge(1, 1));
276+
assert!(graph.add_edge(1, 1, nop).is_err());
235277
}
236278

237279
#[test]
238280
fn test_digraph() {
239281
let mut graph = DiGraph::default();
240282

241283
// Add some safe edges
242-
assert!(graph.add_edge(0, 1));
243-
assert!(graph.add_edge(1, 2));
244-
assert!(graph.add_edge(2, 3));
245-
assert!(graph.add_edge(4, 2));
284+
assert!(graph.add_edge(0, 1, nop).is_ok());
285+
assert!(graph.add_edge(1, 2, nop).is_ok());
286+
assert!(graph.add_edge(2, 3, nop).is_ok());
287+
assert!(graph.add_edge(4, 2, nop).is_ok());
246288

247289
// Try to add an edge that introduces a cycle
248-
assert!(!graph.add_edge(3, 1));
290+
assert!(graph.add_edge(3, 1, nop).is_err());
249291

250292
// Add an edge that should reorder 0 to be after 4
251-
assert!(graph.add_edge(4, 0));
293+
assert!(graph.add_edge(4, 0, nop).is_ok());
252294
}
253295

254296
/// Fuzz the DiGraph implementation by adding a bunch of valid edges.
255297
///
256298
/// This test generates all possible forward edges in a 100-node graph consisting of natural
257299
/// numbers, shuffles them, then adds them to the graph. This will always be a valid directed,
258300
/// acyclic graph because there is a trivial order (the natural numbers) but because the edges
259-
/// are added in a random order the DiGraph will still occassionally need to reorder nodes.
301+
/// are added in a random order the DiGraph will still occasionally need to reorder nodes.
260302
#[test]
261303
fn fuzz_digraph() {
262304
// Note: this fuzzer is quadratic in the number of nodes, so this cannot be too large or it
@@ -277,7 +319,7 @@ mod tests {
277319
let mut graph = DiGraph::default();
278320

279321
for (x, y) in edges {
280-
assert!(graph.add_edge(x, y));
322+
assert!(graph.add_edge(x, y, nop).is_ok());
281323
}
282324
}
283325
}

0 commit comments

Comments
 (0)