parking_lot/
elision.rs

Help
1// Copyright 2016 Amanieu d'Antras
2//
3// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5// http://opensource.org/licenses/MIT>, at your option. This file may not be
6// copied, modified, or distributed except according to those terms.
7
8#[cfg(all(feature = "hardware-lock-elision", any(target_arch = "x86", target_arch = "x86_64")))]
9use std::arch::asm;
10use std::sync::atomic::AtomicUsize;
11
12// Extension trait to add lock elision primitives to atomic types
13pub trait AtomicElisionExt {
14    type IntType;
15
16    // Perform a compare_exchange and start a transaction
17    fn elision_compare_exchange_acquire(
18        &self,
19        current: Self::IntType,
20        new: Self::IntType,
21    ) -> Result<Self::IntType, Self::IntType>;
22
23    // Perform a fetch_sub and end a transaction
24    fn elision_fetch_sub_release(&self, val: Self::IntType) -> Self::IntType;
25}
26
27// Indicates whether the target architecture supports lock elision
28#[inline]
29pub fn have_elision() -> bool {
30    cfg!(all(
31        feature = "hardware-lock-elision",
32        any(target_arch = "x86", target_arch = "x86_64"),
33    ))
34}
35
36// This implementation is never actually called because it is guarded by
37// have_elision().
38#[cfg(not(all(feature = "hardware-lock-elision", any(target_arch = "x86", target_arch = "x86_64"))))]
39impl AtomicElisionExt for AtomicUsize {
40    type IntType = usize;
41
42    #[inline]
43    fn elision_compare_exchange_acquire(&self, _: usize, _: usize) -> Result<usize, usize> {
44        unreachable!();
45    }
46
47    #[inline]
48    fn elision_fetch_sub_release(&self, _: usize) -> usize {
49        unreachable!();
50    }
51}
52
53#[cfg(all(feature = "hardware-lock-elision", any(target_arch = "x86", target_arch = "x86_64")))]
54impl AtomicElisionExt for AtomicUsize {
55    type IntType = usize;
56
57    #[inline]
58    fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> {
59        unsafe {
60            use core::arch::asm;
61            let prev: usize;
62            #[cfg(target_pointer_width = "32")]
63            asm!(
64                "xacquire",
65                "lock",
66                "cmpxchg [{:e}], {:e}",
67                in(reg) self,
68                in(reg) new,
69                inout("eax") current => prev,
70            );
71            #[cfg(target_pointer_width = "64")]
72            asm!(
73                "xacquire",
74                "lock",
75                "cmpxchg [{}], {}",
76                in(reg) self,
77                in(reg) new,
78                inout("rax") current => prev,
79            );
80            if prev == current {
81                Ok(prev)
82            } else {
83                Err(prev)
84            }
85        }
86    }
87
88    #[inline]
89    fn elision_fetch_sub_release(&self, val: usize) -> usize {
90        unsafe {
91            use core::arch::asm;
92            let prev: usize;
93            #[cfg(target_pointer_width = "32")]
94            asm!(
95                "xrelease",
96                "lock",
97                "xadd [{:e}], {:e}",
98                in(reg) self,
99                inout(reg) val.wrapping_neg() => prev,
100            );
101            #[cfg(target_pointer_width = "64")]
102            asm!(
103                "xrelease",
104                "lock",
105                "xadd [{}], {}",
106                in(reg) self,
107                inout(reg) val.wrapping_neg() => prev,
108            );
109            prev
110        }
111    }
112}
parking_lot/elision.rs

parking_lot/
elision.rs
