avoid the compare_exchange_strong() member function

9c16a4ba · Nickolai Zeldovich · c2391d31 · 9c16a4ba · 9c16a4ba · 9c16a4ba
--- a/atomic.hh
+++ b/atomic.hh
@@ -13,3 +13,31 @@
 #define _GLIBCXX_ATOMIC_BUILTINS_8 1

 #include "atomic_std.h"
+
+template<class T>
+bool
+cmpxch(std::atomic<T> *a, T expected, T desired)
+{
+  return a->compare_exchange_weak(expected, desired);
+}
+
+template<class T>
+bool
+cmpxch(volatile std::atomic<T> *a, T expected, T desired)
+{
+  return a->compare_exchange_weak(expected, desired);
+}
+
+template<class T>
+bool
+cmpxch_update(std::atomic<T> *a, T *expected, T desired)
+{
+  return a->compare_exchange_weak(*expected, desired);
+}
+
+template<class T>
+bool
+cmpxch_update(volatile std::atomic<T> *a, T *expected, T desired)
+{
+  return a->compare_exchange_weak(*expected, desired);
+}
--- a/crange.cc
+++ b/crange.cc
@@ -392,7 +392,7 @@ void crange::add_index(int l, range *e, range *p1, range *s1)
  if (l >= e->nlevel-1) return;
  if (MARKED(e->next[l+1])) return;
  // crange_check(cr, NULL);
-  if (e->curlevel.compare_exchange_strong(l, l+1)) {
+  if (cmpxch(&e->curlevel, l, l+1)) {
    assert(e->curlevel < e->nlevel);
    // this is the core inserting at level l+1, but some core may be deleting
    struct range *s = WOMARK(s1);

--- a/fs.cc
+++ b/fs.cc
@@ -299,7 +299,7 @@ iget(u32 dev, u32 inum)
    ins->remove(mkpair(victim->dev, victim->inum), &victim);
    gc_delayed(victim);
  } else {
-    if (!icache_free[mycpu()->id].x.compare_exchange_strong(cur_free, cur_free-1))
+    if (!cmpxch(&icache_free[mycpu()->id].x, cur_free, cur_free-1))
      goto retry_evict;
  }

@@ -644,8 +644,7 @@ dir_init(struct inode *dp)
    brelse(bp, 0);
  }

-  decltype(dir) expect_null = 0;
-  if (!dp->dir.compare_exchange_strong(expect_null, dir)) {
+  if (!cmpxch(&dp->dir, (decltype(dir)) 0, dir)) {
    // XXX free all the dirents
    delete dir;
  }

--- a/gc.cc
+++ b/gc.cc
@@ -82,7 +82,7 @@ gc_move_to_tofree_cpu(int c, u64 epoch)
  assert(gc_state[c].delayed[fe].epoch == epoch-(NEPOCH-2));   // XXX race with setting epoch = 0
  // unhook list for fe epoch atomically; this shouldn't fail
  head = gc_state[c].delayed[fe].head;
-  while (!gc_state[c].delayed[fe].head.compare_exchange_strong(head, (rcu_freed*)0)) {}
+  while (!cmpxch_update(&gc_state[c].delayed[fe].head, &head, (rcu_freed*) 0)) {}

  // insert list into tofree list so that each core can free in parallel and free its elements
  if(gc_state[c].tofree[fe].epoch != gc_state[c].delayed[fe].epoch) {
@@ -91,7 +91,7 @@ gc_move_to_tofree_cpu(int c, u64 epoch)
 	    gc_state[c].delayed[fe].epoch.load());
    assert(0);
  }
-  assert(gc_state[c].tofree[fe].head.exchange(head) == 0);
+  assert(cmpxch(&gc_state[c].tofree[fe].head, (rcu_freed*) 0, head));

  // move delayed NEPOCH's adhead
  gc_state[c].delayed[fe].epoch += NEPOCH;
@@ -109,7 +109,7 @@ gc_move_to_tofree(u64 epoch)
  for (int c = 0; c < ncpu; c++) {
    gc_move_to_tofree_cpu(c, epoch);
  }
-  assert(global_epoch.compare_exchange_strong(epoch, epoch+1));
+  assert(cmpxch(&global_epoch, epoch, epoch+1));
 }

 // If all threads have seen global_epoch, we can move elements in global_epoch-2 to tofreelist
@@ -169,7 +169,7 @@ gc_delayed(rcu_freed *e)
  }
  e->_rcu_epoch = myepoch;
  e->_rcu_next = gc_state[c].delayed[myepoch % NEPOCH].head;
-  while (!gc_state[c].delayed[myepoch % NEPOCH].head.compare_exchange_strong(e->_rcu_next, e)) {}
+  while (!cmpxch_update(&gc_state[c].delayed[myepoch % NEPOCH].head, &e->_rcu_next, e)) {}
  popcli();
 }


--- a/ns.hh
+++ b/ns.hh
@@ -90,8 +90,7 @@ class xns : public rcu_freed {
      }

      e->next = root.load();
-      auto expect = e->next.load();
-      if (table[i].chain.compare_exchange_strong(expect, e))
+      if (cmpxch(&table[i].chain, e->next.load(), e))
        return 0;
    }
  }
@@ -121,21 +120,19 @@ class xns : public rcu_freed {
      auto pe = &table[i].chain;

      for (;;) {
-        auto e = *pe;
+        auto e = pe->load();
        if (!e)
          return false;

        if (e->key == key && (!vp || e->val == *vp)) {
-          int zero = 0;
-          if (!e->next_lock.compare_exchange_strong(zero, 1))
+          if (!cmpxch(&e->next_lock, 0, 1))
            break;
-          if (!pelock->compare_exchange_strong(zero, 1)) {
+          if (!cmpxch(pelock, 0, 1)) {
            e->next_lock = 0;
            break;
          }

-          auto expect = e.load(); /* XXX c_e_s replaces first arg! */
-          if (!pe->compare_exchange_strong(expect, e->next)) {
+          if (!cmpxch(pe, e, e->next.load())) {
            *pelock = 0;
            e->next_lock = 0;
            break;