use versioned pointers to avoid aba race in kalloc

883a02e7 · Nickolai Zeldovich · c71c5366 · 883a02e7 · 883a02e7 · 883a02e7
--- a/include/kalloc.hh
+++ b/include/kalloc.hh
 #include "atomic.hh"

+template<class T>
+struct vptr {
+  u128 _a;
+  T ptr() const { return (T) (_a & 0xffffffffffffffffULL); }
+  u64 v() const { return _a >> 64; }
+
+  vptr(T p, u64 v) : _a((((u128)v)<<64) | (u64) p) {}
+  vptr(u128 a) : _a(a) {}
+};
+
+template<class T>
+class versioned {
+ private:
+  std::atomic<u128> _a;
+
+ public:
+  vptr<T> load() { return vptr<T>(_a.load()); }
+  bool compare_exchange(const vptr<T> &expected, T desired) {
+    vptr<T> n(desired, expected.v());
+    return cmpxch(&_a, expected._a, n._a);
+  }
+};
+
 struct run {
  struct run *next;
 };
@@ -8,7 +31,7 @@ struct kmem {
  char name[MAXNAME];
  u64 size;
  u64 ninit;
-  std::atomic<run*> freelist;
+  versioned<run*> freelist;
  std::atomic<u64> nfree;
 } __mpalign__;


--- a/include/types.h
+++ b/include/types.h
@@ -6,8 +6,10 @@ typedef unsigned int    u32;
 typedef int             s32;
 typedef unsigned long   u64;
 typedef long            s64;
+#ifdef XV6
 typedef unsigned __int128 u128;
 typedef __int128        s128;
+#endif
 typedef u64             uptr;
 typedef uptr            paddr;


--- a/kernel/kalloc.cc
+++ b/kernel/kalloc.cc
@@ -128,9 +128,13 @@ kfree_pool(struct kmem *m, char *v)
    memset(v, 1, m->size);

  r = (struct run*)v;
-  r->next = m->freelist;
-  while (!cmpxch_update(&m->freelist, &r->next, r))
-    ; /* spin */
+  for (;;) {
+    auto headval = m->freelist.load();
+    r->next = headval.ptr();
+    if (m->freelist.compare_exchange(headval, r))
+      break;
+  }
+
  m->nfree++;
  if (kinited)
    mtunlabel(mtrace_label_block, r);
@@ -169,13 +173,19 @@ kalloc_pool(struct kmem *km)
    int cn = (i + startcpu) % NCPU;
    m = &km[cn];

-    r = m->freelist;
-    run *nxt = r->next;
-    while (r && !cmpxch_update(&m->freelist, &r, nxt))
-      ; /* spin */
-
-    if (r && r->next != nxt)
-      panic("kalloc_pool: aba race %p %p %p\n", r, r->next, nxt);
+    for (;;) {
+      auto headval = m->freelist.load();
+      r = headval.ptr();
+      if (!r)
+        break;
+
+      run *nxt = r->next;
+      if (m->freelist.compare_exchange(headval, nxt)) {
+        if (r->next != nxt)
+          panic("kalloc_pool: aba race %p %p %p\n", r, r->next, nxt);
+        break;
+      }
+    }

    if (r) {
      m->nfree--;