fix iteration bug; forgot to git add

bcbe622a · Nickolai Zeldovich · 3e6c70a4 · bcbe622a · bcbe622a · bcbe622a
--- a/include/radix.hh
+++ b/include/radix.hh
+#pragma once
+
+/*
+ * A page-table-like structure for mapping fixed-length keys to void* ptrs.
+ */
+
+#include "markptr.hh"
+
+enum { bits_per_level = 9 };
+enum { key_bits = 36 };
+enum { radix_levels = (key_bits + bits_per_level - 1) / bits_per_level };
+
+class radix_elem : public rcu_freed {
+ private:
+  bool deleted_;
+  std::atomic<u64> ref_;
+
+ public:
+  radix_elem() : rcu_freed("radix_elem"), deleted_(false), ref_(0) {}
+  bool deleted() { return deleted_; }
+  void decref() { if (--ref_ == 0) { deleted_ = true; gc_delayed(this); } }
+  void incref() { ref_++; }
+};
+
+struct radix_node {
+  markptr<void> ptr[1 << bits_per_level];
+  radix_node() {
+    for (int i = 0; i < sizeof(ptr) / sizeof(ptr[0]); i++)
+      ptr[i] = 0;
+  }
+  NEW_DELETE_OPS(radix_node)
+};
+
+struct radix;
+
+struct radix_range {
+  radix* r_;
+  u64 start_;
+  u64 size_;
+
+  radix_range(radix* r, u64 start, u64 size);
+  radix_range(radix_range&&);
+  ~radix_range();
+
+  void replace(u64 start, u64 size, radix_elem* val);
+
+  radix_range(const radix_range&) = delete;
+  void operator=(const radix_range&) = delete;
+};
+
+struct radix {
+  markptr<void> root_;
+  u32 shift_;
+
+  radix(u32 shift) : root_(0), shift_(shift) {
+    root_.ptr() = new radix_node();
+  }
+  radix_elem* search(u64 key);
+  radix_range search_lock(u64 start, u64 size);
+  NEW_DELETE_OPS(radix)
+};
+
+struct radix_iterator {
+  const radix* r_;
+  u64 k_;
+
+  radix_iterator(const radix* r, u64 k) : r_(r), k_(k) {}
+  radix_iterator &operator++() { k_++; return *this; }
+  radix_elem* operator*();
+  bool operator==(const radix_iterator &other) {
+    return r_ == other.r_ && k_ == other.k_; }
+  bool operator!=(const radix_iterator &other) {
+    return r_ != other.r_ || k_ != other.k_; }
+};
+
+static inline radix_iterator
+begin(const radix &r) { return radix_iterator(&r, 0); }
+
+static inline radix_iterator
+end(const radix &r) { return radix_iterator(&r, ~0ULL); }
+// What we really need is one-past-the-last...
+
+static inline radix_iterator
+begin(const radix_range &rr) { return radix_iterator(rr.r_, rr.start_); }
+
+static inline radix_iterator
+end(const radix_range &rr) { return radix_iterator(rr.r_, rr.start_ + rr.size_); }
+
--- a/kernel/radix.cc
+++ b/kernel/radix.cc
+#include "types.h"
+#include "atomic.hh"
+#include "spinlock.h"
+#include "kernel.hh"
+#include "cpputil.hh"
+#include "radix.hh"
+
+template<class CB>
+void
+descend(u64 key, markptr<void> *n, u32 level, CB cb)
+{
+  // for now, we only support exact multiples of bits_per_level
+  assert(key_bits == bits_per_level * radix_levels);
+  assert(n);
+
+  void *v = n->ptr();
+  if (v == 0) {
+    radix_node *new_rn = new radix_node();
+    if (n->ptr().cmpxch_update(&v, (void*) new_rn))
+      v = new_rn;
+    else
+      delete new_rn;
+  }
+
+  radix_node *rn = (radix_node*) v;
+
+  u64 idx = key >> (bits_per_level * level);
+  idx &= (1<<bits_per_level)-1;
+  markptr<void> *vptr = &rn->ptr[idx];
+  if (level == 0)
+    cb(vptr);
+  else
+    descend(key, vptr, level-1, cb);
+}
+
+radix_elem*
+radix::search(u64 key)
+{
+  radix_elem *result = 0;
+  descend(key >> shift_, &root_, radix_levels-1, [&result](markptr<void> *v) {
+      result = (radix_elem*) v->ptr().load();
+    });
+  return result;
+}
+
+radix_range
+radix::search_lock(u64 start, u64 size)
+{
+  return radix_range(this, start >> shift_, size >> shift_);
+}
+
+radix_range::radix_range(radix *r, u64 start, u64 size)
+  : r_(r), start_(start), size_(size)
+{
+  for (u64 k = start_; k != start_ + size_; k++)
+    descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
+        while (!v->mark().xchg(true))
+          ; // spin
+      });
+}
+
+radix_range::~radix_range()
+{
+  if (!r_)
+    return;
+
+  for (u64 k = start_; k != start_ + size_; k++)
+    descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
+        v->mark() = false;
+      });
+}
+
+void
+radix_range::replace(u64 start, u64 size, radix_elem *val)
+{
+  start = start >> r_->shift_;
+  size = size >> r_->shift_;
+
+  assert(start >= start_);
+  assert(start + size <= start_ + size_);
+
+  for (u64 k = start; k != start + size; k++)
+    descend(k, &r_->root_, radix_levels-1, [val](markptr<void> *v) {
+        void* cur = v->ptr().load();
+        while (!v->ptr().cmpxch_update(&cur, val))
+          ; // spin
+        val->incref();
+        if (cur)
+          ((radix_elem*) cur)->decref();
+      });
+}
+
+radix_elem*
+radix_iterator::operator*()
+{
+  radix_elem *result = 0;
+  descend(k_, (markptr<void>*) &r_->root_, radix_levels-1, [&result](markptr<void> *v) {
+      result = (radix_elem*) v->ptr().load();
+    });
+  return result;
+}
--- a/kernel/vm.cc
+++ b/kernel/vm.cc
@@ -221,9 +221,11 @@ vmap::copy(int share)
  for (auto r: cr) {
 #endif
 #if VM_RADIX
+  void *last = 0;
  for (auto r: rx) {
-    if (!r)
+    if (!r || r == last)
      continue;
+    last = r;
 #endif
    vma *e = (vma *) r;

@@ -235,6 +237,9 @@ vmap::copy(int share)
      if (e->va_type != COW) {
        vma *repl = new vma(this, e->vma_start, e->vma_end, COW, e->n);
        replace_vma(e, repl);
+#if VM_RADIX
+        last = repl;
+#endif
        updatepages(pml4, e->vma_start, e->vma_end, [](atomic<pme_t>* p) {
            for (;;) {
              pme_t v = p->load();
@@ -264,13 +269,13 @@ vmap::copy(int share)
 #if VM_RADIX
    auto span = nm->rx.search_lock(ne->vma_start, ne->vma_end - ne->vma_start);
 #endif
-    for (auto x __attribute__((unused)): span) {
+    for (auto x: span) {
 #if VM_RADIX
      if (!x)
        continue;
 #endif
-      cprintf("non-empty span: %p (orig 0x%lx--0x%lx)\n",
-              x, ne->vma_start, ne->vma_end);
+      cprintf("non-empty span 0x%lx--0x%lx in %p: %p\n",
+              ne->vma_start, ne->vma_end, &nm->rx, x);
      assert(0);  /* span must be empty */
    }
 #if VM_CRANGE