Merge branch 'scale-amd64' of git+ssh://amsterdam.csail.mit.edu/home/am0/6.828/xv6 into scale-amd64

d959c79e · Silas Boyd-Wickizer · 4eef1217 · 530e5e3f · d959c79e · d959c79e
--- a/include/radix.hh
+++ b/include/radix.hh
@@ -57,6 +57,10 @@ struct radix {
  }
  radix_elem* search(u64 key);
  radix_range search_lock(u64 start, u64 size);
+
+  // k is shifted value.
+  u64 skip_empty(u64 k) const;
+
  NEW_DELETE_OPS(radix)
 };

@@ -64,8 +68,8 @@ struct radix_iterator {
  const radix* r_;
  u64 k_;

-  radix_iterator(const radix* r, u64 k) : r_(r), k_(k) {}
-  radix_iterator &operator++() { k_++; return *this; }
+  radix_iterator(const radix* r, u64 k) : r_(r), k_(r->skip_empty(k)) {}
+  radix_iterator &operator++() { k_++; k_ = r_->skip_empty(k_); return *this; }
  radix_elem* operator*();
  bool operator==(const radix_iterator &other) {
    return r_ == other.r_ && k_ == other.k_; }

--- a/kernel/radix.cc
+++ b/kernel/radix.cc
@@ -5,32 +5,39 @@
 #include "cpputil.hh"
 #include "radix.hh"

+// Returns the level we stopped at.
 template<class CB>
-void
-descend(u64 key, markptr<void> *n, u32 level, CB cb)
+u32
+descend(u64 key, markptr<void> *n, u32 level, CB cb, bool create)
 {
-  // for now, we only support exact multiples of bits_per_level
-  assert(key_bits == bits_per_level * radix_levels);
+  static_assert(key_bits == bits_per_level * radix_levels,
+                "for now, we only support exact multiples of bits_per_level");
  assert(n);

  void *v = n->ptr();
-  if (v == 0) {
+  if (v == 0 && create) {
    radix_node *new_rn = new radix_node();
    if (n->ptr().cmpxch_update(&v, (void*) new_rn))
      v = new_rn;
    else
      delete new_rn;
  }
+  // Node isn't there. Just return.
+  if (v == 0) {
+    return level+1;
+  }

  radix_node *rn = (radix_node*) v;

  u64 idx = key >> (bits_per_level * level);
  idx &= (1<<bits_per_level)-1;
  markptr<void> *vptr = &rn->ptr[idx];
-  if (level == 0)
+  if (level == 0) {
    cb(vptr);
-  else
-    descend(key, vptr, level-1, cb);
+    return level;
+  } else {
+    return descend(key, vptr, level-1, cb, create);
+  }
 }

 radix_elem*
@@ -39,7 +46,7 @@ radix::search(u64 key)
  radix_elem *result = 0;
  descend(key >> shift_, &root_, radix_levels-1, [&result](markptr<void> *v) {
      result = (radix_elem*) v->ptr().load();
-    });
+    }, false);
  return result;
 }

@@ -49,14 +56,37 @@ radix::search_lock(u64 start, u64 size)
  return radix_range(this, start >> shift_, size >> shift_);
 }

+u64
+radix::skip_empty(u64 k) const
+{
+  u64 next_k = k;
+  while (next_k < (1UL<<key_bits)) {
+    // Does next_k exist?
+    // FIXME: evil evil const_cast
+    u32 level = descend(next_k, const_cast<markptr<void>*>(&root_),
+                        radix_levels-1, [](markptr<void> *v){}, false);
+    if (level == 0) {
+      return next_k;
+    }
+    u64 mask = 1UL<<(bits_per_level * level);
+    // Skip past everything we know is missing.
+    next_k = (next_k & ~(mask-1)) + mask;
+  }
+  // Nope, no successor.
+  return ~0ULL;
+}
+
 radix_range::radix_range(radix *r, u64 start, u64 size)
  : r_(r), start_(start), size_(size)
 {
-  for (u64 k = start_; k != start_ + size_; k++)
-    descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
-        while (!v->mark().xchg(true))
-          ; // spin
-      });
+  for (u64 k = start_; k != start_ + size_; k++) {
+    if (descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
+          while (!v->mark().xchg(true))
+            ; // spin
+        }, true) != 0) {
+      panic("radix_range");
+    }
+  }
 }

 radix_range::~radix_range()
@@ -64,10 +94,13 @@ radix_range::~radix_range()
  if (!r_)
    return;

-  for (u64 k = start_; k != start_ + size_; k++)
-    descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
-        v->mark() = false;
-      });
+  for (u64 k = start_; k != start_ + size_; k++) {
+    if (descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
+          v->mark() = false;
+        }, true) != 0) {
+      panic("~radix_range");
+    }
+  }
 }

 void
@@ -79,15 +112,19 @@ radix_range::replace(u64 start, u64 size, radix_elem *val)
  assert(start >= start_);
  assert(start + size <= start_ + size_);

-  for (u64 k = start; k != start + size; k++)
-    descend(k, &r_->root_, radix_levels-1, [val](markptr<void> *v) {
-        void* cur = v->ptr().load();
-        while (!v->ptr().cmpxch_update(&cur, val))
-          ; // spin
-        val->incref();
-        if (cur)
-          ((radix_elem*) cur)->decref();
-      });
+  for (u64 k = start; k != start + size; k++) {
+    if (descend(k, &r_->root_, radix_levels-1, [val](markptr<void> *v) {
+          void* cur = v->ptr().load();
+          while (!v->ptr().cmpxch_update(&cur, val))
+            ; // spin
+          if (val)
+            val->incref();
+          if (cur)
+            ((radix_elem*) cur)->decref();
+        }, true)) {
+      panic("radix_range::replace");
+    }
+  }
 }

 radix_elem*
@@ -96,6 +133,6 @@ radix_iterator::operator*()
  radix_elem *result = 0;
  descend(k_, (markptr<void>*) &r_->root_, radix_levels-1, [&result](markptr<void> *v) {
      result = (radix_elem*) v->ptr().load();
-    });
+    }, false);
  return result;
 }
--- a/kernel/vm.cc
+++ b/kernel/vm.cc
@@ -640,7 +640,16 @@ vmap::sbrk(ssize_t n, uptr *addr)
 #if VM_RADIX
  auto span = rx.search_lock(newstart, newn + PGSIZE);
 #endif
+#if VM_CRANGE
+  for (auto r: span) {
+#endif
+#if VM_RADIX
+  void *last = 0;
  for (auto r: span) {
+    if (!r || r == last)
+      continue;
+    last = r;
+#endif
    vma *e = (vma*) r;

    if (e->vma_start <= newstart) {