Merge branch 'scale-amd64' of ssh://amsterdam.csail.mit.edu/home/am0/6.828/xv6 into scale-amd64

eb689cec · Frans Kaashoek · 119872d2 · ef131a54 · eb689cec · eb689cec
--- a/include/crange.hh
+++ b/include/crange.hh
 #pragma once

+#include "markptr.hh"
+
 struct crange;
 struct crange_locked;
 struct range;
 class range_iterator;

-template<class T>
-class markptr_ptr;
-
-template<class T>
-class markptr_mark;
-
-template<class T>
-class markptr {
- protected:
-  std::atomic<uptr> _p;
-
- public:
-  markptr() : _p(0) {}
-  markptr(T* v) : _p((uptr) v) {}
-  markptr(const markptr<T> &v) : _p(v._p.load()) {}
-
-  void operator=(T* v) { _p = (uptr) v; }
-  void operator=(const markptr<T> &v) { _p = v._p.load(); }
-
-  bool operator!=(const markptr<T> &v) const { return _p != v._p; }
-  bool operator==(const markptr<T> &v) const { return _p == v._p; }
-
-  markptr_ptr<T>& ptr() {
-    return *(markptr_ptr<T>*) this;
-  }
-
-  markptr_mark<T>& mark() {
-    return *(markptr_mark<T>*) this;
-  }
-
-  // Convenience operator to avoid having to write out xx.ptr()->...
-  T* operator->() { return ptr(); }
-
-  bool cmpxch(markptr<T> expected, markptr<T> desired) {
-    uptr ee = expected._p.load();
-    return _p.compare_exchange_weak(ee, desired._p.load());
-  }
-};
-
-template<class T>
-class markptr_ptr : private markptr<T> {
- public:
-  void operator=(T *p) {
-    uptr p0, p1;
-    do {
-      p0 = markptr<T>::_p.load();
-      p1 = (p0 & 1) | (uptr) p;
-    } while (!markptr<T>::_p.compare_exchange_weak(p0, p1));
-  }
-
-  T* load() const {
-    return (T*) (markptr<T>::_p.load() & ~1);
-  }
-
-  operator T*() const { return load(); }
-};
-
-template<class T>
-class markptr_mark : public markptr<T> {
- public:
-  void operator=(bool m) {
-    uptr p0, p1;
-    do {
-      p0 = markptr<T>::_p.load();
-      p1 = (p0 & ~1) | !!m;
-    } while (!markptr<T>::_p.compare_exchange_weak(p0, p1));
-  }
-
-  bool load() const {
-    return markptr<T>::_p.load() & 1;
-  }
-
-  operator bool() const { return load(); }
-};
-
 struct range : public rcu_freed {
 private:
  const u64 key;

--- a/include/markptr.hh
+++ b/include/markptr.hh
+#pragma once
+
+template<class T>
+class markptr_ptr;
+
+template<class T>
+class markptr_mark;
+
+template<class T>
+class markptr {
+ protected:
+  std::atomic<uptr> _p;
+
+ public:
+  markptr() : _p(0) {}
+  markptr(T* v) : _p((uptr) v) {}
+  markptr(const markptr<T> &v) : _p(v._p.load()) {}
+
+  void operator=(T* v) { _p = (uptr) v; }
+  void operator=(const markptr<T> &v) { _p = v._p.load(); }
+
+  bool operator!=(const markptr<T> &v) const { return _p != v._p; }
+  bool operator==(const markptr<T> &v) const { return _p == v._p; }
+
+  markptr_ptr<T>& ptr() {
+    return *(markptr_ptr<T>*) this;
+  }
+
+  markptr_mark<T>& mark() {
+    return *(markptr_mark<T>*) this;
+  }
+
+  // Convenience operator to avoid having to write out xx.ptr()->...
+  T* operator->() { return ptr(); }
+
+  bool cmpxch(markptr<T> expected, markptr<T> desired) {
+    uptr ee = expected._p.load();
+    return _p.compare_exchange_weak(ee, desired._p.load());
+  }
+};
+
+template<class T>
+class markptr_ptr : private markptr<T> {
+ public:
+  void operator=(T* p) {
+    uptr p0, p1;
+    do {
+      p0 = markptr<T>::_p.load();
+      p1 = (p0 & 1) | (uptr) p;
+    } while (!markptr<T>::_p.compare_exchange_weak(p0, p1));
+  }
+
+  bool cmpxch_update(T** expected, T* desired) {
+    uptr p0, p1;
+    do {
+      p0 = markptr<T>::_p.load();
+      p1 = (p0 & 1) | (uptr) desired;
+
+      T* cur = (T*) (p0 & ~1);
+      if (cur != *expected) {
+        *expected = cur;
+        return false;
+      }
+    } while (!markptr<T>::_p.compare_exchange_weak(p0, p1));
+
+    return true;
+  }
+
+  T* load() const {
+    return (T*) (markptr<T>::_p.load() & ~1);
+  }
+
+  operator T*() const { return load(); }
+};
+
+template<class T>
+class markptr_mark : public markptr<T> {
+ public:
+  void operator=(bool m) {
+    xchg(m);
+  }
+
+  bool xchg(bool m) {
+    uptr p0, p1;
+    do {
+      p0 = markptr<T>::_p.load();
+      p1 = (p0 & ~1) | !!m;
+    } while (!markptr<T>::_p.compare_exchange_weak(p0, p1));
+    return p0 & 1;
+  }
+
+  bool load() const {
+    return markptr<T>::_p.load() & 1;
+  }
+
+  operator bool() const { return load(); }
+};
--- a/include/radix.hh
+++ b/include/radix.hh
+#pragma once
+
+/*
+ * A page-table-like structure for mapping fixed-length keys to void* ptrs.
+ */
+
+#include "markptr.hh"
+
+enum { bits_per_level = 9 };
+enum { key_bits = 36 };
+enum { radix_levels = (key_bits + bits_per_level - 1) / bits_per_level };
+
+class radix_elem : public rcu_freed {
+ private:
+  bool deleted_;
+  std::atomic<u64> ref_;
+
+ public:
+  radix_elem() : rcu_freed("radix_elem"), deleted_(false), ref_(0) {}
+  bool deleted() { return deleted_; }
+  void decref() { if (--ref_ == 0) { deleted_ = true; gc_delayed(this); } }
+  void incref() { ref_++; }
+};
+
+struct radix_node {
+  markptr<void> ptr[1 << bits_per_level];
+  radix_node() {
+    for (int i = 0; i < sizeof(ptr) / sizeof(ptr[0]); i++)
+      ptr[i] = 0;
+  }
+  NEW_DELETE_OPS(radix_node)
+};
+
+struct radix;
+
+struct radix_range {
+  radix* r_;
+  u64 start_;
+  u64 size_;
+
+  radix_range(radix* r, u64 start, u64 size);
+  radix_range(radix_range&&);
+  ~radix_range();
+
+  void replace(u64 start, u64 size, radix_elem* val);
+
+  radix_range(const radix_range&) = delete;
+  void operator=(const radix_range&) = delete;
+};
+
+struct radix {
+  markptr<void> root_;
+  u32 shift_;
+
+  radix(u32 shift) : root_(0), shift_(shift) {
+    root_.ptr() = new radix_node();
+  }
+  radix_elem* search(u64 key);
+  radix_range search_lock(u64 start, u64 size);
+  NEW_DELETE_OPS(radix)
+};
+
+struct radix_iterator {
+  const radix* r_;
+  u64 k_;
+
+  radix_iterator(const radix* r, u64 k) : r_(r), k_(k) {}
+  radix_iterator &operator++() { k_++; return *this; }
+  radix_elem* operator*();
+  bool operator==(const radix_iterator &other) {
+    return r_ == other.r_ && k_ == other.k_; }
+  bool operator!=(const radix_iterator &other) {
+    return r_ != other.r_ || k_ != other.k_; }
+};
+
+static inline radix_iterator
+begin(const radix &r) { return radix_iterator(&r, 0); }
+
+static inline radix_iterator
+end(const radix &r) { return radix_iterator(&r, ~0ULL); }
+// What we really need is one-past-the-last...
+
+static inline radix_iterator
+begin(const radix_range &rr) { return radix_iterator(rr.r_, rr.start_); }
+
+static inline radix_iterator
+end(const radix_range &rr) { return radix_iterator(rr.r_, rr.start_ + rr.size_); }
+
--- a/include/vm.hh
+++ b/include/vm.hh
@@ -2,9 +2,13 @@
 #include "atomic.hh"
 #include "crange_arch.hh"
 #include "crange.hh"
+#include "radix.hh"
 #include "cpputil.hh"
 #include "hwvm.hh"

+#define VM_CRANGE 1
+#define VM_RADIX  0
+
 using std::atomic;

 // A memory object (physical pages or inode).
@@ -34,7 +38,14 @@ struct vmnode {
 // a specific memory object.
 enum vmatype { PRIVATE, COW };

-struct vma : public range {
+struct vma
+#if VM_CRANGE
+  : public range
+#endif
+#if VM_RADIX
+  : public radix_elem
+#endif
+{
  const uptr vma_start;        // start of mapping
  const uptr vma_end;          // one past the last byte
  const enum vmatype va_type;
@@ -50,7 +61,14 @@ struct vma : public range {
 // An address space: a set of vmas plus h/w page table.
 // The elements of e[] are not ordered by address.
 struct vmap {
+#if VM_CRANGE
  struct crange cr;
+#endif
+
+#if VM_RADIX
+  struct radix rx;
+#endif
+
  atomic<u64> ref;
  pgmap *const pml4;           // Page table
  char *const kshared;

--- a/kernel/Makefrag
+++ b/kernel/Makefrag
@@ -30,6 +30,7 @@ OBJS = \
 	pipe.o \
 	proc.o \
 	gc.o \
+        radix.o \
 	rnd.o \
 	sampler.o \
 	sched.o \

--- a/kernel/kmalloc.cc
+++ b/kernel/kmalloc.cc
@@ -68,6 +68,7 @@ bucket(u64 nbytes)
    11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
    12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
    12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
+    12,
  };

  assert(nbytes <= PGSIZE);

--- a/kernel/proc.cc
+++ b/kernel/proc.cc
@@ -298,9 +298,14 @@ growproc(int n)
  // sbrk() would start to use the next region (e.g. the stack).
  uptr newstart = PGROUNDUP(curbrk);
  s64 newn = PGROUNDUP(n + curbrk - newstart);
+#if VM_CRANGE
  range *prev = 0;
  auto span = m->cr.search_lock(newstart, newn + PGSIZE);
-  for (range *r: span) {
+#endif
+#if VM_RADIX
+  auto span = m->rx.search_lock(newstart, newn + PGSIZE);
+#endif
+  for (auto r: span) {
    vma *e = (vma*) r;

    if (e->vma_start <= newstart) {
@@ -311,7 +316,9 @@ growproc(int n)

      newn -= e->vma_end - newstart;
      newstart = e->vma_end;
+#if VM_CRANGE
      prev = e;
+#endif
    } else {
      cprintf("growproc: overlap with existing mapping; brk %lx n %d\n",
              curbrk, n);
@@ -332,7 +339,13 @@ growproc(int n)
    return -1;
  }

+#if VM_CRANGE
  span.replace(prev, repl);
+#endif
+
+#if VM_RADIX
+  span.replace(newstart, newn, repl);
+#endif

  myproc()->brk += n;
  return 0;

--- a/kernel/radix.cc
+++ b/kernel/radix.cc
+#include "types.h"
+#include "atomic.hh"
+#include "spinlock.h"
+#include "kernel.hh"
+#include "cpputil.hh"
+#include "radix.hh"
+
+template<class CB>
+void
+descend(u64 key, markptr<void> *n, u32 level, CB cb)
+{
+  // for now, we only support exact multiples of bits_per_level
+  assert(key_bits == bits_per_level * radix_levels);
+  assert(n);
+
+  void *v = n->ptr();
+  if (v == 0) {
+    radix_node *new_rn = new radix_node();
+    if (n->ptr().cmpxch_update(&v, (void*) new_rn))
+      v = new_rn;
+    else
+      delete new_rn;
+  }
+
+  radix_node *rn = (radix_node*) v;
+
+  u64 idx = key >> (bits_per_level * level);
+  idx &= (1<<bits_per_level)-1;
+  markptr<void> *vptr = &rn->ptr[idx];
+  if (level == 0)
+    cb(vptr);
+  else
+    descend(key, vptr, level-1, cb);
+}
+
+radix_elem*
+radix::search(u64 key)
+{
+  radix_elem *result = 0;
+  descend(key >> shift_, &root_, radix_levels-1, [&result](markptr<void> *v) {
+      result = (radix_elem*) v->ptr().load();
+    });
+  return result;
+}
+
+radix_range
+radix::search_lock(u64 start, u64 size)
+{
+  return radix_range(this, start >> shift_, size >> shift_);
+}
+
+radix_range::radix_range(radix *r, u64 start, u64 size)
+  : r_(r), start_(start), size_(size)
+{
+  for (u64 k = start_; k != start_ + size_; k++)
+    descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
+        while (!v->mark().xchg(true))
+          ; // spin
+      });
+}
+
+radix_range::~radix_range()
+{
+  if (!r_)
+    return;
+
+  for (u64 k = start_; k != start_ + size_; k++)
+    descend(k, &r_->root_, radix_levels-1, [](markptr<void> *v) {
+        v->mark() = false;
+      });
+}
+
+void
+radix_range::replace(u64 start, u64 size, radix_elem *val)
+{
+  start = start >> r_->shift_;
+  size = size >> r_->shift_;
+
+  assert(start >= start_);
+  assert(start + size <= start_ + size_);
+
+  for (u64 k = start; k != start + size; k++)
+    descend(k, &r_->root_, radix_levels-1, [val](markptr<void> *v) {
+        void* cur = v->ptr().load();
+        while (!v->ptr().cmpxch_update(&cur, val))
+          ; // spin
+        val->incref();
+        if (cur)
+          ((radix_elem*) cur)->decref();
+      });
+}
+
+radix_elem*
+radix_iterator::operator*()
+{
+  radix_elem *result = 0;
+  descend(k_, (markptr<void>*) &r_->root_, radix_levels-1, [&result](markptr<void> *v) {
+      result = (radix_elem*) v->ptr().load();
+    });
+  return result;
+}
--- a/kernel/vm.cc
+++ b/kernel/vm.cc
@@ -118,8 +118,10 @@ vmnode::demand_load()
 * vma
 */

-vma::vma(vmap *vmap, uptr start, uptr end, enum vmatype vtype, vmnode *vmn)
-  : range(&vmap->cr, start, end-start),
+vma::vma(vmap *vmap, uptr start, uptr end, enum vmatype vtype, vmnode *vmn) :
+#if VM_CRANGE
+    range(&vmap->cr, start, end-start),
+#endif
    vma_start(start), vma_end(end), va_type(vtype), n(vmn)
 {
  if (n)
@@ -136,8 +138,14 @@ vma::~vma()
 * vmap
 */

-vmap::vmap()
-  : cr(10), ref(1), pml4(setupkvm()), kshared((char*) ksalloc(slab_kshared))
+vmap::vmap() :
+#if VM_CRANGE
+    cr(10),
+#endif
+#if VM_RADIX
+    rx(PGSHIFT),
+#endif
+    ref(1), pml4(setupkvm()), kshared((char*) ksalloc(slab_kshared))
 {
  if (pml4 == 0) {
    cprintf("vmap_alloc: setupkvm out of memory\n");
@@ -181,12 +189,24 @@ vmap::decref()
 bool
 vmap::replace_vma(vma *a, vma *b)
 {
+#if VM_CRANGE
  auto span = cr.search_lock(a->vma_start, a->vma_end - a->vma_start);
+#endif
+#if VM_RADIX
+  auto span = rx.search_lock(a->vma_start, a->vma_end - a->vma_start);
+#endif
  if (a->deleted())
    return false;
  for (auto e: span)
    assert(a == e);
+#if VM_CRANGE
  span.replace(b);
+#endif
+#if VM_RADIX
+  span.replace(a->vma_start, b->vma_start-a->vma_start, 0);
+  span.replace(b->vma_start, b->vma_end-b->vma_start, b);
+  span.replace(b->vma_end, a->vma_end-b->vma_end, 0);
+#endif
  return true;
 }

@@ -197,7 +217,16 @@ vmap::copy(int share)
  if(nm == 0)
    return 0;

-  for (range *r: cr) {
+#if VM_CRANGE
+  for (auto r: cr) {
+#endif
+#if VM_RADIX
+  void *last = 0;
+  for (auto r: rx) {
+    if (!r || r == last)
+      continue;
+    last = r;
+#endif
    vma *e = (vma *) r;

    struct vma *ne;
@@ -208,6 +237,9 @@ vmap::copy(int share)
      if (e->va_type != COW) {
        vma *repl = new vma(this, e->vma_start, e->vma_end, COW, e->n);
        replace_vma(e, repl);
+#if VM_RADIX
+        last = repl;
+#endif
        updatepages(pml4, e->vma_start, e->vma_end, [](atomic<pme_t>* p) {
            for (;;) {
              pme_t v = p->load();
@@ -231,10 +263,27 @@ vmap::copy(int share)
      goto err;
    }

+#if VM_CRANGE
    auto span = nm->cr.search_lock(ne->vma_start, ne->vma_end - ne->vma_start);
-    for (auto x __attribute__((unused)): span)
+#endif
+#if VM_RADIX
+    auto span = nm->rx.search_lock(ne->vma_start, ne->vma_end - ne->vma_start);
+#endif
+    for (auto x: span) {
+#if VM_RADIX
+      if (!x)
+        continue;
+#endif
+      cprintf("non-empty span 0x%lx--0x%lx in %p: %p\n",
+              ne->vma_start, ne->vma_end, nm, x);
      assert(0);  /* span must be empty */
+    }
+#if VM_CRANGE
    span.replace(ne);
+#endif
+#if VM_RADIX
+    span.replace(ne->vma_start, ne->vma_end-ne->vma_start, ne);
+#endif
  }

  if (share)
@@ -259,7 +308,13 @@ vmap::lookup(uptr start, uptr len)
  if (start + len < start)
    panic("vmap::lookup bad len");

-  range *r = cr.search(start, len);
+#if VM_CRANGE
+  auto r = cr.search(start, len);
+#endif
+#if VM_RADIX
+  assert(len <= PGSIZE);
+  auto r = rx.search(start);
+#endif
  if (r != 0) {
    vma *e = (vma *) r;
    if (e->vma_end <= e->vma_start)
@@ -282,10 +337,20 @@ vmap::insert(vmnode *n, uptr vma_start, int dotlb)
  {
    // new scope to release the search lock before tlbflush
    u64 len = n->npages * PGSIZE;
+#if VM_CRANGE
    auto span = cr.search_lock(vma_start, len);
+#endif
+#if VM_RADIX
+    auto span = rx.search_lock(vma_start, len);
+#endif
    for (auto r: span) {
+#if VM_RADIX
+      if (!r)
+        continue;
+#endif
      vma *rvma = (vma*) r;
-      cprintf("vmap::insert: overlap with 0x%lx--0x%lx\n", rvma->vma_start, rvma->vma_end);
+      cprintf("vmap::insert: overlap with %p: 0x%lx--0x%lx\n",
+              rvma, rvma->vma_start, rvma->vma_end);
      return -1;
    }

@@ -297,7 +362,12 @@ vmap::insert(vmnode *n, uptr vma_start, int dotlb)
      return -1;
    }

+#if VM_CRANGE
    span.replace(e);
+#endif
+#if VM_RADIX
+    span.replace(e->vma_start, e->vma_end-e->vma_start, e);
+#endif
  }

  bool needtlb = false;
@@ -327,7 +397,12 @@ vmap::remove(uptr vma_start, uptr len)
    // new scope to release the search lock before tlbflush
    uptr vma_end = vma_start + len;

+#if VM_CRANGE
    auto span = cr.search_lock(vma_start, len);
+#endif
+#if VM_RADIX
+    auto span = rx.search_lock(vma_start, len);
+#endif
    for (auto r: span) {
      vma *rvma = (vma*) r;
      if (rvma->vma_start < vma_start || rvma->vma_end > vma_end) {
@@ -338,7 +413,12 @@ vmap::remove(uptr vma_start, uptr len)

    // XXX handle partial unmap

+#if VM_CRANGE
    span.replace(0);
+#endif
+#if VM_RADIX
+    span.replace(vma_start, len, 0);
+#endif
  }

  bool needtlb = false;