Merge branch 'scale-amd64' of git+ssh://pdos.csail.mit.edu/home/am0/6.828/xv6 into scale-amd64

530e5e3f · David Benjamin · cd125819 · 6d02373b · 530e5e3f · 530e5e3f
--- a/.dir-locals.el
+++ b/.dir-locals.el
+((c-mode
+  (indent-tabs-mode . nil)
+  (c-file-style . "bsd")
+  (c-basic-offset . 2))
+ (c++-mode
+  (indent-tabs-mode . nil)
+  (c-file-style . "bsd")
+  (c-basic-offset . 2))
+ )
--- a/Makefile
+++ b/Makefile
@@ -89,7 +89,7 @@ gdb: $(KERN)
 ## mtrace
 ##
 mscan.syms: $(KERN)
-	$(NM) -S $< > $@
+	$(NM) -C -S $< > $@
 mscan.kern: $(KERN)
 	cp $< $@

--- a/README
+++ b/README
@@ -3,8 +3,12 @@
  You need to build and install mtrace:
    git+ssh://amsterdam.csail.mit.edu/home/am6/mpdev/qemu.git -b mtrace
-  #define MTRACE 1 in param.h
+  #define MTRACE 1 in param.h  (for qemu!)
+  If mtrace isn't cloned next to the xv6-scale repository, then set
+  QEMUSRC in config.mk to the directory containing mtrace-magic.h.
+  Set MTRACE in config.mk to the mtrace QEMU binary's path.
  $ make mscan.out
+  or make mtrace.out to generate just the trace file and not the summary.
 * Networking with lwIP
@@ -57,3 +61,11 @@
  $ apt-get install libjemalloc-dev
  $ make HW=user
  $ ./o.user/utest
+* abstract sharing
+  Obtain and configure mtrace as described above.
+  Disable DEBUG and enable MTRACE in param.h.
+  $ make QEMUSMP=8 mtrace.out
+  Run asharing in xv6 to generate abstract sharing traces
+  $ mscan --abstract-scopes --unexpected-sharing
--- a/bin/Makefrag
+++ b/bin/Makefrag
@@ -13,6 +13,8 @@ UPROGS= \
 	ls \
 	mapbench \
 	maptest \
+	mkdir \
+	mktree \
 	sh \
 	halt \
 	time \
@@ -26,10 +28,11 @@ UPROGS= \
 	wqsh \
 	cp \
 	perf \
+	asharing \
 	xls \
-	xdu
+	xdu \
-# pdu
+	wqtest \
-# pls
+	rm
 ifeq ($(HAVE_LWIP),y)
 UPROGS += \

--- a/bin/asharing.cc
+++ b/bin/asharing.cc
+// Tests to drive abstract sharing analysis
+#include "types.h"
+#include "user.h"
+#include "fcntl.h"
+#include "mtrace.h"
+#include "pthread.h"
+static int cpu;
+static pthread_barrier_t bar;
+enum { ncore = 8 };
+void
+next()
+{
+  if (setaffinity(cpu) < 0) {
+    cpu = 0;
+    if (setaffinity(cpu) < 0)
+      die("sys_setaffinity(%d) failed", cpu);
+  }
+  cpu++;
+}
+void*
+vmsharing(void* arg)
+{
+  u64 i = (u64) arg;
+  volatile char *p = (char*)(0x40000UL + i * 4096);
+  if (map((void *) p, 4096) < 0)
+    die("map failed");
+  if (unmap((void *) p, 4096) < 0)
+    die("unmap failed");
+  return 0;
+}
+void*
+fssharing(void* arg)
+{
+  u64 i = (u64) arg;
+  // Note that we keep these files open; otherwise all of these
+  // operations will share the abstract FD object and we won't get any
+  // results.
+  char filename[32];
+  snprintf(filename, sizeof(filename), "f%d", i);
+  open(filename, O_CREATE|O_RDWR);
+  pthread_barrier_wait(&bar);
+  for (u64 j = 0; j < ncore; j++) {
+    snprintf(filename, sizeof(filename), "f%d", j);
+    open(filename, O_RDWR);
+  }
+  return 0;
+}
+int
+main(int ac, char **av)
+{
+  void* (*op)(void*) = 0;
+  if (ac == 2 && strcmp(av[1], "vm") == 0)
+    op = vmsharing;
+  else if (ac == 2 && strcmp(av[1], "fs") == 0)
+    op = fssharing;
+  else
+    fprintf(1, "usage: %s vm|fs\n", av[0]);
+  if (op) {
+    mtenable_type(mtrace_record_ascope, "xv6-asharing");
+    pthread_barrier_init(&bar, 0, ncore);
+    for (u64 i = 0; i < ncore; i++) {
+      next();
+      pthread_t tid;
+      pthread_create(&tid, 0, op, (void*) i);
+    }
+    mtdisable("xv6-asharing");
+  }
+}
--- a/bin/mkdir.cc
+++ b/bin/mkdir.cc
+#include "types.h"
+#include "stat.h"
+#include "user.h"
+int
+main(int argc, char *argv[])
+{
+  int i;
+  if (argc < 2)
+    die("ussage: mkdir files...");
+  for(i = 1; i < argc; i++) {
+    if (mkdir(argv[i]) < 0)
+      die("mkdir: %s failed to create", argv[i]);
+  }
+}
--- a/bin/mktree.cc
+++ b/bin/mktree.cc
+#include "types.h"
+#include "stat.h"
+#include "user.h"
+#include "lib.h"
+#include "fcntl.h"
+static void
+dolevel(int fd, int branch, int depth)
+{
+  if (depth > 0) {
+    for (int i = 0; i < branch; i++) {
+      char name = 'a' + i;
+      if (mkdirat(fd, &name) < 0)
+        die("mkdirat");
+      int nfd = openat(fd, &name, O_RDONLY);
+      if (nfd < 0)
+        die("openat");
+      dolevel(nfd, branch, depth-1);
+    }
+  }
+  close(fd);
+}
+int
+main(int ac, char **av)
+{
+  if (ac < 4)
+    die("usage: %s dir branch depth", av[0]);
+  const char *dir = av[1];
+  int branch = atoi(av[2]);
+  int depth = atoi(av[3]);
+  if (mkdir(dir))
+    die("mkdir");
+  int fd = open(dir, O_RDONLY);
+  if (fd < 0)
+    die("open");
+  dolevel(fd, branch, depth);
+}
--- a/bin/rm.cc
+++ b/bin/rm.cc
+#include "types.h"
+#include "stat.h"
+#include "user.h"
+int
+main(int argc, char *argv[])
+{
+  int i;
+  if(argc < 2)
+    die("Usage: rm files...");
+  for(i = 1; i < argc; i++){
+    if(unlink(argv[i]) < 0)
+      die("rm: %s failed to delete\n", argv[i]);
+  }
+  exit();
+}
--- a/bin/wqtest.cc
+++ b/bin/wqtest.cc
+#include "types.h"
+#include "user.h"
+#include "lib.h"
+#include "amd64.h"
+#include "wq.hh"
+#define NEW_DELETE_OPS(classname)                                   \
+  static void* operator new(unsigned long nbytes) {                 \
+    assert(nbytes == sizeof(classname));                            \
+    return malloc(sizeof(classname));                               \
+  }                                                                 \
+                                                                    \
+  static void operator delete(void *p) {                            \
+    free(p);                                                        \
+  }
+struct testwork : public work {
+  testwork(forframe *b) : barrier_(b) {}
+  virtual void run() {
+    barrier_->dec();
+    delete this;
+  }
+  NEW_DELETE_OPS(testwork);
+  struct forframe *barrier_;
+};
+static void
+test0(void)
+{
+  enum { pushes = 100 };
+  struct forframe wqbarrier(pushes);
+  printf("test0...\n");
+  for (int i = 0; i < pushes; i++) {
+    testwork *w = new testwork(&wqbarrier);
+    wq_push(w);
+  }
+  while (!wqbarrier.zero())
+    nop_pause();
+  printf("test0 done\n");
+}
+struct forkwork : public work {
+  forkwork(forframe *b) : barrier_(b) {}
+  virtual void run() {
+    int pid;
+    pid = fork(0);
+    if (pid < 0)
+      die("forkwork::run: fork");
+    else if (pid == 0)
+      exit();
+    wait();
+    barrier_->dec();
+    delete this;
+  }
+  NEW_DELETE_OPS(forkwork);
+  struct forframe *barrier_;
+};
+static void
+testfork(void)
+{
+  enum { forks = 100 };
+  struct forframe wqbarrier(forks);
+  printf("testfork...\n");
+  for (int i = 0; i < forks; i++) {
+    forkwork *w = new forkwork(&wqbarrier);
+    wq_push(w);
+  }
+  while (!wqbarrier.zero())
+    nop_pause();
+  printf("testfork done\n");
+}
+struct execwork : public work {
+  execwork(forframe *b) : barrier_(b) {}
+  virtual void run() {
+    int pid;
+    pid = fork(0);
+    if (pid < 0)
+      die("execwork::run: fork");
+    else if (pid == 0) {
+      static const char *args[] = { "echo", 0 };
+      exec(args[0], args);
+      die("execwork: exec failed");
+    }
+    wait();
+    barrier_->dec();
+    delete this;
+  }
+  static void test(void) {
+    enum { execs = 100 };
+    struct forframe wqbarrier(execs);
+    printf("testexec...\n");
+    for (int i = 0; i < execs; i++) {
+      execwork *w = new execwork(&wqbarrier);
+      wq_push(w);
+    }
+    while (!wqbarrier.zero())
+      nop_pause();
+    printf("testexec done\n");
+  }
+  NEW_DELETE_OPS(execwork);
+  struct forframe *barrier_;
+};
+int
+main(int ac, char **av)
+{
+  initwq();
+  test0();
+  testfork();
+  execwork::test();
+  exitwq();
+  return 0;
+}
--- a/include/cpputil.hh
+++ b/include/cpputil.hh
@@ -133,7 +133,7 @@ extern void *__dso_handle;
 #define NEW_DELETE_OPS(classname)                                   \
  static void* operator new(unsigned long nbytes) {                 \
    assert(nbytes == sizeof(classname));                            \
-    return kmalloc(sizeof(classname));                              \
+    return kmalloc(sizeof(classname), #classname);                  \
  }                                                                 \
                                                                    \
  static void* operator new(unsigned long nbytes, classname *buf) { \

--- a/include/elf.hh
+++ b/include/elf.hh
@@ -40,10 +40,49 @@ struct proghdr {
  Elf64_Xword align;            // Segment alignment, file & memory
 };
+struct elfnote {
+  Elf64_Word namesz;            // Name size
+  Elf64_Word descsz;	        // Content size
+  Elf64_Word type;	        // Content type
+};
 // Values for Proghdr type
 #define ELF_PROG_LOAD           1
+#define ELF_PROG_NOTE           4
 // Flag bits for Proghdr flags
 #define ELF_PROG_FLAG_EXEC      1
 #define ELF_PROG_FLAG_WRITE     2
 #define ELF_PROG_FLAG_READ      4
+// All known .note types
+#define ELF_NOTE_XV6_ADDR       1
+// xv6-specific address note
+struct xv6_addrdesc {
+  Elf64_Word id;
+  Elf64_Addr vaddr;
+};
+struct xv6_addrnote {
+  struct elfnote elfnote;
+  // name is 0 bytes
+  struct xv6_addrdesc desc;
+};
+// All xv6-specific IDs for notes about addresses
+#define XV6_ADDR_ID_WQ 1
+#define DEFINE_XV6_ADDRNOTE(xname, xid, xvaddr)                         \
+  const struct xv6_addrnote xname PROG_NOTE_ATTRIBUTE = {               \
+    elfnote: {                                                          \
+      namesz: 0,                                                        \
+      descsz: sizeof(((xv6_addrnote *)nullptr)->desc),                  \
+      type: ELF_NOTE_XV6_ADDR                                           \
+    },                                                                  \
+    desc: {                                                             \
+      id: (xid),                                                        \
+      vaddr: (Elf64_Addr)(xvaddr) }                                     \
+  }
+#define PROG_NOTE_ATTRIBUTE __attribute__ ((section(".note"), used))
--- a/include/hwvm.hh
+++ b/include/hwvm.hh
@@ -10,7 +10,7 @@ extern pgmap kpml4;
 void            freevm(pgmap *pml4);
 pgmap*          setupkvm(void);
-int             setupkshared(pgmap *pml4, char *kshared);
+int             mapkva(pgmap *pml4, char* kva, uptr uva, size_t size);
 std::atomic<pme_t>* walkpgdir(pgmap *pml4, u64, int);
 void            tlbflush(void);

--- a/include/kalloc.hh
+++ b/include/kalloc.hh
@@ -63,6 +63,7 @@ enum {
  slab_perf,
  slab_kshared,
  slab_wq,
+  slab_userwq,
  slab_type_max
 };

--- a/include/kern_c.h
+++ b/include/kern_c.h
@@ -29,7 +29,7 @@ long sys_fstat(int, struct stat*);
 long sys_getpid(void);
 long sys_kill(int);
 long sys_link(const char*, const char*);
-long sys_mkdir(const char*);
+long sys_mkdirat(int, const char*);
 long sys_mknod(const char*, int, int);
 long sys_openat(int, const char*, int);
 long sys_pipe(int*);
@@ -51,6 +51,8 @@ long sys_pread(int fd, void *ubuf, size_t count, off_t offset);
 long sys_async(int, size_t, off_t, u32, u32);
 long sys_script(void *addr, u64 len, u64 chunk);
 long sys_setfs(u64 base);
+long sys_wqwait(void);
+long sys_setaffinity(int cpu);
 extern long (*syscalls[])(u64, u64, u64, u64, u64);
 // other exported/imported functions

--- a/include/kernel.hh
+++ b/include/kernel.hh
@@ -5,13 +5,9 @@ extern "C" {
 }
 #include "atomic.hh"
+#include "memlayout.h"
 #include <stdarg.h>
-#define KBASE   0xFFFFFF0000000000ull
-#define KCODE   0xFFFFFFFFC0000000ull
-#define KSHARED 0xFFFFF00000000000ull
-#define USERTOP 0x0000800000000000ull
 #define KCSEG (2<<3)  /* kernel code segment */
 #define KDSEG (3<<3)  /* kernel data segment */
@@ -72,6 +68,7 @@ void            vcprintf(const char *fmt, va_list ap);
 void            panic(const char*, ...) 
                  __noret__ __attribute__((format(printf, 1, 2)));
 void            kerneltrap(struct trapframe *tf) __noret__;
+void            vsnprintf(char *buf, u32 n, const char *fmt, va_list ap);
 void            snprintf(char *buf, u32 n, const char *fmt, ...);
 void            printtrace(u64 rbp);
@@ -125,13 +122,13 @@ void            idlezombie(struct proc*);
 void            ioapicenable(int irq, int cpu);
 // kalloc.c
-char*           kalloc(void);
+char*           kalloc(const char *name);
 void            kfree(void*);
 void*           ksalloc(int slabtype);
 void            ksfree(int slabtype, void*);
-void*           kmalloc(u64 nbytes);
+void*           kmalloc(u64 nbytes, const char *name);
 void            kmfree(void*, u64 nbytes);
-int             kmalign(void **p, int align, u64 size);
+int             kmalign(void **p, int align, u64 size, const char *name);
 void            kmalignfree(void *, int align, u64 size);
 void            verifyfree(char *ptr, u64 nbytes);
 void            kminit(void);
@@ -170,7 +167,6 @@ int             piperead(struct pipe*, char*, int);
 int             pipewrite(struct pipe*, char*, int);
 // proc.c
-struct proc*    allocproc(void);
 struct proc*    copyproc(struct proc*);
 void            finishproc(struct proc*);
 void            exit(void);

--- a/include/kmtrace.hh
+++ b/include/kmtrace.hh
+#pragma once
 #include "mtrace.h"
 #if MTRACE
 // Tell mtrace about switching threads
 struct kstack_tag {
@@ -61,6 +64,51 @@ static inline void mtresume(struct proc *p)
 #define mtrec() mtrace_call_set(1, ~0ull)
 #define mtign() mtrace_call_set(0, ~0ull)
+class mt_ascope
+{
+  char name[32];
+public:
+  explicit mt_ascope(const char *fmt, ...)
+  {
+    va_list ap;
+    va_start(ap, fmt);
+    vsnprintf(name, sizeof(name) - 1, fmt, ap);
+    va_end(ap);
+    mtrace_ascope_register(0, name);
+  }
+  ~mt_ascope()
+  {
+    mtrace_ascope_register(1, name);
+  }
+};
+static inline void mtreadavar(const char *fmt, ...)
+{
+  char name[32];
+  va_list ap;
+  va_start(ap, fmt);
+  vsnprintf(name, sizeof(name), fmt, ap);
+  va_end(ap);
+  mtrace_avar_register(0, name);
+}
+static inline void mtwriteavar(const char *fmt, ...)
+{
+  char name[32];
+  va_list ap;
+  va_start(ap, fmt);
+  vsnprintf(name, sizeof(name), fmt, ap);
+  va_end(ap);
+  mtrace_avar_register(1, name);
+}
 #else
 #define mtstart(ip, p) do { } while (0)
 #define mtstop(p) do { } while (0)
@@ -70,4 +118,13 @@ static inline void mtresume(struct proc *p)
 #define mtign(cpu) do { } while (0)
 #define mtrec(cpu) do { } while (0)
 #define mtign(cpu) do { } while (0)
+class mt_ascope
+{
+public:
+  explicit mt_ascope(const char *fmt, ...) {}
+};
+#define mtreadavar(fmt, ...) do { } while (0)
+#define mtwriteavar(fmt, ...) do { } while (0)
 #endif
--- a/include/memlayout.h
+++ b/include/memlayout.h
+#define KBASE      0xFFFFFF0000000000ull
+#define KCODE      0xFFFFFFFFC0000000ull
+#define KSHARED    0xFFFFF00000000000ull
+#define USERWQ     0xFFFFF00100000000ull
+#define USERTOP    0x0000800000000000ull
+#define UWQSTACK   0x0000700000000000ull
--- a/include/mtrace.h
+++ b/include/mtrace.h
@@ -33,8 +33,9 @@ char* strncpy(char *s, const char *t, size_t n);
  mtrace_lock_register(RET_IP(), ptr, lockname(ptr), mtrace_lockop_release, 0)
 // Enable/disable all mtrace logging
-#define mtenable(name)  mtrace_enable_set(1, name)
+#define mtenable(name)  mtrace_enable_set(mtrace_record_movement, name)
-#define mtdisable(name) mtrace_enable_set(0, name)
+#define mtenable_type(type, name)  mtrace_enable_set(type, name)
+#define mtdisable(name) mtrace_enable_set(mtrace_record_disable, name)
 // Log the number of operations 
 static inline void mtops(u64 n)
@@ -54,6 +55,7 @@ static inline void mtops(u64 n)
 #define mtrec(cpu) do { } while (0)
 #define mtign(cpu) do { } while (0)
 #define mtenable(name) do { } while (0)
+#define mtenable_type(type, name) do { } while (0)
 #define mtdisable(name) do { } while (0)
 #define mtops(n) do { } while (0)
 #endif
--- a/include/proc.hh
+++ b/include/proc.hh
@@ -7,6 +7,9 @@
 #include "file.hh"
 #include "filetable.hh"
+class uwq;
+class uwq_worker;
 // Saved registers for kernel context switches.
 // (also implicitly defined in swtch.S)
 struct context {
@@ -43,6 +46,8 @@ enum procstate { EMBRYO, SLEEPING, RUNNABLE, RUNNING, ZOMBIE };
 // Per-process state
 struct proc : public rcu_freed {
  struct vmap *vmap;           // va -> vma
+  uwq* uwq;
+  uwq_worker* worker;
  char *kstack;                // Bottom of kernel stack for this process
  volatile int pid;            // Process ID
  struct proc *parent;         // Parent process
@@ -61,7 +66,6 @@ struct proc : public rcu_freed {
  struct condvar cv;
  std::atomic<u64> epoch;      // low 8 bits are depth count
  char lockname[16];
-  int on_runq;
  int cpu_pin;
 #if MTRACE
  struct mtrace_stacks mtrace_stacks;
@@ -76,15 +80,19 @@ struct proc : public rcu_freed {
  LIST_ENTRY(proc) cv_sleep;   // Linked list of processes sleeping on a cv
  u64 user_fs_;
-  proc(int npid);
-  ~proc(void);
  virtual void do_gc(void) { delete this; }
-  NEW_DELETE_OPS(proc)
  void set_state(enum procstate s);
  enum procstate get_state(void) const { return state_; }
+  int set_cpu_pin(int cpu);
+  static proc* alloc();
 private:
+  proc(int npid);
+  ~proc(void);
+  proc& operator=(const proc&);
+  proc(const proc& x);
+  NEW_DELETE_OPS(proc);
  enum procstate state_;       // Process state  
 };
--- a/include/syscall.h
+++ b/include/syscall.h
@@ -13,7 +13,7 @@
 #define SYS_unlink 12
 #define SYS_fstat  13
 #define SYS_link   14
-#define SYS_mkdir  15
+#define SYS_mkdirat 15
 #define SYS_chdir  16
 #define SYS_dup    17
 #define SYS_getpid 18
@@ -31,4 +31,6 @@
 #define SYS_async  30
 #define SYS_script 31
 #define SYS_setfs  32
-#define SYS_ncount 33   /* total number of system calls */
+#define SYS_wqwait 33
+#define SYS_setaffinity 34
+#define SYS_ncount 35   /* total number of system calls */
--- a/include/user.h
+++ b/include/user.h
@@ -18,6 +18,7 @@ int unlink(const char*);
 int fstat(int fd, struct stat*);
 int link(const char*, const char*);
 int mkdir(const char*);
+int mkdirat(int dirfd, const char *pathname);
 int chdir(const char*);
 int dup(int);
 int getpid(void);
@@ -31,6 +32,7 @@ ssize_t pread(int, void*, size_t, off_t);
 int async(int, size_t, off_t, u32, u32);
 int script(void *addr, u64 len, u64 chunk);
 int setfs(u64 base);
+int setaffinity(int cpu);
 // ulib.c
 int stat(char*, struct stat*);

--- a/include/uwq.hh
+++ b/include/uwq.hh
+#pragma once
+struct padded_length {
+  volatile u64 v_ __mpalign__;;
+  __padout__;
+};
+#if defined (XV6_KERNEL)
+bool uwq_trywork(void);
+#define NWORKERS (NCPU-1)
+struct uwq;
+struct uwq_worker {
+  uwq_worker(uwq*, proc*);
+  long wait();
+  void exit();
+  uwq* uwq_;
+  proc *proc_;
+  bool running_;
+  struct spinlock lock_;
+  struct condvar cv_;
+  NEW_DELETE_OPS(uwq_worker);
+};
+struct uwq : public referenced, public rcu_freed {
+  friend struct uwq_worker;
+  static uwq* alloc(vmap* vmap, filetable *ftable);
+  bool  haswork() const;
+  bool  tryworker();
+  void  setuentry(uptr uentry);
+  virtual void do_gc(void) { delete this; }
+protected:
+  virtual void onzero() const;
+private:
+  uwq(vmap* vmap, filetable* ftable, padded_length *len);
+  ~uwq();
+  uwq& operator=(const uwq&);
+  uwq(const uwq& x);
+  proc* allocworker();
+  void  finish();
+  NEW_DELETE_OPS(uwq);
+  struct spinlock lock_;
+  vmap* vmap_;
+  filetable* ftable_;
+  padded_length* len_;
+  uptr uentry_;
+  uptr ustack_;
+  std::atomic<u64> uref_;
+  uwq_worker* worker_[NWORKERS];
+};
+#endif
--- a/include/vm.hh
+++ b/include/vm.hh
@@ -5,10 +5,13 @@
 #include "radix.hh"
 #include "cpputil.hh"
 #include "hwvm.hh"
+#include "uwq.hh"
 #define VM_CRANGE 1
 #define VM_RADIX  0
+struct padded_length;
 using std::atomic;
 // A memory object (physical pages or inode).
@@ -17,7 +20,6 @@ enum vmntype { EAGER, ONDEMAND };
 struct vmnode {
  const u64 npages;
  atomic<char*> page[128];
-  atomic<u64> ref;
  const enum vmntype type;
  struct inode *const ip;
  const u64 offset;
@@ -27,11 +29,15 @@ struct vmnode {
         inode *i = 0, u64 off = 0, u64 s = 0);
  ~vmnode();
  void decref();
+  void incref();
+  u64 ref();
  int allocpg();
  vmnode* copy();
  int demand_load();
-  NEW_DELETE_OPS(vmnode)
+  NEW_DELETE_OPS(vmnode);
+private:
+  atomic<u64> ref_;
 };
 // A mapping of a chunk of an address space to
@@ -78,6 +84,8 @@ struct vmap {
  bool replace_vma(vma *a, vma *b);
  void decref();
+  void incref();
  vmap* copy(int share);
  vma* lookup(uptr start, uptr len);
  int insert(vmnode *n, uptr va_start, int dotlb);

--- a/include/wq.hh
+++ b/include/wq.hh
@@ -35,7 +35,7 @@ struct cwork : public work {
 #define xmalloc(n)     malloc(n)
 #define xfree(p, sz)   free(p)
 #elif defined(XV6_KERNEL)
-#define xmalloc(n)   kmalloc(n) 
+#define xmalloc(n)   kmalloc(n, "xmalloc")
 #define xfree(p, sz) kmfree(p, sz)
 #else
 #define xmalloc(n)     malloc(n)

--- a/include/wquser.hh
+++ b/include/wquser.hh
@@ -4,10 +4,16 @@
 #include "user.h"
 #include "wq.hh"
 #include "pthread.h"
+#include "memlayout.h"
+#include "uwq.hh"
+#include "atomic.hh"
+#include "lib.h"
+#include "elf.hh"
 typedef struct uspinlock wqlock_t;
 static pthread_key_t idkey;
+static std::atomic<int> nextid;
 static volatile int exiting;
 int
@@ -22,6 +28,18 @@ allocwq(unsigned long nbytes)
  return malloc(nbytes);
 }
+static inline padded_length*
+allocklen(unsigned long nbytes)
+{
+  static bool alloced;
+  if (alloced)
+    die("allocklen: allocing more than once");
+  if (nbytes > USERWQSIZE)
+    die("allocklen: too large");
+  alloced = true;
+  return (padded_length*)USERWQ;
+}
 static inline void
 wqlock_acquire(wqlock_t *lock)
 {
@@ -46,42 +64,32 @@ wqlock_init(wqlock_t *lock)
  initlock(lock);
 }
-static void
+extern "C" long wqwait(void);
-setaffinity(int c)
-{
-  // XXX(sbw)
-}
-static void*
+static void __attribute__((used))
-workerth(void *x)
+initworker(void)
 {
-  u64 c = (u64)x;
+  int id;
+  forkt_setup(0);
-  setaffinity(c);
+  id = nextid++;
-  pthread_setspecific(idkey, (void*)c);
+  if (id >= NCPU)
-  while (!exiting)
+    die("initworker: to man IDs");
-    wq_trywork();
+  pthread_setspecific(idkey, (void*)(u64)id);
+  while (1) {
-  return 0;
+    if (!wq_trywork())
+      assert(wqwait() == 0);
+  }
 }
+DEFINE_XV6_ADDRNOTE(xnote, XV6_ADDR_ID_WQ, &initworker);
 static inline void
 wqarch_init(void)
 {
-  pthread_t th;
-  int r;
  if (pthread_key_create(&idkey, 0))
    die("wqarch_init: pthread_key_create");
-  pthread_setspecific(idkey, 0);
+  int id = nextid++;
-  setaffinity(0);
+  pthread_setspecific(idkey, (void*)(u64)id);
-  for (int i = 1; i < NCPU; i++) {
-    r = pthread_create(&th, 0, workerth, (void*)(u64)i);
-    if (r < 0)
-      die("wqarch_init: pthread_create");
-  }
 }
 static inline void

--- a/kernel/Makefrag
+++ b/kernel/Makefrag
@@ -42,6 +42,8 @@ OBJS = \
 	sysfile.o \
 	sysproc.o \
 	uart.o \
+        user.o \
+	uwq.o \
 	vm.o \
 	trap.o \
 	trapasm.o \

--- a/kernel/bootdata.c
+++ b/kernel/bootdata.c
@@ -39,7 +39,7 @@ long (*syscalls[])(u64, u64, u64, u64, u64) = {
  SYSCALL(getpid),
  SYSCALL(kill),
  SYSCALL(link),
-  SYSCALL(mkdir),
+  SYSCALL(mkdirat),
  SYSCALL(mknod),
  SYSCALL(openat),
  SYSCALL(pipe),
@@ -61,5 +61,6 @@ long (*syscalls[])(u64, u64, u64, u64, u64) = {
  SYSCALL(async),
  SYSCALL(script),
  SYSCALL(setfs),
+  SYSCALL(wqwait),
+  SYSCALL(setaffinity),
 };
--- a/kernel/cpprt.cc
+++ b/kernel/cpprt.cc
@@ -7,7 +7,7 @@
 void *
 operator new[](unsigned long nbytes)
 {
-  u64 *x = (u64*) kmalloc(nbytes + sizeof(u64));
+  u64 *x = (u64*) kmalloc(nbytes + sizeof(u64), "array");
  *x = nbytes + sizeof(u64);
  return x+1;
 }

--- a/kernel/exec.cc
+++ b/kernel/exec.cc
@@ -26,6 +26,36 @@ struct eargs {
  char **argv;
 };
+static int
+donotes(struct inode *ip, uwq *uwq, u64 off)
+{
+  struct proghdr ph;
+  struct elfnote note;
+  if (readi(ip, (char*)&ph, off, sizeof(ph)) != sizeof(ph))
+    return -1;
+  if (readi(ip, (char*)&note, ph.offset, sizeof(note)) != sizeof(note))
+    return -1;
+  if (note.type == ELF_NOTE_XV6_ADDR) {
+    struct xv6_addrdesc desc;
+    if (note.descsz != sizeof(desc))
+      return -1;
+    if (readi(ip, (char*)&desc,
+              ph.offset+__offsetof(struct xv6_addrnote, desc),
+              sizeof(desc)) != sizeof(desc))
+      return -1;
+    if (desc.id == XV6_ADDR_ID_WQ) {
+      uwq->setuentry(desc.vaddr);
+      return 0;
+    }
+  }
+  return -1;
+}
 static void
 dosegment(struct eargs *args, u64 off)
 {
@@ -149,15 +179,19 @@ exec(const char *path, char **argv)
 {
  struct inode *ip = nullptr;
  struct vmap *vmp = nullptr;
+  uwq* uwq = nullptr;
  struct elfhdr elf;
  struct proghdr ph;
  u64 off;
  int i;
  struct vmap *oldvmap;
  if((ip = namei(myproc()->cwd, path)) == 0)
    return -1;
+  if(myproc()->worker != nullptr)
+    return -1;
  gc_begin_epoch();
  // Check ELF header
@@ -171,6 +205,9 @@ exec(const char *path, char **argv)
  if((vmp = vmap::alloc()) == 0)
    goto bad;
+  if((uwq = uwq::alloc(vmp, myproc()->ftable)) == 0)
+    goto bad;
  // Arguments for work queue
  struct eargs args;
  args.proc = myproc();
@@ -186,7 +223,12 @@ exec(const char *path, char **argv)
             off+__offsetof(struct proghdr, type), 
             sizeof(type)) != sizeof(type))
      goto bad;
-    if(type != ELF_PROG_LOAD)
+    if (type == ELF_PROG_NOTE) {
+      if (donotes(ip, uwq, off) < 0) {
+        cilk_abort(-1);
+        break;
+      }
+    } if(type != ELF_PROG_LOAD)
      continue;
    cilk_call(dosegment, &args, off);
  }
@@ -203,7 +245,10 @@ exec(const char *path, char **argv)
  // Commit to the user image.
  oldvmap = myproc()->vmap;
  myproc()->vmap = vmp;
-  myproc()->tf->rip = elf.entry;  // main
+  if (myproc()->uwq != nullptr)
+    myproc()->uwq->dec();
+  myproc()->uwq = uwq;
+  myproc()->tf->rip = elf.entry;
  switchvm(myproc());
  oldvmap->decref();
@@ -215,7 +260,8 @@ exec(const char *path, char **argv)
  cprintf("exec failed\n");
  if(vmp)
    vmp->decref();
+  if(uwq)
+    uwq->dec();
  gc_end_epoch();
  return 0;
 }
--- a/kernel/fs.cc
+++ b/kernel/fs.cc
@@ -22,6 +22,7 @@
 #include "buf.hh"
 #include "file.hh"
 #include "cpu.hh"
+#include "kmtrace.hh"
 #define min(a, b) ((a) < (b) ? (a) : (b))
 static void itrunc(struct inode*);
@@ -757,6 +758,16 @@ namex(inode *cwd, const char *path, int nameiparent, char *name)
    ip = idup(cwd);
  while((r = skipelem(&path, name)) == 1){
+    // XXX Doing this here requires some annoying reasoning about all
+    // of the callers of namei/nameiparent.  Also, since the abstract
+    // scope is implicit, it might be wrong (or non-existent) and
+    // documenting the abstract object sets of each scope becomes
+    // difficult and probably unmaintainable.  We have to compute this
+    // information here because it's the only place that's canonical.
+    // Maybe this should return the set of inodes traversed and let
+    // the caller declare the variables?  Would it help for the caller
+    // to pass in an abstract scope?
+    mtreadavar("inode:%x.%x", ip->dev, ip->inum);
    next = 0;
    if(next == 0){
      if(ip->type == 0)
@@ -785,6 +796,7 @@ namex(inode *cwd, const char *path, int nameiparent, char *name)
    gc_end_epoch();
    return 0;
  }
+  mtreadavar("inode:%x.%x", ip->dev, ip->inum);
  gc_end_epoch();
  return ip;
 }

--- a/kernel/hwvm.cc
+++ b/kernel/hwvm.cc
@@ -10,9 +10,14 @@
 #include "condvar.h"
 #include "proc.hh"
 #include "vm.hh"
+#include "wq.hh"
 using namespace std;
+static const char *levelnames[] = {
+  "PT", "PD", "PDP", "PML4"
+};
 static pgmap*
 descend(pgmap *dir, u64 va, u64 flags, int create, int level)
 {
@@ -28,7 +33,7 @@ retry:
  } else {
    if (!create)
      return nullptr;
-    next = (pgmap*) kalloc();
+    next = (pgmap*) kalloc(levelnames[level-1]);
    if (!next)
      return nullptr;
    memset(next, 0, PGSIZE);
@@ -83,7 +88,7 @@ setupkvm(void)
  pgmap *pml4;
  int k;
-  if((pml4 = (pgmap*)kalloc()) == 0)
+  if((pml4 = (pgmap*)kalloc("PML4")) == 0)
    return 0;
  k = PX(3, KBASE);
  memset(&pml4->e[0], 0, 8*k);
@@ -92,13 +97,36 @@ setupkvm(void)
 }
 int
-setupkshared(pgmap *pml4, char *kshared)
+mapkva(pgmap *pml4, char* kva, uptr uva, size_t size)
 {
-  for (u64 off = 0; off < KSHAREDSIZE; off+=4096) {
+  for (u64 off = 0; off < size; off+=4096) {
-    atomic<pme_t> *pte = walkpgdir(pml4, (u64) (KSHARED+off), 1);
+    atomic<pme_t> *pte = walkpgdir(pml4, (u64) (uva+off), 1);
    if (pte == nullptr)
-      panic("setupkshared: oops");
+      return -1;
-    *pte = v2p(kshared+off) | PTE_P | PTE_U | PTE_W;
+    *pte = v2p(kva+off) | PTE_P | PTE_U | PTE_W;
+  }
+  return 0;
+}
+int
+setupuvm(pgmap *pml4, char *kshared, char *uwq)
+{
+  struct todo {
+    char *kvm;
+    char *uvm;
+    size_t size;
+  } todo[] = {
+    { kshared, (char*)KSHARED, KSHAREDSIZE },
+    { uwq,     (char*)USERWQ,  USERWQSIZE }
+  };
+  for (int i = 0; i < NELEM(todo); i++) {
+    for (u64 off = 0; off < todo[i].size; off+=4096) {
+      atomic<pme_t> *pte = walkpgdir(pml4, (u64) (todo[i].uvm+off), 1);
+      if (pte == nullptr)
+        return -1;
+      *pte = v2p(todo[i].kvm+off) | PTE_P | PTE_U | PTE_W;
+    }
  }
  return 0;
 }

--- a/kernel/idle.cc
+++ b/kernel/idle.cc
@@ -8,6 +8,7 @@
 #include "sched.hh"
 #include "percpu.hh"
 #include "wq.hh"
+#include "uwq.hh"
 #include "kmtrace.hh"
 struct idle {
@@ -107,7 +108,7 @@ idleloop(void)
        // If we don't have an heir, try to allocate one
        if (idlem->heir == nullptr) {
          struct proc *p;
-          p = allocproc();
+          p = proc::alloc();
          if (p == nullptr)
            break;
          snprintf(p->name, sizeof(p->name), "idleh_%u", mycpu()->id);          
@@ -118,6 +119,9 @@ idleloop(void)
          idlem->heir = p;
        }
+        if (uwq_trywork())
+          break;
        worked = wq_trywork();
        // If we are no longer the idle thread, exit
        if (worked && idlem->cur != myproc())
@@ -131,9 +135,9 @@ idleloop(void)
 void
 initidle(void)
 {
-  struct proc *p = allocproc();
+  struct proc *p = proc::alloc();
  if (!p)
-    panic("initidle allocproc");
+    panic("initidle proc::alloc");
  SLIST_INIT(&idlem[cpunum()].zombies);
  initlock(&idlem[cpunum()].lock, "idle_lock", LOCKSTAT_IDLE);

--- a/kernel/kalloc.cc
+++ b/kernel/kalloc.cc
@@ -160,7 +160,7 @@ kmemprint()
 }
 static char*
-kalloc_pool(struct kmem *km)
+kalloc_pool(struct kmem *km, const char *name)
 {
  struct run *r = 0;
  struct kmem *m;
@@ -196,7 +196,8 @@ kalloc_pool(struct kmem *km)
    return 0;
  }
-  mtlabel(mtrace_label_block, r, m->size, "kalloc", sizeof("kalloc"));
+  if (name)
+    mtlabel(mtrace_label_block, r, m->size, name, strlen(name));
  if (ALLOC_MEMSET && m->size <= 16384)
    memset(r, 2, m->size);
@@ -207,17 +208,17 @@ kalloc_pool(struct kmem *km)
 // Returns a pointer that the kernel can use.
 // Returns 0 if the memory cannot be allocated.
 char*
-kalloc(void)
+kalloc(const char *name)
 {
  if (!kinited)
    return pgalloc();
-  return kalloc_pool(kmems);
+  return kalloc_pool(kmems, name);
 }
 void *
 ksalloc(int slab)
 {
-  return kalloc_pool(slabmem[slab]);
+  return kalloc_pool(slabmem[slab], slabmem[slab]->name);
 }
 void
@@ -278,6 +279,10 @@ initkalloc(u64 mbaddr)
    slabmem[slab_wq][c].size = PGROUNDUP(wq_size());
    slabmem[slab_wq][c].ninit = NCPU;
+    strncpy(slabmem[slab_userwq][c].name, " uwq", MAXNAME);
+    slabmem[slab_userwq][c].size = USERWQSIZE;
+    slabmem[slab_userwq][c].ninit = CPUKSTACKS;
    for (int i = 0; i < slab_type_max; i++) {
      slabmem[i][c].name[0] = (char) c + '0';
      slabinit(&slabmem[i][c], &p, &k);

--- a/kernel/kmalloc.cc
+++ b/kernel/kmalloc.cc
@@ -34,10 +34,10 @@ kminit(void)
 }
 // get more space for freelists[c].buckets[b]
-int
+static int
 morecore(int c, int b)
 {
-  char *p = kalloc();
+  char *p = kalloc(nullptr);
  if(p == 0)
    return -1;
@@ -78,7 +78,7 @@ bucket(u64 nbytes)
 }
 void *
-kmalloc(u64 nbytes)
+kmalloc(u64 nbytes, const char *name)
 {
  int b = bucket(nbytes);
@@ -103,10 +103,11 @@ kmalloc(u64 nbytes)
    }
  }
+  mtlabel(mtrace_label_heap, (void*) h, nbytes, name, strlen(name));
  if (ALLOC_MEMSET)
    memset(h, 4, (1<<b));
-  mtlabel(mtrace_label_heap, (void*) h, nbytes, "kmalloc'ed", sizeof("kmalloc'ed"));
  return h;
 }
@@ -132,9 +133,9 @@ kmfree(void *ap, u64 nbytes)
 }
 int
-kmalign(void **p, int align, u64 size)
+kmalign(void **p, int align, u64 size, const char *name)
 {
-  void *mem = kmalloc(size + (align-1) + sizeof(void*));
+  void *mem = kmalloc(size + (align-1) + sizeof(void*), name);
  char *amem = ((char*)mem) + sizeof(void*);
  amem += align - ((uptr)amem & (align - 1));
  ((void**)amem)[-1] = mem;

--- a/kernel/net.cc
+++ b/kernel/net.cc
@@ -32,7 +32,7 @@ netfree(void *va)
 void *
 netalloc(void)
 {
-  return kalloc();
+  return kalloc("(netalloc)");
 }
 int
@@ -278,7 +278,7 @@ netbind(int sock, void *xaddr, int xaddrlen)
  void *addr;
  long r;
-  addr = kmalloc(xaddrlen);
+  addr = kmalloc(xaddrlen, "sockaddr");
  if (addr == nullptr)
    return -1;
@@ -314,7 +314,7 @@ netaccept(int sock, void *xaddr, void *xaddrlen)
  if (umemcpy(&len, lenptr, sizeof(*lenptr)))
    return -1;
-  addr = kmalloc(len);
+  addr = kmalloc(len, "sockaddr");
  if (addr == nullptr)
    return -1;
@@ -352,7 +352,7 @@ netwrite(int sock, char *ubuf, int len)
  int cc;
  int r;
-  kbuf = kalloc();
+  kbuf = kalloc("(netwrite)");
  if (kbuf == nullptr)
    return -1;
@@ -375,7 +375,7 @@ netread(int sock, char *ubuf, int len)
  int cc;
  int r;
-  kbuf = kalloc();
+  kbuf = kalloc("(netread)");
  if (kbuf == nullptr)
    return -1;

--- a/kernel/pipe.cc
+++ b/kernel/pipe.cc
@@ -30,7 +30,7 @@ pipealloc(struct file **f0, struct file **f1)
  *f0 = *f1 = 0;
  if((*f0 = file::alloc()) == 0 || (*f1 = file::alloc()) == 0)
    goto bad;
-  if((p = (pipe*)kmalloc(sizeof(*p))) == 0)
+  if((p = (pipe*)kmalloc(sizeof(*p), "pipe")) == 0)
    goto bad;
  p->readopen = 1;
  p->writeopen = 1;

--- a/kernel/proc.cc
+++ b/kernel/proc.cc
@@ -27,7 +27,7 @@ mycpuid(void)
 }
 xns<u32, proc*, proc_hash> *xnspid __mpalign__;
-static struct proc *bootproc __mpalign__;
+struct proc *bootproc __mpalign__;
 #if MTRACE
 struct kstack_tag kstack_tag[NCPU];
@@ -36,10 +36,10 @@ struct kstack_tag kstack_tag[NCPU];
 enum { sched_debug = 0 };
 proc::proc(int npid) :
-  rcu_freed("proc"), vmap(0), kstack(0),
+  rcu_freed("proc"), vmap(0), uwq(0), worker(0), kstack(0),
  pid(npid), parent(0), tf(0), context(0), killed(0),
  ftable(0), cwd(0), tsc(0), curcycles(0), cpuid(0), epoch(0),
-  on_runq(-1), cpu_pin(0), runq(0), oncv(0), cv_wakeup(0),
+  cpu_pin(0), runq(0), oncv(0), cv_wakeup(0),
  user_fs_(0), state_(EMBRYO)
 {
  snprintf(lockname, sizeof(lockname), "cv:proc:%d", pid);
@@ -85,6 +85,30 @@ proc::set_state(enum procstate s)
  state_ = s;
 }
+int
+proc::set_cpu_pin(int cpu)
+{
+  if (cpu < -1 || cpu >= ncpu)
+    return -1;
+  acquire(&lock);
+  if (myproc() != this)
+    panic("set_cpu_pin not implemented for non-current proc");
+  if (cpu == -1) {
+    cpu_pin = 0;
+    release(&lock);
+    return 0;
+  }
+  // Since we're the current proc, there's no runq to get off.
+  // post_swtch will put us on the new runq.
+  cpuid = cpu;
+  cpu_pin = 1;
+  myproc()->set_state(RUNNABLE);
+  sched();
+  assert(mycpu()->id == cpu);
+  return 0;
+}
 // Give up the CPU for one scheduling round.
 void
 yield(void)
@@ -174,18 +198,15 @@ freeproc(struct proc *p)
  gc_delayed(p);
 }
-// Look in the process table for an UNUSED proc.
+proc*
-// If found, change state to EMBRYO and initialize
+proc::alloc(void)
-// state required to run in the kernel.
-// Otherwise return 0.
-struct proc*
-allocproc(void)
 {
-  struct proc *p;
  char *sp;
+  proc* p;
  p = new proc(xnspid->allockey());
-  if (p == 0) return 0;
+  if (p == nullptr)
+    return nullptr;
  p->cpuid = mycpu()->id;
  initprocgc(p);
@@ -230,43 +251,6 @@ allocproc(void)
  return p;
 }
-// Set up first user process.
-void
-inituser(void)
-{
-  struct proc *p;
-  extern u8 _initcode_start[];
-  extern u64 _initcode_size;
-  p = allocproc();
-  p->ftable = new filetable();
-  if (p->ftable == nullptr)
-    panic("userinit: new filetable");
-  bootproc = p;
-  if((p->vmap = vmap::alloc()) == 0)
-    panic("userinit: out of vmaps?");
-  vmnode *vmn =  new vmnode(PGROUNDUP(_initcode_size) / PGSIZE);
-  if(vmn == 0)
-    panic("userinit: vmn_allocpg");
-  if(p->vmap->insert(vmn, 0, 1) < 0)
-    panic("userinit: vmap_insert");
-  if(p->vmap->copyout(0, _initcode_start, _initcode_size) < 0)
-    panic("userinit: copyout");
-  memset(p->tf, 0, sizeof(*p->tf));
-  p->tf->cs = UCSEG | 0x3;
-  p->tf->ds = UDSEG | 0x3;
-  p->tf->ss = p->tf->ds;
-  p->tf->rflags = FL_IF;
-  p->tf->rsp = PGSIZE;
-  p->tf->rip = 0x0;  // beginning of initcode.S
-  safestrcpy(p->name, "initcode", sizeof(p->name));
-  p->cwd = 0; // forkret will fix in the process's context
-  acquire(&p->lock);
-  addrun(p);
-  release(&p->lock);
-}
 void
 initproc(void)
 {
@@ -357,7 +341,7 @@ fork(int flags)
  // cprintf("%d: fork\n", myproc()->pid);
  // Allocate process.
-  if((np = allocproc()) == 0)
+  if((np = proc::alloc()) == 0)
    return -1;
  if(flags == 0) {
@@ -377,6 +361,7 @@ fork(int flags)
  np->parent = myproc();
  *np->tf = *myproc()->tf;
+  np->cpu_pin = myproc()->cpu_pin;
  // Clear %eax so that fork returns 0 in the child.
  np->tf->rax = 0;
@@ -411,10 +396,12 @@ fork(int flags)
 void
 finishproc(struct proc *p)
 {
-  ksfree(slab_stack, p->kstack);
-  p->kstack = 0;
  if (p->vmap != nullptr)
    p->vmap->decref();
+  if (p->uwq != nullptr)
+    p->uwq->dec();
+  ksfree(slab_stack, p->kstack);
+  p->kstack = 0;
  if (!xnspid->remove(p->pid, &p))
    panic("wait: ns_remove");
  p->pid = 0;
@@ -477,7 +464,7 @@ threadalloc(void (*fn)(void *), void *arg)
 {
  struct proc *p;
-  p = allocproc();
+  p = proc::alloc();
  if (p == nullptr)
    return 0;

--- a/kernel/rnd.cc
+++ b/kernel/rnd.cc
@@ -3,7 +3,7 @@
 struct seed {
  u64 v;
-} __mapalign__;
+} __mpalign__;
 static struct seed seeds[NCPU] __mpalign__;
 u64 

--- a/kernel/sampler.cc
+++ b/kernel/sampler.cc
@@ -188,7 +188,7 @@ sampread(struct inode *ip, char *dst, u32 off, u32 n)
    u64 len = LOGHEADER_SZ;
    u64 cc;
-    hdr = (logheader*) kmalloc(len);
+    hdr = (logheader*) kmalloc(len, "logheader");
    if (hdr == nullptr)
      return -1;
    hdr->ncpus = NCPU;

--- a/kernel/sched.cc
+++ b/kernel/sched.cc
@@ -60,7 +60,9 @@ sched(void)
  struct proc *next = schednext();
  if (next == nullptr) {
-    if (myproc()->get_state() != RUNNABLE) {
+    if (myproc()->get_state() != RUNNABLE ||
+        // proc changed its CPU pin?
+        myproc()->cpuid != mycpu()->id) {
      next = idleproc();
    } else {
      myproc()->set_state(RUNNING);

--- a/kernel/spinlock.cc
+++ b/kernel/spinlock.cc
@@ -24,7 +24,7 @@ void*
 klockstat::operator new(unsigned long nbytes)
 {
  assert(nbytes == sizeof(klockstat));
-  return kmalloc(sizeof(klockstat));
+  return kmalloc(sizeof(klockstat), "klockstat");
 }
 void

--- a/kernel/sysfile.cc
+++ b/kernel/sysfile.cc
@@ -11,6 +11,7 @@
 #include "fcntl.h"
 #include "cpu.hh"
 #include "net.hh"
+#include "kmtrace.hh"
 static bool
 getfile(int fd, sref<file> *f)
@@ -280,6 +281,13 @@ sys_openat(int dirfd, const char *path, int omode)
  if(argcheckstr(path) < 0)
    return -1;
+  // Reads the dirfd FD, dirfd's inode, the inodes of all files in
+  // path; writes the returned FD
+  mt_ascope ascope("%s(%d,%s,%d)", __func__, dirfd, path, omode);
+  mtwriteavar("thread:%x", myproc()->pid);
+  mtreadavar("inode:%x.%x", cwd->dev, cwd->inum);
  if(omode & O_CREATE){
    if((ip = create(cwd, path, T_FILE, 0, 0)) == 0)
      return -1;
@@ -309,6 +317,7 @@ sys_openat(int dirfd, const char *path, int omode)
    return -1;
  }
  iunlock(ip);
+  mtwriteavar("fd:%x.%x", myproc()->pid, fd);
  f->type = file::FD_INODE;
  f->ip = ip;
@@ -319,11 +328,25 @@ sys_openat(int dirfd, const char *path, int omode)
 }
 long
-sys_mkdir(const char *path)
+sys_mkdirat(int dirfd, const char *path)
 {
+  struct inode *cwd;
  struct inode *ip;
-  if(argcheckstr(path) < 0 || (ip = create(myproc()->cwd, path, T_DIR, 0, 0)) == 0)
+  if (dirfd == AT_FDCWD) {
+    cwd = myproc()->cwd;
+  } else {
+    // XXX(sbw) do we need the sref while we touch fdir->ip?
+    sref<file> fdir;
+    if (!getfile(dirfd, &fdir) || fdir->type != file::FD_INODE)
+      return -1;
+    cwd = fdir->ip;
+  }
+  if (argcheckstr(path) < 0)
+    return -1;
+  ip = create(cwd, path, T_DIR, 0, 0);
+  if (ip == nullptr)
    return -1;
  iunlockput(ip);
  return 0;

--- a/kernel/sysproc.cc
+++ b/kernel/sysproc.cc
@@ -9,6 +9,7 @@
 #include "cpu.hh"
 #include "vm.hh"
 #include "sperf.hh"
+#include "kmtrace.hh"
 long
 sys_fork(int flags)
@@ -87,6 +88,13 @@ sys_map(uptr addr, u64 len)
 {
  ANON_REGION(__func__, &perfgroup);
+#if MTRACE
+  mt_ascope ascope("%s(%p,%lx)", __func__, addr, len);
+  mtwriteavar("thread:%x", myproc()->pid);
+  for (uptr i = PGROUNDDOWN(addr); i < PGROUNDUP(addr + len); i += PGSIZE)
+    mtwriteavar("page:%016x", i);
+#endif
  vmnode *vmn = new vmnode(PGROUNDUP(len) / PGSIZE);
  if (vmn == 0)
    return -1;
@@ -104,6 +112,13 @@ sys_unmap(uptr addr, u64 len)
 {
  ANON_REGION(__func__, &perfgroup);
+#if MTRACE
+  mt_ascope ascope("%s(%p,%lx)", __func__, addr, len);
+  mtwriteavar("thread:%x", myproc()->pid);
+  for (uptr i = PGROUNDDOWN(addr); i < PGROUNDUP(addr + len); i += PGSIZE)
+    mtwriteavar("page:%016x", i);
+#endif
  uptr align_addr = PGROUNDDOWN(addr);
  uptr align_len = PGROUNDUP(addr + len) - align_addr;
  if (myproc()->vmap->remove(align_addr, align_len) < 0)
@@ -131,3 +146,9 @@ sys_setfs(u64 base)
  switchvm(p);
  return 0;
 }
+long
+sys_setaffinity(int cpu)
+{
+  return myproc()->set_cpu_pin(cpu);
+}
--- a/kernel/trap.cc
+++ b/kernel/trap.cc
@@ -158,6 +158,7 @@ trap(struct trapframe *tf)
 #endif
        return;
      }
+      cprintf("pagefault: failed\n");
      cli();
    }

--- a/kernel/user.cc
+++ b/kernel/user.cc
+#include "types.h"
+#include "kernel.hh"
+#include "mmu.h"
+#include "amd64.h"
+#include "spinlock.h"
+#include "condvar.h"
+#include "queue.h"
+#include "proc.hh"
+#include "cpu.hh"
+#include "bits.hh"
+#include "vm.hh"
+extern struct proc *bootproc;
+// Set up first user process.
+void
+inituser(void)
+{
+  struct proc *p;
+  extern u8 _initcode_start[];
+  extern u64 _initcode_size;
+  p = proc::alloc();
+  p->ftable = new filetable();
+  if (p->ftable == nullptr)
+    panic("userinit: new filetable");
+  bootproc = p;
+  if((p->vmap = vmap::alloc()) == 0)
+    panic("userinit: out of vmaps?");
+  vmnode *vmn =  new vmnode(PGROUNDUP(_initcode_size) / PGSIZE);
+  if(vmn == 0)
+    panic("userinit: vmn_allocpg");
+  if(p->vmap->insert(vmn, 0, 1) < 0)
+    panic("userinit: vmap_insert");
+  if(p->vmap->copyout(0, _initcode_start, _initcode_size) < 0)
+    panic("userinit: copyout");
+  memset(p->tf, 0, sizeof(*p->tf));
+  p->tf->cs = UCSEG | 0x3;
+  p->tf->ds = UDSEG | 0x3;
+  p->tf->ss = p->tf->ds;
+  p->tf->rflags = FL_IF;
+  p->tf->rsp = PGSIZE;
+  p->tf->rip = 0x0;  // beginning of initcode.S
+  safestrcpy(p->name, "initcode", sizeof(p->name));
+  p->cwd = 0; // forkret will fix in the process's context
+  acquire(&p->lock);
+  addrun(p);
+  release(&p->lock);
+}
--- a/kernel/uwq.cc
+++ b/kernel/uwq.cc
+#include "types.h"
+#include "amd64.h"
+#include "kernel.hh"
+#include "cpu.hh"
+#include "gc.hh"
+#include "percpu.hh"
+#include "spinlock.h"
+#include "condvar.h"
+#include "proc.hh"
+#include "uwq.hh"
+#include "vm.hh"
+#include "kalloc.hh"
+#include "bits.hh"
+extern "C" {
+#include "kern_c.h"
+}
+bool
+uwq_trywork(void)
+{
+  // Returning true means uwq added a thread to the run queue
+  u64 i, k;
+  // A "random" victim CPU
+  k = rdtsc();
+  for (i = 0; i < NCPU; i++) {
+    u64 j = (i+k) % NCPU;
+    if (j == mycpuid())
+      continue;
+    struct cpu *c = &cpus[j];
+    // The gc_epoch is for p and uwq
+    scoped_gc_epoch xgc();
+    barrier();
+    struct proc *p = c->proc;
+    if (p == nullptr || p->uwq == nullptr)
+      continue;
+    uwq* uwq = p->uwq;
+    if (uwq->haswork()) {
+      if (uwq->tryworker())
+        return true;
+      break;
+    }
+  }
+  return false;
+}
+long
+sys_wqwait(void)
+{
+  uwq_worker* w = myproc()->worker;
+  if (w == nullptr)
+    return -1;
+  return w->wait();
+}
+//
+// uwq_worker
+//
+uwq_worker::uwq_worker(uwq* u, proc* p)
+  : uwq_(u), proc_(p), running_(false)
+{
+  initlock(&lock_, "worker_lock", 0);
+  initcondvar(&cv_, "worker_cv");
+}
+void
+uwq_worker::exit(void)
+{
+  if (--uwq_->uref_ == 0)
+    gc_delayed(uwq_);
+  release(&lock_);
+  delete this;
+  ::exit();
+}
+long
+uwq_worker::wait(void)
+{
+  acquire(&lock_);
+  if (uwq_->ref() == 0)
+    this->exit();
+  running_ = false;
+  cv_sleep(&cv_, &lock_);
+  if (uwq_->ref() == 0)
+    this->exit();
+  release(&lock_);
+  return 0;
+}
+//
+// uwq
+//
+uwq*
+uwq::alloc(vmap* vmap, filetable *ftable)
+{
+  padded_length* len;
+  uwq* u;
+  len = (padded_length*) ksalloc(slab_userwq);  
+  if (len == nullptr)
+    return nullptr;
+  ftable->incref();
+  vmap->incref();
+  u = new uwq(vmap, ftable, len);
+  if (u == nullptr) {
+    ftable->decref();
+    vmap->decref();
+    ksfree(slab_userwq, len);
+    return nullptr;
+  }
+  u->inc();
+  if (mapkva(vmap->pml4, (char*)len, USERWQ, USERWQSIZE)) {
+    ftable->decref();
+    vmap->decref();
+    ksfree(slab_userwq, len);
+    u->dec();
+    return nullptr;
+  }
+  return u;
+}
+uwq::uwq(vmap* vmap, filetable *ftable, padded_length *len) 
+  : rcu_freed("uwq"),
+    vmap_(vmap), ftable_(ftable), len_(len),
+    uentry_(0), ustack_(UWQSTACK), uref_(0)
+{
+  for (int i = 0; i < NCPU; i++)
+    len_[i].v_ = 0;
+  initlock(&lock_, "uwq_lock", 0);
+  memset(worker_, 0, sizeof(worker_));
+}
+uwq::~uwq(void)
+{ 
+  if (len_ != nullptr)
+    ksfree(slab_userwq, len_);
+  vmap_->decref();
+  ftable_->decref();
+}
+bool
+uwq::haswork(void) const
+{
+  if (len_ == nullptr)
+    return false;
+  for (int i = 0; i < NCPU; i++) {
+    if (len_[i].v_ > 0) {
+      return true;
+    }
+  }
+  return false;
+}
+bool
+uwq::tryworker(void)
+{
+  // Try to start a worker thread
+  scoped_acquire lock0(&lock_);
+  if (ref() == 0)
+    return false;
+  int slot = -1;
+  for (int i = 0; i < NWORKERS; i++) {
+    if (worker_[i] == nullptr) {
+      if (slot == -1)
+        slot = i;
+      continue;
+    }
+    uwq_worker *w = worker_[i];
+    if (w->running_)
+      continue;
+    else {
+      scoped_acquire lock1(&w->lock_);
+      proc* p = w->proc_;
+      acquire(&p->lock);
+      p->cpuid = mycpuid();
+      release(&p->lock);
+      w->running_ = true;
+      cv_wakeup(&w->cv_);
+      return true;
+    }
+  }
+  if (slot != -1) {
+    proc* p = allocworker();
+    if (p != nullptr) {
+      uwq_worker* w = new uwq_worker(this, p);
+      assert(w != nullptr);
+      ++uref_;
+      p->worker = w;
+      w->running_ = true;
+      acquire(&p->lock);
+      p->cpuid = mycpuid();
+      addrun(p);
+      release(&p->lock);
+      worker_[slot] = w;
+      return true;
+    }
+  }
+  return nullptr;
+}
+void
+uwq::finish(void)
+{
+  bool gcnow = true;
+  scoped_acquire lock0(&lock_);
+  for (int i = 0; i < NWORKERS; i++) {
+    if (worker_[i] != nullptr) {
+      uwq_worker* w = worker_[i];
+      gcnow = false;
+      acquire(&w->lock_);
+      cv_wakeup(&w->cv_);
+      release(&w->lock_);
+    }
+  }
+  if (gcnow)
+    gc_delayed(this);
+}
+void
+uwq::onzero() const
+{
+  uwq *u = (uwq*)this;
+  u->finish();
+}
+void
+uwq::setuentry(uptr uentry)
+{
+  uentry_ = uentry;
+}
+proc*
+uwq::allocworker(void)
+{
+  uptr uentry = uentry_;
+  if (uentry == 0)
+    return nullptr;
+  proc* p = proc::alloc();
+  if (p == nullptr)
+    return nullptr;
+  safestrcpy(p->name, "uwq_worker", sizeof(p->name));
+  // finishproc will drop these refs
+  vmap_->incref();
+  ftable_->incref();
+  p->vmap = vmap_;
+  p->ftable = ftable_;
+  struct vmnode *vmn;
+  if ((vmn = new vmnode(UWQSTACKPAGES)) == nullptr) {
+    finishproc(p);
+    return nullptr;
+  }
+  uptr stacktop = ustack_ + (UWQSTACKPAGES*PGSIZE);
+  if (vmap_->insert(vmn, ustack_, 1) < 0) {
+    delete vmn;
+    finishproc(p);
+    return nullptr;
+  }
+  ustack_ += (UWQSTACKPAGES*PGSIZE);
+  p->tf->rsp = stacktop - 8;
+  p->tf->rip = uentry;
+  p->tf->cs = UCSEG | 0x3;
+  p->tf->ds = UDSEG | 0x3;
+  p->tf->ss = p->tf->ds;
+  p->tf->rflags = FL_IF;
+  return p;
+}
--- a/kernel/vm.cc
+++ b/kernel/vm.cc
@@ -14,6 +14,8 @@
 #include "crange.hh"
 #include "cpputil.hh"
 #include "sperf.hh"
+#include "uwq.hh"
+#include "kmtrace.hh"
 enum { vm_debug = 0 };
@@ -22,7 +24,7 @@ enum { vm_debug = 0 };
 */
 vmnode::vmnode(u64 npg, vmntype ntype, inode *i, u64 off, u64 s)
-  : npages(npg), ref(0), type(ntype), ip(i), offset(off), sz(s)
+  : npages(npg), type(ntype), ip(i), offset(off), sz(s), ref_(0)
 {
  if (npg > NELEM(page))
    panic("vmnode too big\n");
@@ -43,12 +45,24 @@ vmnode::~vmnode()
 }
 void
-vmnode::decref()
+vmnode::decref(void)
 {
-  if(--ref == 0)
+  if(--ref_ == 0)
    delete this;
 }
+void
+vmnode::incref(void)
+{
+  ++ref_;
+}
+u64
+vmnode::ref(void)
+{
+  return ref_.load();
+}
 int
 vmnode::allocpg()
 {
@@ -56,7 +70,7 @@ vmnode::allocpg()
    if (page[i])
      continue;
-    char *p = kalloc();
+    char *p = kalloc("(vmnode::allocpg)");
    if (!p) {
      cprintf("allocpg: out of memory, leaving half-filled vmnode\n");
      return -1;
@@ -125,7 +139,7 @@ vma::vma(vmap *vmap, uptr start, uptr end, enum vmatype vtype, vmnode *vmn) :
    vma_start(start), vma_end(end), va_type(vtype), n(vmn)
 {
  if (n)
-    n->ref++;
+    n->incref();
 }
 vma::~vma()
@@ -144,15 +158,15 @@ vmap::alloc(void)
  return new vmap();
 }
-vmap::vmap() :
+vmap::vmap() : 
 #if VM_CRANGE
-    cr(10),
+  cr(10),
 #endif
 #if VM_RADIX
-    rx(PGSHIFT),
+  rx(PGSHIFT),
 #endif
-    ref(1), pml4(setupkvm()), kshared((char*) ksalloc(slab_kshared)),
+  ref(1), pml4(setupkvm()), kshared((char*) ksalloc(slab_kshared)),
-    brk_(0)
+  brk_(0)
 {
  initlock(&brklock_, "brk_lock", LOCKSTAT_VM);
  if (pml4 == 0) {
@@ -165,8 +179,8 @@ vmap::vmap() :
    goto err;
  }
-  if (setupkshared(pml4, kshared)) {
+  if (mapkva(pml4, kshared, KSHARED, KSHAREDSIZE)) {
-    cprintf("vmap::vmap: setupkshared out of memory\n");
+    cprintf("vmap::vmap: mapkva out of memory\n");
    goto err;
  }
@@ -195,6 +209,12 @@ vmap::decref()
    delete this;
 }
+void
+vmap::incref()
+{
+  ++ref;
+}
 bool
 vmap::replace_vma(vma *a, vma *b)
 {
@@ -517,7 +537,7 @@ vmap::pagefault(uptr va, u32 err)
  u64 npg = (PGROUNDDOWN(va) - m->vma_start) / PGSIZE;
  if (vm_debug)
    cprintf("pagefault: err 0x%x va 0x%lx type %d ref %lu pid %d\n",
-            err, va, m->va_type, m->n->ref.load(), myproc()->pid);
+            err, va, m->va_type, m->n->ref(), myproc()->pid);
  if (m->n && !m->n->page[npg])
    if (m->n->allocpg() < 0)
@@ -546,7 +566,6 @@ vmap::pagefault(uptr va, u32 err)
  if (m->va_type == COW) {
    *pte = v2p(m->n->page[npg]) | PTE_P | PTE_U | PTE_COW;
  } else {
-    assert(m->n->ref == 1);
    *pte = v2p(m->n->page[npg]) | PTE_P | PTE_U | PTE_W;
  }
@@ -556,6 +575,12 @@ vmap::pagefault(uptr va, u32 err)
 int
 pagefault(struct vmap *vmap, uptr va, u32 err)
 {
+#if MTRACE
+  mt_ascope ascope("%s(%p)", __func__, va);
+  mtwriteavar("thread:%x", myproc()->pid);
+  mtwriteavar("page:%016x", PGROUNDDOWN(va));
+#endif
  return vmap->pagefault(va, err);
 }

--- a/lib/Makefrag
+++ b/lib/Makefrag
-$(O)/lib/%.o: CFLAGS:=$(CFLAGS)
+$(O)/lib/%.o: CFLAGS:=$(CFLAGS) -DXV6_USER
-$(O)/lib/%.o: CXXFLAGS:=$(CXXFLAGS)
+$(O)/lib/%.o: CXXFLAGS:=$(CXXFLAGS) -DXV6_USER
 ULIB = ulib.o usys.o printf.o umalloc.o uthread.o fmt.o stream.o ipc.o \
       threads.o crt.o wq.o perf.o

--- a/lib/threads.cc
+++ b/lib/threads.cc
@@ -55,7 +55,7 @@ pthread_getspecific(pthread_key_t key)
 int
 pthread_setspecific(pthread_key_t key, void* value)
 {
-  __asm volatile("movq %0, %%fs:(%1)" : : "r" (value), "r" ((u64) key * 8));
+  __asm volatile("movq %0, %%fs:(%1)" : : "r" (value), "r" ((u64) key * 8) : "memory");
  return 0;
 }

--- a/lib/ulib.c
+++ b/lib/ulib.c
@@ -151,6 +151,12 @@ open(const char *path, int omode)
  return openat(AT_FDCWD, path, omode);
 }
+int
+mkdir(const char *path)
+{
+  return mkdirat(AT_FDCWD, path);
+}
 extern void __cxa_pure_virtual(void);
 void __cxa_pure_virtual(void)
 { 

--- a/lib/usys.S
+++ b/lib/usys.S
@@ -30,7 +30,7 @@ SYSCALL(mknod)
 SYSCALL(unlink)
 SYSCALL(fstat)
 SYSCALL(link)
-SYSCALL(mkdir)
+SYSCALL(mkdirat)
 SYSCALL(chdir)
 SYSCALL(dup)
 SYSCALL(getpid)
@@ -48,3 +48,5 @@ SYSCALL(pread)
 SYSCALL(async)
 SYSCALL(script)
 SYSCALL(setfs)
+SYSCALL(wqwait)
+SYSCALL(setaffinity)
--- a/lib/wq.cc
+++ b/lib/wq.cc
@@ -21,6 +21,8 @@ public:
 private:
  work *steal(int c);
  work *pop(int c);
+  void inclen(int c);
+  void declen(int c);
  struct wqueue {
    work *w[NSLOTS];
@@ -38,6 +40,10 @@ private:
  percpu<wqueue> q_;
  percpu<stat> stat_;
+#if defined(XV6_USER)
+  padded_length* len_;
+#endif
 };
 static wq *wq_;
@@ -95,6 +101,10 @@ wq::wq(void)
  for (i = 0; i < NCPU; i++)
    wqlock_init(&q_[i].lock);
+#if defined(XV6_USER)
+  len_ = allocklen(NCPU*sizeof(padded_length));
+#endif
 }
 void
@@ -107,6 +117,22 @@ wq::dump(void)
            stat_[i].pop, stat_[i].steal);
 }
+inline void
+wq::inclen(int c)
+{
+#if defined(XV6_USER)
+  __sync_fetch_and_add(&len_[c].v_, 1);
+#endif
+}
+inline void
+wq::declen(int c)
+{
+#if defined(XV6_USER)
+  __sync_fetch_and_sub(&len_[c].v_, 1);
+#endif
+}
 int
 wq::push(work *w)
 {
@@ -123,6 +149,7 @@ wq::push(work *w)
  q_->w[i] = w;
  barrier();
  q_->head++;
+  inclen(mycpuid());
  stat_->push++;
  popcli();
  return 0;
@@ -148,6 +175,7 @@ wq::pop(int c)
  i = (i-1) & (NSLOTS-1);
  w = q->w[i];
  q->head--;
+  declen(c);
  wqlock_release(&q->lock);
  stat_->pop++;
@@ -171,6 +199,7 @@ wq::steal(int c)
  i = i & (NSLOTS-1);
  w = q->w[i];
  q->tail++;
+  declen(c);
  wqlock_release(&q->lock);
  stat_->steal++;

--- a/net/sys_arch.cc
+++ b/net/sys_arch.cc
@@ -208,7 +208,7 @@ sys_thread_new(const char *name, lwip_thread_fn thread, void *arg,
  struct lwip_thread *lt;
  struct proc *p;
-  lt = (struct lwip_thread*) kmalloc(sizeof(*lt));
+  lt = (struct lwip_thread*) kmalloc(sizeof(*lt), "lwip_thread");
  if (lt == nullptr)
    return 0;
  lt->thread = thread;

--- a/param.h
+++ b/param.h
@@ -23,6 +23,8 @@
 #define VERIFYFREE    0  // Unreliable, e.g. vma's vmnode pointer gets reused
 #define ALLOC_MEMSET  DEBUG
 #define KSHAREDSIZE   (32 << 10)
+#define USERWQSIZE    (1 << 14)
+#define UWQSTACKPAGES 2
 #define WQSHIFT       7
 #define CILKENABLE    0
 #if defined(HW_josmp)
@@ -30,7 +32,7 @@
 #define MTRACE       0
 #define PERFSIZE     (1<<20ull)
 #elif defined(HW_qemu)
-#define NCPU         4   // maximum number of CPUs
+#define NCPU         8   // maximum number of CPUs
 #define MTRACE       0
 #define PERFSIZE     (16<<20ull)
 #elif defined(HW_ud0)