Merge branch 'scale-amd64' of git+ssh://pdos.csail.mit.edu/home/am0/6.828/xv6 into scale-amd64

ff1eef00 · Nickolai Zeldovich · bfdbe916 · 275a8c9f · ff1eef00 · ff1eef00
--- a/include/kernel.hh
+++ b/include/kernel.hh
@@ -126,7 +126,8 @@ void            ideintr(void);
 void            iderw(struct buf*);
 // idle.cc
-extern struct proc *idlep[NCPU];
+struct proc *   idleproc(void);
+void            idlezombie(struct proc*);
 // ioapic.c
 void            ioapicenable(int irq, int cpu);
@@ -179,6 +180,7 @@ int             pipewrite(struct pipe*, char*, int);
 // proc.c
 struct proc*    allocproc(void);
 struct proc*    copyproc(struct proc*);
+void            finishproc(struct proc*);
 void            exit(void);
 int             fork(int);
 int             growproc(int);
@@ -190,7 +192,7 @@ void            userinit(void);
 int             wait(void);
 void            yield(void);
 struct proc*    threadalloc(void (*fn)(void*), void *arg);
-void            threadpin(void (*fn)(void*), void *arg, const char *name, int cpu);
+struct proc*    threadpin(void (*fn)(void*), void *arg, const char *name, int cpu);
 // prof.c
 extern int profenable;
@@ -262,7 +264,7 @@ void            freework(struct work *w);
 // cilk.c
 #if CILKENABLE
-void            cilk_push(void *rip, u64 arg0, u64 arg1);
+void            cilk_push(void (*fn)(uptr, uptr), u64 arg0, u64 arg1);
 void            cilk_start(void);
 void            cilk_end(void);
 void            cilk_dump(void);

--- a/include/lockstat.h
+++ b/include/lockstat.h
@@ -42,11 +42,13 @@ struct klockstat;
 // Debug knobs
 #define LOCKSTAT_BIO       1
+#define LOCKSTAT_CILK      1
 #define LOCKSTAT_CONDVAR   0
 #define LOCKSTAT_CONSOLE   1
 #define LOCKSTAT_CRANGE    1
 #define LOCKSTAT_FS        1
 #define LOCKSTAT_GC        1
+#define LOCKSTAT_IDLE      1
 #define LOCKSTAT_KALLOC    1
 #define LOCKSTAT_KMALLOC   1
 #define LOCKSTAT_NET       1

--- a/include/percpu.hh
+++ b/include/percpu.hh
+template <typename T>
+struct percpu {
+  int myid() {
+    return mycpu()->id;
+  }
+  const T* operator->() const {
+    return cpu(myid());
+  }
+  T* operator->() {
+    return cpu(myid());
+  }
+  T& operator*() {
+    return *cpu(myid());
+  }
+  T& operator[](int id) { 
+    return *cpu(id);
+  }
+  const T& operator[](int id) const { 
+    return *cpu(id);
+  }
+  T* cpu(int id) {
+    return &pad_[id].v_;
+  }
+  struct {
+    T v_ __mpalign__;
+    __padout__;
+  } pad_[NCPU];
+};
--- a/kernel/async.cc
+++ b/kernel/async.cc
@@ -92,3 +92,4 @@ sys_async(int fd, size_t count, off_t off,
  msg->submitted = 1;
  return 0;
 }
--- a/kernel/cilk.cc
+++ b/kernel/cilk.cc
@@ -36,6 +36,7 @@
 #include "queue.h"
 #include "proc.hh"
 #include "mtrace.h"
+#include "wq.hh"
 #define NSLOTS (1 << CILKSHIFT)
@@ -59,7 +60,6 @@ struct cilkthread {
 struct cilkstat {
  u64 push;
  u64 full;
-  u64 pop;
  u64 steal;
  __padout__;
 } __mpalign__;
@@ -67,12 +67,6 @@ struct cilkstat {
 static struct cilkqueue queue[NCPU] __mpalign__;
 static struct cilkstat stat[NCPU] __mpalign__;
-static struct cilkqueue *
-cilk_cur(void)
-{
-  return &queue[mycpu()->id];
-}
 static struct cilkframe *
 cilk_frame(void)
 {
@@ -85,73 +79,20 @@ cilk_stat(void)
  return &stat[mycpu()->id];
 }
-static int
-__cilk_push(struct cilkqueue *q, struct cilkthread *t)
-{
-  int i;
-  i = q->head;
-  if ((i - q->tail) == NSLOTS) {
-    cilk_stat()->full++;
-    return -1;
-  }
-  i = i & (NSLOTS-1);
-  q->thread[i] = t;
-  q->head++;
-  cilk_stat()->push++;
-  return 0;
-}
-static struct cilkthread *
-__cilk_pop(struct cilkqueue *q)
-{
-  int i;
-  acquire(&q->lock);
-  i = q->head;
-  if ((i - q->tail) == 0) {
-    release(&q->lock);
-    return 0;
-  }
-  i = (i-1) & (NSLOTS-1);
-  q->head--;
-  release(&q->lock);
-  cilk_stat()->pop++;
-  return q->thread[i];
-}
-static struct cilkthread *
-__cilk_steal(struct cilkqueue *q)
-{
-  int i;
-  acquire(&q->lock);
-  i = q->tail;
-  if ((i - q->head) == 0) {
-    release(&q->lock);
-    return 0;
-  }
-  i = i & (NSLOTS-1);
-  q->tail++;
-  release(&q->lock);
-  cilk_stat()->steal++;
-  return q->thread[i];
-}
 static void
-__cilk_run(struct cilkthread *th)
+__cilk_run(struct work *w, void *xfn, void *arg0, void *arg1, void *xframe)
 {
-  void (*fn)(uptr arg0, uptr arg1) = (void(*)(uptr,uptr))th->rip;
+  void (*fn)(uptr arg0, uptr arg1) = (void(*)(uptr,uptr))xfn;
+  struct cilkframe *frame = (struct cilkframe *)xframe;
  struct cilkframe *old = mycpu()->cilkframe;
-  mycpu()->cilkframe = th->frame;
+  if (old != frame)
-  fn(th->arg0, th->arg1);
+    cilk_stat()->steal++;
+  mycpu()->cilkframe = frame;
+  fn((uptr)arg0, (uptr)arg1);
  mycpu()->cilkframe = old;
-  th->frame->ref--;
+  frame->ref--;
-  kfree(th);
 }
 // Add the (rip, arg0, arg1) work to the local work queue.
@@ -160,56 +101,27 @@ __cilk_run(struct cilkthread *th)
 void
 cilk_push(void (*fn)(uptr, uptr), u64 arg0, u64 arg1)
 {
-  struct cilkthread *th;
+  struct work *w;
-  th = (struct cilkthread *) kalloc();
+  w = allocwork();
-  if (th == nullptr) {
+  if (w == nullptr) {
    fn(arg0, arg1);
    return;
  }
-  th->rip = (uptr) fn;
+  w->rip = (void*)__cilk_run;
-  th->arg0 = arg0;
+  w->arg0 = (void*)fn;
-  th->arg1 = arg1;
+  w->arg1 = (void*)arg0;
-  th->frame = cilk_frame();
+  w->arg2 = (void*)arg1;
+  w->arg3 = (void*)cilk_frame();
-  if (__cilk_push(cilk_cur(), th)) {
-    kfree(th);
+  if (wq_push(w)) {
+    freework(w);
    fn(arg0, arg1);
-  } else
+    cilk_stat()->full++;
+  } else {
    cilk_frame()->ref++;
-}
+    cilk_stat()->push++;
-// Try to execute one cilkthread.
-// Check local queue then steal from other queues. 
-int
-cilk_trywork(void)
-{
-  struct cilkthread *th;
-  int i;
-  pushcli();
-  th = __cilk_pop(cilk_cur());
-  if (th != nullptr) {
-    __cilk_run(th);
-    popcli();
-    return 1;
-  }
-  // XXX(sbw) should be random
-  for (i = 0; i < NCPU; i++) {
-    if (i == mycpu()->id)
-        continue;
-    th = __cilk_steal(&queue[i]);
-    if (th != nullptr) {
-      __cilk_run(th);
-      popcli();
-      return 1;
-    }
  }
-  popcli();
-  return 0;
 }
 // Start a new work queue frame.
@@ -229,21 +141,9 @@ cilk_start(void)
 void
 cilk_end(void)
 {
-  while (cilk_frame()->ref != 0) {
+  while (cilk_frame()->ref != 0)
-    struct cilkthread *th;
+    wq_trywork();
-    int i;
-    while ((th = __cilk_pop(cilk_cur())) != nullptr)
-      __cilk_run(th);
-    for (i = 0; i < NCPU; i++) {
-      th = __cilk_steal(&queue[i]);
-      if (th != nullptr) {
-        __cilk_run(th);
-        break;
-      }
-    }
-  }
  mycpu()->cilkframe = 0;
  popcli();
 }
@@ -253,8 +153,8 @@ cilk_dump(void)
 {
  int i;
  for (i = 0; i < NCPU; i++)
-    cprintf("push %lu full %lu pop %lu steal %lu\n",
+    cprintf("push %lu full %lu steal %lu\n",
-            stat[i].push, stat[i].full, stat[i].pop, stat[i].steal);
+            stat[i].push, stat[i].full, stat[i].steal);
 }
 static void
@@ -285,7 +185,7 @@ testcilk(void)
    running = 0;
  } else {
    while (running)
-      cilk_trywork();
+      wq_trywork();
  }
  popcli();
 }

--- a/kernel/idle.cc
+++ b/kernel/idle.cc
@@ -6,8 +6,75 @@
 #include "proc.hh"
 #include "cpu.hh"
 #include "sched.hh"
+#include "percpu.hh"
-struct proc *idlep[NCPU] __mpalign__;
+struct idle {
+  struct proc *cur;
+  struct proc *heir;
+  SLIST_HEAD(zombies, proc) zombies;
+  struct spinlock lock;
+};
+static percpu<idle> idlem;
+void idleloop(void);
+struct proc *
+idleproc(void)
+{
+  assert(mycpu()->ncli > 0);
+  return idlem->cur;
+}
+void
+idlezombie(struct proc *p)
+{
+  acquire(&idlem[mycpu()->id].lock);
+  SLIST_INSERT_HEAD(&idlem[mycpu()->id].zombies, p, child_next);
+  release(&idlem[mycpu()->id].lock);
+}
+void
+idlebequeath(void)
+{
+  // Only the current idle thread may call this function
+  assert(mycpu()->ncli > 0);
+  assert(myproc() == idlem->cur);
+  assert(idlem->heir != nullptr);
+  idlem->cur = idlem->heir;
+  acquire(&idlem->heir->lock);
+  idlem->heir->set_state(RUNNABLE);
+  release(&idlem->heir->lock);
+}
+static void
+idleheir(void *x)
+{
+  post_swtch();
+  idlem->heir = nullptr;
+  idleloop();
+}
+static inline void
+finishzombies(void)
+{
+  struct idle *i = &idlem[mycpu()->id];
+  if (!SLIST_EMPTY(&i->zombies)) {
+    struct proc *p, *np;
+    acquire(&i->lock);
+    SLIST_FOREACH_SAFE(p, &i->zombies, child_next, np) {
+      SLIST_REMOVE(&i->zombies, p, proc, child_next);
+      finishproc(p);
+    }
+    release(&i->lock);
+  }
+}
 void
 idleloop(void)
@@ -29,11 +96,31 @@ idleloop(void)
    myproc()->set_state(RUNNABLE);
    sched();
+    finishzombies();
    if (steal() == 0) {
      int worked;
      do {
        assert(mycpu()->ncli == 0);
+        // If we don't have an heir, try to allocate one
+        if (idlem->heir == nullptr) {
+          struct proc *p;
+          p = allocproc();
+          if (p == nullptr)
+            break;
+          snprintf(p->name, sizeof(p->name), "idleh_%u", mycpu()->id);          
+          p->cpuid = mycpu()->id;
+          p->cpu_pin = 1;
+          p->context->rip = (u64)idleheir;
+          p->cwd = nullptr;
+          idlem->heir = p;
+        }
        worked = wq_trywork();
+        // If we are no longer the idle thread, exit
+        if (worked && idlem->cur != myproc())
+          exit();
      } while(worked);
      sti();
    }
@@ -43,13 +130,16 @@ idleloop(void)
 void
 initidle(void)
 {
-  // allocate a fake PID for each scheduler thread
  struct proc *p = allocproc();
  if (!p)
    panic("initidle allocproc");
+  SLIST_INIT(&idlem[cpunum()].zombies);
+  initlock(&idlem[cpunum()].lock, "idle_lock", LOCKSTAT_IDLE);
  snprintf(p->name, sizeof(p->name), "idle_%u", cpunum());
  mycpu()->proc = p;
+  myproc()->cpuid = cpunum();
  myproc()->cpu_pin = 1;
-  idlep[cpunum()] = p;
+  idlem->cur = p;
 }
--- a/kernel/main.cc
+++ b/kernel/main.cc
@@ -83,6 +83,7 @@ cmain(u64 mbmagic, u64 mbaddr)
  initkalloc(mbaddr);
  initproc();      // process table
  initsched();     // scheduler run queues
+  initidle();
  initgc();        // gc epochs and threads
  initbio();       // buffer cache
  initinode();     // inode cache
@@ -95,7 +96,6 @@ cmain(u64 mbmagic, u64 mbaddr)
  initlockstat();
  initpci();
  initnet();
-  initidle();
  if (VERBOSE)
    cprintf("ncpu %d %lu MHz\n", ncpu, cpuhz / 1000000);

--- a/kernel/proc.cc
+++ b/kernel/proc.cc
@@ -152,9 +152,13 @@ exit(void)
  // Parent might be sleeping in wait().
  acquire(&(myproc()->lock));
  // Kernel threads might not have a parent
  if (myproc()->parent != nullptr)
-      cv_wakeup(&(myproc()->parent->cv));
+    cv_wakeup(&(myproc()->parent->cv));
+  else
+    idlezombie(myproc());
  if (wakeupinit)
    cv_wakeup(&bootproc->cv); 
@@ -458,6 +462,21 @@ fork(int flags)
  return pid;
 }
+void
+finishproc(struct proc *p)
+{
+  ksfree(slab_stack, p->kstack);
+  p->kstack = 0;
+  p->vmap->decref();
+  if (!xnspid->remove(p->pid, &p))
+    panic("wait: ns_remove");
+  p->pid = 0;
+  p->parent = 0;
+  p->name[0] = 0;
+  p->killed = 0;
+  freeproc(p);
+}
 // Wait for a child process to exit and return its pid.
 // Return -1 if this process has no children.
 int
@@ -478,16 +497,7 @@ wait(void)
 	  pid = p->pid;
 	  SLIST_REMOVE(&myproc()->childq, p, proc, child_next);
 	  release(&myproc()->lock);
-	  ksfree(slab_stack, p->kstack);
+          finishproc(p);
-	  p->kstack = 0;
-	  p->vmap->decref();
-	  if (!xnspid->remove(p->pid, &p))
-	    panic("wait: ns_remove");
-	  p->pid = 0;
-	  p->parent = 0;
-	  p->name[0] = 0;
-	  p->killed = 0;
-          freeproc(p);
 	  return pid;
 	}
 	release(&p->lock);
@@ -533,12 +543,12 @@ threadalloc(void (*fn)(void *), void *arg)
  p->context->rip = (u64)threadstub;
  p->context->r12 = (u64)fn;
  p->context->r13 = (u64)arg;
-  p->parent = myproc();
+  p->parent = nullptr;
-  p->cwd = 0;
+  p->cwd = nullptr;
  return p;
 }
-void
+struct proc*
 threadpin(void (*fn)(void*), void *arg, const char *name, int cpu)
 {
  struct proc *p;
@@ -553,4 +563,5 @@ threadpin(void (*fn)(void*), void *arg, const char *name, int cpu)
  acquire(&p->lock);
  addrun(p);
  release(&p->lock);
+  return p;
 }
--- a/kernel/sched.cc
+++ b/kernel/sched.cc
@@ -26,7 +26,7 @@ void
 post_swtch(void)
 {
  if (mycpu()->prev->get_state() == RUNNABLE && 
-      mycpu()->prev != idlep[mycpu()->id])
+      mycpu()->prev != idleproc())
    addrun(mycpu()->prev);
  release(&mycpu()->prev->lock);
 }
@@ -42,6 +42,11 @@ sched(void)
  if(!holding(&myproc()->lock))
    panic("sched proc->lock");
 #endif
+  if (myproc() == idleproc() && 
+      myproc()->get_state() != RUNNABLE) {
+    extern void idlebequeath(void);
+    idlebequeath();
+  }
  if(mycpu()->ncli != 1)
    panic("sched locks");
  if(myproc()->get_state() == RUNNING)
@@ -59,7 +64,7 @@ sched(void)
  struct proc *next = schednext();
  if (next == nullptr) {
    if (myproc()->get_state() != RUNNABLE) {
-      next = idlep[mycpu()->id];
+      next = idleproc();
    } else {
      myproc()->set_state(RUNNING);
      mycpu()->intena = intena;

--- a/kernel/wq.cc
+++ b/kernel/wq.cc
@@ -175,11 +175,11 @@ wq_trywork(void)
  struct work *w;
  int i;
-  pushcli();
+  //pushcli();
  w = __wq_pop(mycpu()->id);
  if (w != nullptr) {
    __wq_run(w);
-    popcli();
+    //popcli();
    return 1;
  }
  // XXX(sbw) should be random
@@ -190,12 +190,12 @@ wq_trywork(void)
    w = __wq_steal(i);
    if (w != nullptr) {
      __wq_run(w);
-      popcli();
+      //popcli();
      return 1;
    }
  }
-  popcli();
+  //popcli();
  return 0;
 }

--- a/param.h
+++ b/param.h
@@ -24,6 +24,7 @@
 #define ALLOC_MEMSET  DEBUG
 #define KSHAREDSIZE   (32 << 10)
 #define WQSHIFT       7
+#define CILKENABLE    0
 #if defined(HW_josmp)
 #define NCPU         16  // maximum number of CPUs
 #define MTRACE       0