A sys_futex implementation

25fa822b · Silas Boyd-Wickizer · d13a7df9 · 25fa822b · 25fa822b · 25fa822b
--- a/bin/Makefrag
+++ b/bin/Makefrag
@@ -36,7 +36,8 @@ UPROGS= \
 	xdu \
 	wqtest \
 	rm \
-	avar
+	avar \
+        futexbench

 ifeq ($(HAVE_LWIP),y)
 UPROGS += \

--- a/bin/futexbench.cc
+++ b/bin/futexbench.cc
+#include "types.h"
+#include "user.h"
+#include "amd64.h"
+#include "pthread.h"
+#include "futex.h"
+#include "errno.h"
+
+static volatile u64 ftx;
+static int iters;
+
+static void*
+worker(void *arg)
+{
+  long r;
+
+  for (int i = 0; i < iters; i++) {
+    r = futex((u64*)&ftx, FUTEX_WAIT, 0, 0);
+    if (r < 0 && r != -EWOULDBLOCK)
+      die("FUTEX_WAIT: %d", r);
+    ftx = 0;
+    r = futex((u64*)&ftx, FUTEX_WAKE, 1, 0);    
+  }
+  return nullptr;
+}
+
+int
+main(int ac, char** av)
+{
+  pthread_t th;
+  long r;
+
+  if (ac < 2)
+    die("usage: %s iters", av[0]);
+
+  iters = atoi(av[1]);
+
+  pthread_create(&th, 0, &worker, 0);
+  for (int i = 0; i < iters; i++) {
+    ftx = 1;
+    r = futex((u64*)&ftx, FUTEX_WAKE, 1, 0);
+    assert(r == 0);
+    r = futex((u64*)&ftx, FUTEX_WAIT, 1, 0);    
+    if (r < 0 && r != -EWOULDBLOCK)
+      die("FUTEX_WAIT: %d", r);
+  }
+
+  wait();
+  printf("done\n");
+  return 0;
+}
--- a/include/errno.h
+++ b/include/errno.h
+#define EAGAIN          11      /* Try again */
+#define EWOULDBLOCK     EAGAIN  /* Operation would block */
--- a/include/futex.h
+++ b/include/futex.h
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
--- a/include/kern_c.h
+++ b/include/kern_c.h
@@ -53,6 +53,7 @@ long sys_script(void *addr, u64 len, u64 chunk);
 long sys_setfs(u64 base);
 long sys_wqwait(void);
 long sys_setaffinity(int cpu);
+long sys_futex(const u64* addr, int op, u64 val, u64 timer);
 extern long (*syscalls[])(u64, u64, u64, u64, u64);

 // other exported/imported functions

--- a/include/kernel.hh
+++ b/include/kernel.hh
@@ -102,6 +102,12 @@ int             dirlink(struct inode*, const char*, u32);
 void            dir_init(struct inode *dp);
 void	        dir_flush(struct inode *dp);

+// futex.cc
+typedef u64* futexkey_t;
+int             futexkey(const u64* useraddr, vmap* vmap, futexkey_t* key);
+long            futexwait(futexkey_t key, u64 val, u64 timer);
+long            futexwake(futexkey_t key, u64 nwake);
+
 // hz.c
 void            microdelay(u64);
 void            inithz(void);

--- a/include/lockstat.h
+++ b/include/lockstat.h
@@ -47,6 +47,7 @@ struct klockstat;
 #define LOCKSTAT_CONSOLE   1
 #define LOCKSTAT_CRANGE    1
 #define LOCKSTAT_FS        1
+#define LOCKSTAT_FUTEX     1
 #define LOCKSTAT_GC        1
 #define LOCKSTAT_IDLE      1
 #define LOCKSTAT_KALLOC    1

--- a/include/proc.hh
+++ b/include/proc.hh
@@ -77,6 +77,7 @@ struct proc : public rcu_freed, public sched_link {
  u64 cv_wakeup;               // Wakeup time for this process
  LIST_ENTRY(proc) cv_waiters; // Linked list of processes waiting for oncv
  LIST_ENTRY(proc) cv_sleep;   // Linked list of processes sleeping on a cv
+  LIST_ENTRY(proc) futex_link;
  u64 user_fs_;
  u64 unmap_tlbreq_;
  int exec_cpuid_;

--- a/include/syscall.h
+++ b/include/syscall.h
@@ -33,4 +33,5 @@
 #define SYS_setfs  32
 #define SYS_wqwait 33
 #define SYS_setaffinity 34
-#define SYS_ncount 35   /* total number of system calls */
+#define SYS_futex 35
+#define SYS_ncount 36   /* total number of system calls */
--- a/include/user.h
+++ b/include/user.h
@@ -35,6 +35,7 @@ int async(int, size_t, off_t, u32, u32);
 int script(void *addr, u64 len, u64 chunk);
 int setfs(u64 base);
 int setaffinity(int cpu);
+long futex(const u64* addr, int op, u64 val, u64 timer);

 // ulib.c
 int stat(char*, struct stat*);

--- a/kernel/Makefrag
+++ b/kernel/Makefrag
@@ -12,6 +12,7 @@ OBJS = \
 	file.o \
 	fmt.o \
 	fs.o \
+        futex.o \
        idle.o \
 	ioapic.o \
 	lapic.o \

--- a/kernel/bootdata.c
+++ b/kernel/bootdata.c
@@ -63,4 +63,5 @@ long (*syscalls[])(u64, u64, u64, u64, u64) = {
  SYSCALL(setfs),
  SYSCALL(wqwait),
  SYSCALL(setaffinity),
+  SYSCALL(futex),
 };
--- a/kernel/futex.cc
+++ b/kernel/futex.cc
+#include "types.h"
+#include "kernel.hh"
+#include "spinlock.h"
+#include "cpputil.hh"
+#include "ns.hh"
+#include "errno.h"
+#include "condvar.h"
+#include "proc.hh"
+#include "cpu.hh"
+
+//
+// futexkey
+//
+// XXX(sbw) this uses kernel addresses as keys and probably
+// isn't complete.  For example, a user thread might wait
+// on a user address, then another thread might unmap that
+// address.
+
+typedef u64* futexkey_t;
+
+u64
+futexkey_hash(futexkey_t const& key)
+{
+  return (u64)key;
+}
+
+u64
+futexkey_val(futexkey_t const& key)
+{
+  return *key;
+}
+
+int
+futexkey(const u64* useraddr, vmap* vmap, futexkey_t* key)
+{
+  u64* kaddr;
+
+  kaddr = (u64*)pagelookup(vmap, (uptr)useraddr);
+  if (kaddr == nullptr) {
+    cprintf("futexkey: pagelookup failed\n");
+    return -1;
+  }
+  *key = kaddr;
+  return 0;
+}
+
+//
+// futexaddr
+//
+struct futexaddr : public referenced, public rcu_freed
+{
+  futexaddr(u64* kaddr);
+  virtual void do_gc();
+  virtual void onzero() const;
+
+  futexkey_t key_;
+  bool inserted_;
+  struct spinlock lock_;
+
+  LIST_HEAD(proclist, proc) list_;
+
+  NEW_DELETE_OPS(futexaddr);
+};
+
+xns<futexkey_t, futexaddr*, futexkey_hash> *nsfutex __mpalign__;
+
+futexaddr::futexaddr(futexkey_t key)
+  : rcu_freed("futexaddr"), key_(key), inserted_(false)
+{
+  initlock(&lock_, "futexaddr::lock_", LOCKSTAT_FUTEX);
+  LIST_INIT(&list_);
+}
+
+void
+futexaddr::do_gc(void)
+{
+  delete this;
+}
+
+void
+futexaddr::onzero(void) const
+{
+  if (inserted_)
+    assert(nsfutex->remove(key_, nullptr));
+  gc_delayed((futexaddr*)this);
+}
+
+long
+futexwait(futexkey_t key, u64 val, u64 timer)
+{
+  futexaddr* fa;
+
+  {
+    scoped_gc_epoch gc;
+  again:
+    fa = nsfutex->lookup(key);
+    if (fa == nullptr) {
+      fa = new futexaddr(key);
+      if (nsfutex->insert(key, fa) < 0) {
+        fa->dec();
+        goto again;
+      }
+      fa->inserted_ = true;
+    } else {
+      if (!fa->tryinc()) {
+        goto again;
+      }
+    }
+  }
+  assert(fa->key_ == key);
+
+  acquire(&fa->lock_);  
+  auto cleanup = scoped_cleanup([&fa](){
+    release(&fa->lock_);
+    fa->dec();
+  });
+
+  if (futexkey_val(fa->key_) != val)
+    return -EWOULDBLOCK;
+  LIST_INSERT_HEAD(&fa->list_, myproc(), futex_link);
+
+  u64 nsecto = timer == 0 ? 0 : timer+nsectime();
+  cv_sleepto(&myproc()->cv, &fa->lock_, nsecto);
+
+  LIST_REMOVE(myproc(), futex_link);
+  return 0;
+}
+
+long
+futexwake(u64* kaddr, u64 nwake)
+{
+  futexaddr* fa;
+  u64 nwoke = 0;
+  proc* p;
+
+  scoped_gc_epoch gc;
+  fa = nsfutex->lookup(kaddr);
+  if (fa == nullptr)
+    return 0;
+  acquire(&fa->lock_);
+  LIST_FOREACH(p, &fa->list_, futex_link) {
+    if (nwoke >= nwake)
+      break;
+    cv_wakeup(&p->cv);
+    nwoke++;
+  }
+  release(&fa->lock_);
+  return 0;
+}
+
+void
+initfutex(void)
+{
+  nsfutex = new xns<futexkey_t, futexaddr*, futexkey_hash>(false);
+  if (nsfutex == 0)
+    panic("initfutex");
+}
--- a/kernel/main.cc
+++ b/kernel/main.cc
@@ -38,6 +38,7 @@ void initwq(void);
 void initsperf(void);
 void initidle(void);
 void initcpprt(void);
+void initfutex(void);
 void idleloop(void);

 static volatile int bstate;
@@ -119,6 +120,7 @@ cmain(u64 mbmagic, u64 mbaddr)
  initdisk();      // disk
  initconsole();
  initwq();
+  initfutex();
  initsamp();
  initlockstat();
  initpci();

--- a/kernel/sysproc.cc
+++ b/kernel/sysproc.cc
@@ -10,6 +10,7 @@
 #include "vm.hh"
 #include "sperf.hh"
 #include "kmtrace.hh"
+#include "futex.h"

 long
 sys_fork(int flags)
@@ -158,3 +159,21 @@ sys_setaffinity(int cpu)
 {
  return myproc()->set_cpu_pin(cpu);
 }
+
+long
+sys_futex(const u64* addr, int op, u64 val, u64 timer)
+{
+  futexkey_t key;
+
+  if (futexkey(addr, myproc()->vmap, &key) < 0)
+    return -1;
+
+  switch(op) {
+  case FUTEX_WAIT:
+    return futexwait(key, val, timer);
+  case FUTEX_WAKE:
+    return futexwake(key, val);
+  default:
+    return -1;
+  }
+}
--- a/lib/usys.S
+++ b/lib/usys.S
@@ -50,3 +50,4 @@ SYSCALL(script)
 SYSCALL(setfs)
 SYSCALL(wqwait)
 SYSCALL(setaffinity)
+SYSCALL(futex)