avoid bucket locks; do locking per element

6d8e14b0 · Nickolai Zeldovich · 796f5c48 · 6d8e14b0 · 6d8e14b0
--- a/ns.c
+++ b/ns.c
@@ -15,11 +15,11 @@
 struct elem {
  int key;
  void *val;
+  int next_lock;
  struct elem * volatile next;
 };

 struct bucket {
-  struct spinlock l;
  struct elem * volatile chain;
 } __attribute__((aligned (CACHELINE)));

@@ -44,8 +44,6 @@ nsalloc(void)
  if (ns == 0)
    panic("nsalloc");
  memset(ns, 0, sizeof(struct ns));
-  for (int i = 0; i < NHASH; i++)
-    initlock(&ns->table[i].l, "bucket");
  ns->nextkey = 1;
  return ns;
 }
@@ -114,8 +112,13 @@ int
 ns_remove(struct ns *ns, int key, void *v)
 {
  uint i = key % NHASH;
-  rcu_begin_write(&ns->table[i].l);
+  rcu_begin_write(0);

+ retry:
+  (void) 0;
+
+  int fakelock = 0;
+  int *pelock = &fakelock;
  struct elem * volatile * pe = &ns->table[i].chain;
  for (;;) {
    struct elem *e = *pe;
@@ -123,10 +126,20 @@ ns_remove(struct ns *ns, int key, void *v)
      break;

    if (e->key == key && (e->val == v || v == 0)) {
-      for (;;)
-	if (__sync_bool_compare_and_swap(pe, e, e->next))
-	  break;
-      rcu_end_write(&ns->table[i].l);
+      if (!__sync_bool_compare_and_swap(&e->next_lock, 0, 1))
+	goto retry;
+      if (!__sync_bool_compare_and_swap(pelock, 0, 1)) {
+	e->next_lock = 0;
+	goto retry;
+      }
+      if (!__sync_bool_compare_and_swap(pe, e, e->next)) {
+	*pelock = 0;
+	e->next_lock = 0;
+	goto retry;
+      }
+
+      *pelock = 0;
+      rcu_end_write(0);
      rcu_delayed(e, kmfree);
      return 0;
    }
@@ -134,7 +147,7 @@ ns_remove(struct ns *ns, int key, void *v)
    pe = &e->next;
  }

-  rcu_end_write(&ns->table[i].l);
+  rcu_end_write(0);
  return -1;
 }


--- a/proc.c
+++ b/proc.c
@@ -68,7 +68,8 @@ allocproc(void)

  // Allocate kernel stack if possible.
  if((p->kstack = kalloc()) == 0){
-    ns_remove(nspid, p->pid, p);
+    if (ns_remove(nspid, p->pid, p) < 0)
+      panic("allocproc: ns_remove");
    rcu_delayed(p, kmfree);
    return 0;
  }
@@ -122,7 +123,8 @@ delrun(struct proc *p)

  if (p->on_runq < 0)
    panic("delrun not on runq");
-  ns_remove(nsrunq, p->on_runq, p);
+  if (ns_remove(nsrunq, p->on_runq, p) < 0)
+    panic("delrun: ns_remove");
  p->on_runq = -1;
 }

@@ -267,7 +269,8 @@ fork(int flags)
      kfree(np->kstack);
      np->kstack = 0;
      np->state = UNUSED;
-      ns_remove(nspid, np->pid, np);
+      if (ns_remove(nspid, np->pid, np) < 0)
+	panic("fork: ns_remove");
      rcu_delayed(np, kmfree);
      return -1;
    }