atomic (lockless) kmalloc

731728ab · Nickolai Zeldovich · 4646ef72 · 731728ab
--- a/kernel/kmalloc.cc
+++ b/kernel/kmalloc.cc
@@ -19,9 +19,8 @@ struct header {
 };
 struct freelist {
-  struct header *buckets[KMMAX+1];
+  std::atomic<header*> buckets[KMMAX+1];
  char name[MAXNAME];
-  struct spinlock lock;
 };
 struct freelist freelists[NCPU];
@@ -32,17 +31,16 @@ kminit(void)
  for (int c = 0; c < NCPU; c++) {
    freelists[c].name[0] = (char) c + '0';
    safestrcpy(freelists[c].name+1, "freelist", MAXNAME-1);
-    initlock(&freelists[c].lock, freelists[c].name, LOCKSTAT_KMALLOC);
  }
 }
 // get more space for freelists[c].buckets[b]
-void
+int
 morecore(int c, int b)
 {
  char *p = kalloc();
  if(p == 0)
-    return;
+    return -1;
  int sz = 1 << b;
  for(char *q = p;
@@ -52,15 +50,14 @@ morecore(int c, int b)
    h->next = freelists[c].buckets[b];
    freelists[c].buckets[b] = h;
  }
+  return 0;
 }
 void *
 kmalloc(u64 nbytes)
 {
  int nn = 1, b = 0;
-  void *r = 0;
-  struct header *h;
-  int c = mycpu()->id;
  while(nn < nbytes && b <= KMMAX){
    nn *= 2;
@@ -71,21 +68,27 @@ kmalloc(u64 nbytes)
  if(b > KMMAX)
    panic("kmalloc too big");
-  acquire(&freelists[c].lock);
+  scoped_gc_epoch gc;
-  if(freelists[c].buckets[b] == 0)
+  struct header *h;
-    morecore(c, b);
+  int c = mycpu()->id;
+  for (;;) {
    h = freelists[c].buckets[b];
-  if(h){
+    if (!h) {
-    freelists[c].buckets[b] = h->next;
+      if (morecore(c, b) < 0) {
-    r = h + 1;
+        cprintf("kmalloc(%d) failed\n", (int) nbytes);
-    h->next = (header*) (long) b;
+        return 0;
+      }
+    } else {
+      if (cmpxch(&freelists[c].buckets[b], h, h->next))
+        break;
    }
-  release(&freelists[c].lock);
+  }
+  void *r = h + 1;
+  h->next = (header*) (long) b;
-  if (r)
  mtlabel(mtrace_label_heap, r, nbytes, "kmalloc'ed", sizeof("kmalloc'ed"));
-  if(r == 0)
-    cprintf("kmalloc(%d) failed\n", (int) nbytes);
  return r;
 }
@@ -96,8 +99,6 @@ kmfree(void *ap)
  struct header *h;
  int b;
-  acquire(&freelists[c].lock);
  h = (struct header *) ((char *)ap - sizeof(struct header));
  b = (long) h->next;
  if(b < 0 || b > KMMAX)
@@ -108,9 +109,10 @@ kmfree(void *ap)
    memset(ap, 3, (1<<b) - sizeof(struct header));
  h->next = freelists[c].buckets[b];
-  freelists[c].buckets[b] = h;
+  while (!cmpxch_update(&freelists[c].buckets[b], &h->next, h))
+    ; /* spin */
  mtunlabel(mtrace_label_heap, ap);
-  release(&freelists[c].lock);
 }
 int