per-core periodic rcu_gc and gc lists

1db560a8 · Nickolai Zeldovich · 446cb46d · 1db560a8 · 1db560a8 · 1db560a8
--- a/defs.h
+++ b/defs.h
@@ -142,6 +142,7 @@ void            rcu_end_write(struct spinlock *);
 void            rcu_begin_read(void);
 void            rcu_end_read(void);
 void            rcu_delayed(void*, void (*dofree)(void*));
+void		rcu_gc(void);

 // swtch.S
 void            swtch(struct context**, struct context*);

--- a/proc.c
+++ b/proc.c
@@ -545,6 +545,12 @@ scheduler(void)
    }

    if (idle[cpu->id]) {
+      int now = ticks;
+      if (now - cpu->last_rcu_gc_ticks > 100) {
+	rcu_gc();
+	cpu->last_rcu_gc_ticks = now;
+      }
+
      sti();
      hlt();
    }

--- a/proc.h
+++ b/proc.h
@@ -109,6 +109,7 @@ struct cpu {
  volatile uint booted;        // Has the CPU started?
  int ncli;                    // Depth of pushcli nesting.
  int intena;                  // Were interrupts enabled before pushcli?
+  int last_rcu_gc_ticks;
  
  // Cpu-local storage variables; see below
  struct cpu *cpu;

--- a/rcu.c
+++ b/rcu.c
@@ -8,17 +8,14 @@
 #include "queue.h"
 #include "proc.h"

-#define NRCU 1000
-
 struct rcu {
  void *item;
  unsigned long epoch;
  struct rcu *rcu;
  void (*dofree)(void *);
 };
-static struct rcu *rcu_delayed_head __attribute__ ((aligned (CACHELINE)));
-static struct rcu *rcu_delayed_tail __attribute__ ((aligned (CACHELINE)));
-static struct rcu *rcu_freelist __attribute__ ((aligned (CACHELINE)));
+static struct rcu *rcu_delayed_head[NCPU] __attribute__ ((aligned (CACHELINE)));
+static struct rcu *rcu_delayed_tail[NCPU] __attribute__ ((aligned (CACHELINE)));
 static uint global_epoch __attribute__ ((aligned (CACHELINE)));
 static uint min_epoch __attribute__ ((aligned (CACHELINE)));
 static struct spinlock rcu_lock __attribute__ ((aligned (CACHELINE)));
@@ -27,28 +24,13 @@ static int delayed_nfree __attribute__ ((aligned (CACHELINE)));
 void
 rcuinit(void)
 {
-  struct rcu *r;
-  int i;
-
  initlock(&rcu_lock, "rcu");
-  for (i = 0; i < NRCU; i++) {
-    r = (struct rcu *) kmalloc(sizeof(struct rcu));
-    memset(r, 0, sizeof(struct rcu));
-    r->rcu = rcu_freelist;
-    rcu_freelist = r;
-  }
-  // cprintf("rcu_init: allocated %d bytes\n", sizeof(struct rcu) * NRCU);
 }

 struct rcu *
 rcu_alloc()
 {
-  struct rcu *r = rcu_freelist;
-  if (r == 0) {
-    panic("rcu_alloc");
-  }
-  rcu_freelist = r->rcu;
-  return r;
+  return kmalloc(sizeof(struct rcu));
 }

 void
@@ -71,7 +53,7 @@ rcu_gc(void)
  ns_enumerate(nspid, rcu_min);
  acquire(&rcu_lock);

-  for (r = rcu_delayed_head; r != NULL; r = nr) {
+  for (r = rcu_delayed_head[cpu->id]; r != NULL; r = nr) {
    if (r->epoch >= min_epoch)
      break;
    // cprintf("free: %d\n", r->epoch);
@@ -80,12 +62,11 @@ rcu_gc(void)
    r->dofree(r->item);
    delayed_nfree--;
    n++;
-    rcu_delayed_head = r->rcu;
-    if (rcu_delayed_head == 0)
-      rcu_delayed_tail = 0;
+    rcu_delayed_head[cpu->id] = r->rcu;
+    if (rcu_delayed_head[cpu->id] == 0)
+      rcu_delayed_tail[cpu->id] = 0;
    nr = r->rcu;
-    r->rcu = rcu_freelist;
-    rcu_freelist = r;
+    kmfree(r);
  }
  release(&rcu_lock);
  // cprintf("rcu_gc: n %d ndelayed_free=%d\n", n, delayed_nfree);
@@ -104,10 +85,11 @@ rcu_delayed(void *e, void (*dofree)(void *))
  r->epoch = global_epoch;
  acquire(&rcu_lock);
  // cprintf("rcu_delayed: %d\n", global_epoch);
-  if (rcu_delayed_tail != 0) 
-    rcu_delayed_tail->rcu = r;
-  rcu_delayed_tail = r;
-  if (rcu_delayed_head == 0) rcu_delayed_head = r;
+  if (rcu_delayed_tail[cpu->id] != 0) 
+    rcu_delayed_tail[cpu->id]->rcu = r;
+  rcu_delayed_tail[cpu->id] = r;
+  if (rcu_delayed_head[cpu->id] == 0)
+    rcu_delayed_head[cpu->id] = r;
  release(&rcu_lock);
  delayed_nfree++;
 }
@@ -139,6 +121,6 @@ rcu_end_write(struct spinlock *l)
  // for other data structures using rcu, use atomic add:
  __sync_fetch_and_add(&global_epoch, 1);
  release(l);
-  rcu_gc();
+  // rcu_gc();
 }