Try pre-zeroing with wqs

2e184ff9 · Silas Boyd-Wickizer · f71b44c4 · 2e184ff9 · 2e184ff9 · 2e184ff9
--- a/kernel/main.cc
+++ b/kernel/main.cc
@@ -22,6 +22,7 @@ void initnmi(void);
 void inittrap(void);
 void initseg(void);
 void initkalloc(u64 mbaddr);
+void initz(void);
 void initrcu(void);
 void initproc(void);
 void initbio(void);
@@ -107,6 +108,7 @@ cmain(u64 mbmagic, u64 mbaddr)
  inittrap();
  initlapic();
  initkalloc(mbaddr);
+  initz();
  initproc();      // process table
  initsched();     // scheduler run queues
  initidle();

--- a/kernel/uaccess.S
+++ b/kernel/uaccess.S
@@ -48,10 +48,17 @@ __fetch_end:
        movl $0, PROC_UACCESS(%r11)
        ret
-.globl clear_page
+.globl zpage
 .align 8
-clear_page:
+zpage:
        movl $4096/8,%ecx
 	xorl %eax,%eax
 	rep stosq
 	ret
+// XXX(sbw) should skip first cache line, prefetchnt, then zero
+.globl zrun_nc
+.align 8
+zrun_nc:
+        jmp zpage
\ No newline at end of file
--- a/kernel/zalloc.cc
+++ b/kernel/zalloc.cc
@@ -4,34 +4,110 @@
 #include "percpu.hh"
 #include "wq.hh"
 #include "kalloc.hh"
+#include "cpputil.hh"
-extern "C" void clear_page(void*);
+extern "C" void zpage(void*);
+extern "C" void zrun_nc(run*);
 struct zallocator {
  run*   run;
+  kmem   kmem;
  wframe frame;
-};
+  void  init(int);
+  char* alloc(const char*);
+  void  free(char*);
+  void  tryrefill();
+};
 percpu<zallocator> z_;
+struct zwork : public work {
+  zwork(wframe* frame, zallocator* zer)
+    : frame_(frame), zer_(zer)
+  {
+    frame_->inc();
+  }
+  virtual void run() {
+    for (int i = 0; i < 32; i++) {
+      struct run* r = (struct run*)kalloc("zpage");
+      if (r == nullptr)
+        break;
+      zrun_nc(r);
+      zer_->kmem.free(r);
+    }
+    frame_->dec();
+    delete this;
+  }
+  wframe* frame_;
+  zallocator* zer_;
+  NEW_DELETE_OPS(zwork);
+};
+//
+// zallocator
+//
+void
+zallocator::tryrefill(void)
+{
+  if (kmem.nfree < 16 && frame.zero()) {
+    zwork* w = new zwork(&frame, this);
+    if (wq_push(w) < 0)
+      delete w;
+  }
+}
+void
+zallocator::init(int c)
+{
+  frame.clear();
+  kmem.name[0] = (char) c + '0';
+  safestrcpy(kmem.name+1, "zmem", MAXNAME-1);
+  kmem.size = PGSIZE;
+}
 char*
-zalloc(const char* name)
+zallocator::alloc(const char* name)
 {
-  char* p = kalloc(name);
+  char* p;
-  if (p)
-    clear_page(p);
+  p = (char*) kmem.alloc(name);
+  if (p == nullptr) {
+    p = kalloc(name);
+    if (p != nullptr)
+      zpage(p);
+  }
+  tryrefill();
  return p;
 }
 void
+zallocator::free(char* p)
+{
+  if (0) 
+    for (int i = 0; i < 4096; i++)
+      assert(p[i] == 0);
+  kmem.free((struct run*)p);
+}
+char*
+zalloc(const char* name)
+{
+  return z_->alloc(name);
+}
+void
 zfree(char* p)
 {
-  kfree(p);
+  z_->free(p);
 }
 void
 initz(void)
 {
-  for (int i = 0; i < NCPU; i++)
+  for (int c = 0; c < NCPU; c++)
-    z_[i].frame.clear();
+    z_[c].init(c);
 }