Merge

b1a55189 · Silas Boyd-Wickizer · 893b2640 · 492d8f96 · b1a55189 · b1a55189
--- a/Makefile
+++ b/Makefile
@@ -89,7 +89,7 @@ gdb: $(KERN)
 ## mtrace
 ##
 mscan.syms: $(KERN)
-	$(NM) -S $< > $@
+	$(NM) -C -S $< > $@
 mscan.kern: $(KERN)
 	cp $< $@

--- a/README
+++ b/README
@@ -4,7 +4,11 @@
    git+ssh://amsterdam.csail.mit.edu/home/am6/mpdev/qemu.git -b mtrace
  #define MTRACE 1 in param.h
+  If mtrace isn't cloned next to the xv6-scale repository, then set
+  QEMUSRC in config.mk to the directory containing mtrace-magic.h.
+  Set MTRACE in config.mk to the mtrace QEMU binary's path.
  $ make mscan.out
+  or make mtrace.out to generate just the trace file and not the summary.
 * Networking with lwIP
@@ -57,3 +61,11 @@
  $ apt-get install libjemalloc-dev
  $ make HW=user
  $ ./o.user/utest
+* abstract sharing
+  Obtain and configure mtrace as described above.
+  Disable DEBUG in param.h.
+  $ make QEMUSMP=8 mtrace.out
+  Run asharing in xv6 to generate abstract sharing traces
+  $ mscan --abstract-scopes --unexpected-sharing
--- a/bin/Makefrag
+++ b/bin/Makefrag
@@ -26,6 +26,7 @@ UPROGS= \
 	wqsh \
 	cp \
 	perf \
+	asharing \
 	xls \
 	xdu \
        wqtest

--- a/bin/asharing.cc
+++ b/bin/asharing.cc
+// Tests to drive abstract sharing analysis
+#include "types.h"
+#include "user.h"
+#include "fcntl.h"
+#include "mtrace.h"
+static int cpu;
+void
+next()
+{
+  if (setaffinity(cpu) < 0) {
+    cpu = 0;
+    if (setaffinity(cpu) < 0)
+      die("sys_setaffinity(%d) failed", cpu);
+  }
+  cpu++;
+}
+void
+vmsharing(void)
+{
+  for (int i = 0; i < 8; i++) {
+    next();
+    volatile char *p = (char*)(0x40000UL + i * 4096);
+    mtenable("xv6-mapsharing");
+    if (map((void *) p, 4096) < 0)
+      die("map failed");
+    mtdisable("xv6-mapsharing");
+    mtenable("xv6-mapsharing");
+    if (unmap((void *) p, 4096) < 0)
+      die("unmap failed");
+    mtdisable("xv6-mapsharing");
+  }
+}
+void
+fssharing(void)
+{
+  // Note that we keep these files open; otherwise all of these
+  // operations will share the abstract FD object and we won't get any
+  // results.
+  next();
+  mtenable("xv6-fssharing");
+  open("a", O_CREATE|O_RDWR);
+  mtdisable("xv6-fssharing");
+  next();
+  mtenable("xv6-fssharing");
+  open("b", O_CREATE|O_RDWR);
+  mtdisable("xv6-fssharing");
+  next();
+  mtenable("xv6-fssharing");
+  open("a", O_RDWR);
+  mtdisable("xv6-fssharing");
+  next();
+  mtenable("xv6-fssharing");
+  open("b", O_RDWR);
+  mtdisable("xv6-fssharing");
+}
+int
+main(int ac, char **av)
+{
+  if (ac == 2 && strcmp(av[1], "vm") == 0)
+    vmsharing();
+  else if (ac == 2 && strcmp(av[1], "fs") == 0)
+    fssharing();
+  else
+    fprintf(1, "usage: %s vm|fs\n", av[0]);
+}
--- a/include/cpputil.hh
+++ b/include/cpputil.hh
@@ -133,7 +133,7 @@ extern void *__dso_handle;
 #define NEW_DELETE_OPS(classname)                                   \
  static void* operator new(unsigned long nbytes) {                 \
    assert(nbytes == sizeof(classname));                            \
-    return kmalloc(sizeof(classname));                              \
+    return kmalloc(sizeof(classname), #classname);                  \
  }                                                                 \
                                                                    \
  static void* operator new(unsigned long nbytes, classname *buf) { \

--- a/include/kern_c.h
+++ b/include/kern_c.h
@@ -52,6 +52,7 @@ long sys_async(int, size_t, off_t, u32, u32);
 long sys_script(void *addr, u64 len, u64 chunk);
 long sys_setfs(u64 base);
 long sys_wqwait(void);
+long sys_setaffinity(int cpu);
 extern long (*syscalls[])(u64, u64, u64, u64, u64);
 // other exported/imported functions

--- a/include/kernel.hh
+++ b/include/kernel.hh
@@ -68,6 +68,7 @@ void            vcprintf(const char *fmt, va_list ap);
 void            panic(const char*, ...) 
                  __noret__ __attribute__((format(printf, 1, 2)));
 void            kerneltrap(struct trapframe *tf) __noret__;
+void            vsnprintf(char *buf, u32 n, const char *fmt, va_list ap);
 void            snprintf(char *buf, u32 n, const char *fmt, ...);
 void            printtrace(u64 rbp);
@@ -121,13 +122,13 @@ void            idlezombie(struct proc*);
 void            ioapicenable(int irq, int cpu);
 // kalloc.c
-char*           kalloc(void);
+char*           kalloc(const char *name);
 void            kfree(void*);
 void*           ksalloc(int slabtype);
 void            ksfree(int slabtype, void*);
-void*           kmalloc(u64 nbytes);
+void*           kmalloc(u64 nbytes, const char *name);
 void            kmfree(void*, u64 nbytes);
-int             kmalign(void **p, int align, u64 size);
+int             kmalign(void **p, int align, u64 size, const char *name);
 void            kmalignfree(void *, int align, u64 size);
 void            verifyfree(char *ptr, u64 nbytes);
 void            kminit(void);

--- a/include/kmtrace.hh
+++ b/include/kmtrace.hh
+#pragma once
 #include "mtrace.h"
 #if MTRACE
 // Tell mtrace about switching threads
 struct kstack_tag {
@@ -61,6 +64,51 @@ static inline void mtresume(struct proc *p)
 #define mtrec() mtrace_call_set(1, ~0ull)
 #define mtign() mtrace_call_set(0, ~0ull)
+class mt_ascope
+{
+  char name[32];
+public:
+  explicit mt_ascope(const char *fmt, ...)
+  {
+    va_list ap;
+    va_start(ap, fmt);
+    vsnprintf(name, sizeof(name) - 1, fmt, ap);
+    va_end(ap);
+    mtrace_ascope_register(0, name);
+  }
+  ~mt_ascope()
+  {
+    mtrace_ascope_register(1, name);
+  }
+};
+static inline void mtreadavar(const char *fmt, ...)
+{
+  char name[32];
+  va_list ap;
+  va_start(ap, fmt);
+  vsnprintf(name, sizeof(name), fmt, ap);
+  va_end(ap);
+  mtrace_avar_register(0, name);
+}
+static inline void mtwriteavar(const char *fmt, ...)
+{
+  char name[32];
+  va_list ap;
+  va_start(ap, fmt);
+  vsnprintf(name, sizeof(name), fmt, ap);
+  va_end(ap);
+  mtrace_avar_register(1, name);
+}
 #else
 #define mtstart(ip, p) do { } while (0)
 #define mtstop(p) do { } while (0)
@@ -70,4 +118,13 @@ static inline void mtresume(struct proc *p)
 #define mtign(cpu) do { } while (0)
 #define mtrec(cpu) do { } while (0)
 #define mtign(cpu) do { } while (0)
+class mt_ascope
+{
+public:
+  explicit mt_ascope(const char *fmt, ...) {}
+};
+#define mtreadavar(fmt, ...) do { } while (0)
+#define mtwriteavar(fmt, ...) do { } while (0)
 #endif
--- a/include/proc.hh
+++ b/include/proc.hh
@@ -66,7 +66,6 @@ struct proc : public rcu_freed {
  struct condvar cv;
  std::atomic<u64> epoch;      // low 8 bits are depth count
  char lockname[16];
-  int on_runq;
  int cpu_pin;
 #if MTRACE
  struct mtrace_stacks mtrace_stacks;
@@ -90,6 +89,8 @@ struct proc : public rcu_freed {
  void set_state(enum procstate s);
  enum procstate get_state(void) const { return state_; }
+  int set_cpu_pin(int cpu);
 private:
  enum procstate state_;       // Process state  
 };
--- a/include/syscall.h
+++ b/include/syscall.h
@@ -32,4 +32,5 @@
 #define SYS_script 31
 #define SYS_setfs  32
 #define SYS_wqwait 33
-#define SYS_ncount 34   /* total number of system calls */
+#define SYS_setaffinity 34
+#define SYS_ncount 35   /* total number of system calls */
--- a/include/user.h
+++ b/include/user.h
@@ -31,6 +31,7 @@ ssize_t pread(int, void*, size_t, off_t);
 int async(int, size_t, off_t, u32, u32);
 int script(void *addr, u64 len, u64 chunk);
 int setfs(u64 base);
+int setaffinity(int cpu);
 // ulib.c
 int stat(char*, struct stat*);

--- a/include/wq.hh
+++ b/include/wq.hh
@@ -35,7 +35,7 @@ struct cwork : public work {
 #define xmalloc(n)     malloc(n)
 #define xfree(p, sz)   free(p)
 #elif defined(XV6_KERNEL)
-#define xmalloc(n)   kmalloc(n) 
+#define xmalloc(n)   kmalloc(n, "xmalloc")
 #define xfree(p, sz) kmfree(p, sz)
 #else
 #define xmalloc(n)     malloc(n)

--- a/kernel/bootdata.c
+++ b/kernel/bootdata.c
@@ -62,4 +62,5 @@ long (*syscalls[])(u64, u64, u64, u64, u64) = {
  SYSCALL(script),
  SYSCALL(setfs),
  SYSCALL(wqwait),
+  SYSCALL(setaffinity),
 };
--- a/kernel/cpprt.cc
+++ b/kernel/cpprt.cc
@@ -7,7 +7,7 @@
 void *
 operator new[](unsigned long nbytes)
 {
-  u64 *x = (u64*) kmalloc(nbytes + sizeof(u64));
+  u64 *x = (u64*) kmalloc(nbytes + sizeof(u64), "array");
  *x = nbytes + sizeof(u64);
  return x+1;
 }

--- a/kernel/fs.cc
+++ b/kernel/fs.cc
@@ -22,6 +22,7 @@
 #include "buf.hh"
 #include "file.hh"
 #include "cpu.hh"
+#include "kmtrace.hh"
 #define min(a, b) ((a) < (b) ? (a) : (b))
 static void itrunc(struct inode*);
@@ -757,6 +758,16 @@ namex(inode *cwd, const char *path, int nameiparent, char *name)
    ip = idup(cwd);
  while((r = skipelem(&path, name)) == 1){
+    // XXX Doing this here requires some annoying reasoning about all
+    // of the callers of namei/nameiparent.  Also, since the abstract
+    // scope is implicit, it might be wrong (or non-existent) and
+    // documenting the abstract object sets of each scope becomes
+    // difficult and probably unmaintainable.  We have to compute this
+    // information here because it's the only place that's canonical.
+    // Maybe this should return the set of inodes traversed and let
+    // the caller declare the variables?  Would it help for the caller
+    // to pass in an abstract scope?
+    mtreadavar("inode:%x.%x", ip->dev, ip->inum);
    next = 0;
    if(next == 0){
      if(ip->type == 0)
@@ -785,6 +796,7 @@ namex(inode *cwd, const char *path, int nameiparent, char *name)
    gc_end_epoch();
    return 0;
  }
+  mtreadavar("inode:%x.%x", ip->dev, ip->inum);
  gc_end_epoch();
  return ip;
 }

--- a/kernel/hwvm.cc
+++ b/kernel/hwvm.cc
@@ -14,6 +14,10 @@
 using namespace std;
+static const char *levelnames[] = {
+  "PT", "PD", "PDP", "PML4"
+};
 static pgmap*
 descend(pgmap *dir, u64 va, u64 flags, int create, int level)
 {
@@ -29,7 +33,7 @@ retry:
  } else {
    if (!create)
      return nullptr;
-    next = (pgmap*) kalloc();
+    next = (pgmap*) kalloc(levelnames[level-1]);
    if (!next)
      return nullptr;
    memset(next, 0, PGSIZE);
@@ -84,7 +88,7 @@ setupkvm(void)
  pgmap *pml4;
  int k;
-  if((pml4 = (pgmap*)kalloc()) == 0)
+  if((pml4 = (pgmap*)kalloc("PML4")) == 0)
    return 0;
  k = PX(3, KBASE);
  memset(&pml4->e[0], 0, 8*k);

--- a/kernel/kalloc.cc
+++ b/kernel/kalloc.cc
@@ -160,7 +160,7 @@ kmemprint()
 }
 static char*
-kalloc_pool(struct kmem *km)
+kalloc_pool(struct kmem *km, const char *name)
 {
  struct run *r = 0;
  struct kmem *m;
@@ -196,7 +196,8 @@ kalloc_pool(struct kmem *km)
    return 0;
  }
-  mtlabel(mtrace_label_block, r, m->size, "kalloc", sizeof("kalloc"));
+  if (name)
+    mtlabel(mtrace_label_block, r, m->size, name, strlen(name));
  if (ALLOC_MEMSET && m->size <= 16384)
    memset(r, 2, m->size);
@@ -207,17 +208,17 @@ kalloc_pool(struct kmem *km)
 // Returns a pointer that the kernel can use.
 // Returns 0 if the memory cannot be allocated.
 char*
-kalloc(void)
+kalloc(const char *name)
 {
  if (!kinited)
    return pgalloc();
-  return kalloc_pool(kmems);
+  return kalloc_pool(kmems, name);
 }
 void *
 ksalloc(int slab)
 {
-  return kalloc_pool(slabmem[slab]);
+  return kalloc_pool(slabmem[slab], slabmem[slab]->name);
 }
 void

--- a/kernel/kmalloc.cc
+++ b/kernel/kmalloc.cc
@@ -34,10 +34,10 @@ kminit(void)
 }
 // get more space for freelists[c].buckets[b]
-int
+static int
 morecore(int c, int b)
 {
-  char *p = kalloc();
+  char *p = kalloc(nullptr);
  if(p == 0)
    return -1;
@@ -78,7 +78,7 @@ bucket(u64 nbytes)
 }
 void *
-kmalloc(u64 nbytes)
+kmalloc(u64 nbytes, const char *name)
 {
  int b = bucket(nbytes);
@@ -103,10 +103,11 @@ kmalloc(u64 nbytes)
    }
  }
+  mtlabel(mtrace_label_heap, (void*) h, nbytes, name, strlen(name));
  if (ALLOC_MEMSET)
    memset(h, 4, (1<<b));
-  mtlabel(mtrace_label_heap, (void*) h, nbytes, "kmalloc'ed", sizeof("kmalloc'ed"));
  return h;
 }
@@ -132,9 +133,9 @@ kmfree(void *ap, u64 nbytes)
 }
 int
-kmalign(void **p, int align, u64 size)
+kmalign(void **p, int align, u64 size, const char *name)
 {
-  void *mem = kmalloc(size + (align-1) + sizeof(void*));
+  void *mem = kmalloc(size + (align-1) + sizeof(void*), name);
  char *amem = ((char*)mem) + sizeof(void*);
  amem += align - ((uptr)amem & (align - 1));
  ((void**)amem)[-1] = mem;

--- a/kernel/net.cc
+++ b/kernel/net.cc
@@ -32,7 +32,7 @@ netfree(void *va)
 void *
 netalloc(void)
 {
-  return kalloc();
+  return kalloc("(netalloc)");
 }
 int
@@ -278,7 +278,7 @@ netbind(int sock, void *xaddr, int xaddrlen)
  void *addr;
  long r;
-  addr = kmalloc(xaddrlen);
+  addr = kmalloc(xaddrlen, "sockaddr");
  if (addr == nullptr)
    return -1;
@@ -314,7 +314,7 @@ netaccept(int sock, void *xaddr, void *xaddrlen)
  if (umemcpy(&len, lenptr, sizeof(*lenptr)))
    return -1;
-  addr = kmalloc(len);
+  addr = kmalloc(len, "sockaddr");
  if (addr == nullptr)
    return -1;
@@ -352,7 +352,7 @@ netwrite(int sock, char *ubuf, int len)
  int cc;
  int r;
-  kbuf = kalloc();
+  kbuf = kalloc("(netwrite)");
  if (kbuf == nullptr)
    return -1;
@@ -375,7 +375,7 @@ netread(int sock, char *ubuf, int len)
  int cc;
  int r;
-  kbuf = kalloc();
+  kbuf = kalloc("(netread)");
  if (kbuf == nullptr)
    return -1;

--- a/kernel/pipe.cc
+++ b/kernel/pipe.cc
@@ -30,7 +30,7 @@ pipealloc(struct file **f0, struct file **f1)
  *f0 = *f1 = 0;
  if((*f0 = file::alloc()) == 0 || (*f1 = file::alloc()) == 0)
    goto bad;
-  if((p = (pipe*)kmalloc(sizeof(*p))) == 0)
+  if((p = (pipe*)kmalloc(sizeof(*p), "pipe")) == 0)
    goto bad;
  p->readopen = 1;
  p->writeopen = 1;

--- a/kernel/proc.cc
+++ b/kernel/proc.cc
@@ -39,7 +39,7 @@ proc::proc(int npid) :
  rcu_freed("proc"), vmap(0), uwq(0), worker(0), kstack(0),
  pid(npid), parent(0), tf(0), context(0), killed(0),
  ftable(0), cwd(0), tsc(0), curcycles(0), cpuid(0), epoch(0),
-  on_runq(-1), cpu_pin(0), runq(0), oncv(0), cv_wakeup(0),
+  cpu_pin(0), runq(0), oncv(0), cv_wakeup(0),
  user_fs_(0), state_(EMBRYO)
 {
  snprintf(lockname, sizeof(lockname), "cv:proc:%d", pid);
@@ -85,6 +85,30 @@ proc::set_state(enum procstate s)
  state_ = s;
 }
+int
+proc::set_cpu_pin(int cpu)
+{
+  if (cpu < -1 || cpu >= ncpu)
+    return -1;
+  acquire(&lock);
+  if (myproc() != this)
+    panic("set_cpu_pin not implemented for non-current proc");
+  if (cpu == -1) {
+    cpu_pin = 0;
+    release(&lock);
+    return 0;
+  }
+  // Since we're the current proc, there's no runq to get off.
+  // post_swtch will put us on the new runq.
+  cpuid = cpu;
+  cpu_pin = 1;
+  myproc()->set_state(RUNNABLE);
+  sched();
+  assert(mycpu()->id == cpu);
+  return 0;
+}
 // Give up the CPU for one scheduling round.
 void
 yield(void)

--- a/kernel/sampler.cc
+++ b/kernel/sampler.cc
@@ -188,7 +188,7 @@ sampread(struct inode *ip, char *dst, u32 off, u32 n)
    u64 len = LOGHEADER_SZ;
    u64 cc;
-    hdr = (logheader*) kmalloc(len);
+    hdr = (logheader*) kmalloc(len, "logheader");
    if (hdr == nullptr)
      return -1;
    hdr->ncpus = NCPU;

--- a/kernel/sched.cc
+++ b/kernel/sched.cc
@@ -60,7 +60,9 @@ sched(void)
  struct proc *next = schednext();
  if (next == nullptr) {
-    if (myproc()->get_state() != RUNNABLE) {
+    if (myproc()->get_state() != RUNNABLE ||
+        // proc changed its CPU pin?
+        myproc()->cpuid != mycpu()->id) {
      next = idleproc();
    } else {
      myproc()->set_state(RUNNING);

--- a/kernel/spinlock.cc
+++ b/kernel/spinlock.cc
@@ -24,7 +24,7 @@ void*
 klockstat::operator new(unsigned long nbytes)
 {
  assert(nbytes == sizeof(klockstat));
-  return kmalloc(sizeof(klockstat));
+  return kmalloc(sizeof(klockstat), "klockstat");
 }
 void

--- a/kernel/sysfile.cc
+++ b/kernel/sysfile.cc
@@ -11,6 +11,7 @@
 #include "fcntl.h"
 #include "cpu.hh"
 #include "net.hh"
+#include "kmtrace.hh"
 static bool
 getfile(int fd, sref<file> *f)
@@ -280,6 +281,12 @@ sys_openat(int dirfd, const char *path, int omode)
  if(argcheckstr(path) < 0)
    return -1;
+  // Reads the dirfd FD, dirfd's inode, the inodes of all files in
+  // path; writes the returned FD
+  mt_ascope ascope("%s(%d,%s,%d)", __func__, dirfd, path, omode);
+  mtreadavar("inode:%x.%x", cwd->dev, cwd->inum);
  if(omode & O_CREATE){
    if((ip = create(cwd, path, T_FILE, 0, 0)) == 0)
      return -1;
@@ -309,6 +316,7 @@ sys_openat(int dirfd, const char *path, int omode)
    return -1;
  }
  iunlock(ip);
+  mtwriteavar("fd:%x.%x", myproc()->pid, fd);
  f->type = file::FD_INODE;
  f->ip = ip;

--- a/kernel/sysproc.cc
+++ b/kernel/sysproc.cc
@@ -9,6 +9,7 @@
 #include "cpu.hh"
 #include "vm.hh"
 #include "sperf.hh"
+#include "kmtrace.hh"
 long
 sys_fork(int flags)
@@ -87,6 +88,12 @@ sys_map(uptr addr, u64 len)
 {
  ANON_REGION(__func__, &perfgroup);
+#if MTRACE
+  mt_ascope ascope("%s(%p,%lx)", __func__, addr, len);
+  for (uptr i = PGROUNDDOWN(addr); i < PGROUNDUP(addr + len); i += PGSIZE)
+    mtwriteavar("page:%016x", i);
+#endif
  vmnode *vmn = new vmnode(PGROUNDUP(len) / PGSIZE);
  if (vmn == 0)
    return -1;
@@ -104,6 +111,12 @@ sys_unmap(uptr addr, u64 len)
 {
  ANON_REGION(__func__, &perfgroup);
+#if MTRACE
+  mt_ascope ascope("%s(%p,%lx)", __func__, addr, len);
+  for (uptr i = PGROUNDDOWN(addr); i < PGROUNDUP(addr + len); i += PGSIZE)
+    mtwriteavar("page:%016x", i);
+#endif
  uptr align_addr = PGROUNDDOWN(addr);
  uptr align_len = PGROUNDUP(addr + len) - align_addr;
  if (myproc()->vmap->remove(align_addr, align_len) < 0)
@@ -131,3 +144,9 @@ sys_setfs(u64 base)
  switchvm(p);
  return 0;
 }
+long
+sys_setaffinity(int cpu)
+{
+  return myproc()->set_cpu_pin(cpu);
+}
--- a/kernel/vm.cc
+++ b/kernel/vm.cc
@@ -15,6 +15,7 @@
 #include "cpputil.hh"
 #include "sperf.hh"
 #include "uwq.hh"
+#include "kmtrace.hh"
 enum { vm_debug = 0 };
@@ -69,7 +70,7 @@ vmnode::allocpg()
    if (page[i])
      continue;
-    char *p = kalloc();
+    char *p = kalloc("(vmnode::allocpg)");
    if (!p) {
      cprintf("allocpg: out of memory, leaving half-filled vmnode\n");
      return -1;
@@ -574,6 +575,11 @@ vmap::pagefault(uptr va, u32 err)
 int
 pagefault(struct vmap *vmap, uptr va, u32 err)
 {
+#if MTRACE
+  mt_ascope ascope("%s(%p)", __func__, va);
+  mtwriteavar("page:%016x", PGROUNDDOWN(va));
+#endif
  return vmap->pagefault(va, err);
 }

--- a/lib/usys.S
+++ b/lib/usys.S
@@ -49,3 +49,4 @@ SYSCALL(async)
 SYSCALL(script)
 SYSCALL(setfs)
 SYSCALL(wqwait)
+SYSCALL(setaffinity)
--- a/net/sys_arch.cc
+++ b/net/sys_arch.cc
@@ -208,7 +208,7 @@ sys_thread_new(const char *name, lwip_thread_fn thread, void *arg,
  struct lwip_thread *lt;
  struct proc *p;
-  lt = (struct lwip_thread*) kmalloc(sizeof(*lt));
+  lt = (struct lwip_thread*) kmalloc(sizeof(*lt), "lwip_thread");
  if (lt == nullptr)
    return 0;
  lt->thread = thread;

--- a/param.h
+++ b/param.h
@@ -32,7 +32,7 @@
 #define MTRACE       0
 #define PERFSIZE     (1<<20ull)
 #elif defined(HW_qemu)
-#define NCPU         4   // maximum number of CPUs
+#define NCPU         8   // maximum number of CPUs
 #define MTRACE       0
 #define PERFSIZE     (16<<20ull)
 #elif defined(HW_ud0)