merge

143534ce · Silas Boyd-Wickizer · c90f84ff · 0b8c8259 · 143534ce · 143534ce
--- a/Makefile
+++ b/Makefile
@@ -36,7 +36,7 @@ COMFLAGS  = -static -fno-builtin -fno-strict-aliasing -O2 -Wall \
 COMFLAGS += $(shell $(CC) -fno-stack-protector -E -x c /dev/null >/dev/null 2>&1 && echo -fno-stack-protector)
 CFLAGS   := $(COMFLAGS) -std=c99 $(CFLAGS)
 CXXFLAGS := $(COMFLAGS) -std=c++0x -Wno-sign-compare -fno-exceptions -fno-rtti -fcheck-new $(CXXFLAGS)
-ASFLAGS   = -Iinclude -m64 -gdwarf-2 -MD
+ASFLAGS   = -Iinclude -m64 -gdwarf-2 -MD -DHW_$(HW) -include param.h
 LDFLAGS   = -m elf_x86_64
 all:

--- a/include/amd64.h
+++ b/include/amd64.h
@@ -138,6 +138,14 @@ writegs(u16 v)
  __asm volatile("movw %0, %%gs" : : "r" (v));
 }
+static inline u16
+readgs(void)
+{
+  u16 v;
+  __asm volatile("movw %%gs, %0" : "=r" (v));
+  return v;
+}
 static inline u64
 readmsr(u32 msr)
 {
@@ -231,6 +239,7 @@ rcr2(void)
 // Layout of the trap frame built on the stack by the
 // hardware and by trapasm.S, and passed to trap().
+// Also used by sysentry (but sparsely populated).
 struct trapframe {
  u16 padding3[7];
  u16 ds;

--- a/include/bits.hh
+++ b/include/bits.hh
@@ -44,6 +44,7 @@
 // FS/GS base registers
 #define MSR_FS_BASE     0xc0000100
 #define MSR_GS_BASE     0xc0000101
+#define MSR_GS_KERNBASE 0xc0000102
 // SYSCALL and SYSRET registers
 #define MSR_STAR        0xc0000081

--- a/include/kalloc.hh
+++ b/include/kalloc.hh
 #include "atomic.hh"
 template<class T>
-struct vptr {
+struct vptr64 {
-  u128 _a;
+  typedef u128 __inttype;
-  T ptr() const { return (T) (_a & 0xffffffffffffffffULL); }
+  typedef T __ptrtype;
+  __inttype _a;
+  T ptr() const { return (T) iptr(); }
+  u64 iptr() const { return _a & 0xffffffffffffffffULL; }
  u64 v() const { return _a >> 64; }
-  vptr(T p, u64 v) : _a((((u128)v)<<64) | (u64) p) {}
+  vptr64(T p, u64 v) : _a((((u128)v)<<64) | (u64) p) {}
-  vptr(u128 a) : _a(a) {}
+  vptr64(u128 a) : _a(a) {}
 };
 template<class T>
+struct vptr48 {
+  typedef u64 __inttype;
+  typedef T __ptrtype;
+  __inttype _a;
+  T ptr() const {
+    u64 i = iptr();
+    if (i & (1ULL << 47))
+      i += 0xffffULL << 48;
+    return (T) i;
+  }
+  u64 iptr() const { return _a & 0xffffffffffffULL; }
+  u16 v() const { return _a >> 48; }
+  vptr48(T p, u16 v) : _a((((u64)v)<<48) | (((u64)p) & 0xffffffffffffULL)) {}
+  vptr48(u64 a) : _a(a) {}
+};
+template<class VPTR>
 class versioned {
 private:
-  std::atomic<u128> _a;
+  std::atomic<typename VPTR::__inttype> _a;
 public:
-  vptr<T> load() { return vptr<T>(_a.load()); }
+  VPTR load() { return VPTR(_a.load()); }
-  bool compare_exchange(const vptr<T> &expected, T desired) {
+  bool compare_exchange(const VPTR &expected, typename VPTR::__ptrtype desired) {
-    vptr<T> n(desired, expected.v());
+    VPTR n(desired, expected.v());
    return cmpxch(&_a, expected._a, n._a);
  }
 };
@@ -31,7 +54,7 @@ struct kmem {
  char name[MAXNAME];
  u64 size;
  u64 ninit;
-  versioned<run*> freelist;
+  versioned<vptr48<run*>> freelist;
  std::atomic<u64> nfree;
 } __mpalign__;

--- a/include/kern_c.h
+++ b/include/kern_c.h
@@ -59,4 +59,5 @@ void threadhelper(void (*fn)(void *), void *arg);
 struct trapframe;
 void trap(struct trapframe *tf);
+void sysentry(void);
+void sysentry_c(void);
--- a/include/kernel.hh
+++ b/include/kernel.hh
@@ -287,6 +287,7 @@ void initpg(void);
 void initmp(void);
 void initlapic(void);
 void inittls(void);
+void initnmi(void);
 void inittrap(void);
 void initseg(void);
 void initkalloc(u64 mbaddr);

--- a/include/mmu.h
+++ b/include/mmu.h
@@ -95,7 +95,7 @@ struct intdesc
 {
  u16 rip0;
  u16 cs;
-  u8 reserved0;
+  u8 ist;
  u8 bits;
  u16 rip1;
  u32 rip2;
@@ -129,3 +129,6 @@ struct taskstate
 	(u16) ((rip)&0xffff), (cs), 0, bits, (u16) (((rip)>>16)&0xffff), \
 	(u32) ((u64)(rip)>>32), 0, \
 }
+#define PROC_KSTACK_OFFSET 40
+#define TRAPFRAME_SIZE     192
--- a/kernel/hwvm.cc
+++ b/kernel/hwvm.cc
@@ -180,6 +180,7 @@ inittls(void)
  c = &cpus[cpunum()];
  writegs(KDSEG);
  writemsr(MSR_GS_BASE, (u64)&c->cpu);
+  writemsr(MSR_GS_KERNBASE, (u64)&c->cpu);
  c->cpu = c;
  c->proc = NULL;
  c->kmem = &kmems[cpunum()];

--- a/kernel/kmalloc.cc
+++ b/kernel/kmalloc.cc
@@ -19,7 +19,7 @@ struct header {
 };
 struct freelist {
-  versioned<header*> buckets[KMMAX+1];
+  versioned<vptr48<header*>> buckets[KMMAX+1];
  char name[MAXNAME];
 };

--- a/kernel/main.cc
+++ b/kernel/main.cc
@@ -6,6 +6,8 @@
 #include "cpu.hh"
 #include "amd64.h"
 #include "hwvm.hh"
+#include "condvar.h"
+#include "proc.hh"
 extern void initidle(void);
 extern void idleloop(void);
@@ -20,6 +22,7 @@ mpboot(void)
  initlapic();
  initsamp();
  initidle();
+  initnmi();
  bstate = 1;
  idleloop();
 }
@@ -97,10 +100,19 @@ cmain(u64 mbmagic, u64 mbaddr)
    cprintf("ncpu %d %lu MHz\n", ncpu, cpuhz / 1000000);
  inituser();      // first user process
+  initnmi();
  bootothers();    // start other processors
  kpml4.e[0] = 0;  // don't need 1 GB identity mapping anymore
  lcr3(rcr3());
+  if (PROC_KSTACK_OFFSET != __offsetof(struct proc, kstack))
+    panic("PROC_KSTACK_OFFSET mismatch: %d %ld\n",
+          PROC_KSTACK_OFFSET, __offsetof(struct proc, kstack));
+  if (TRAPFRAME_SIZE != sizeof(trapframe))
+    panic("TRAPFRAME_SIZE mismatch: %d %ld\n",
+          TRAPFRAME_SIZE, sizeof(trapframe));
  idleloop();
  panic("Unreachable");
 }
--- a/kernel/proc.cc
+++ b/kernel/proc.cc
@@ -281,7 +281,6 @@ growproc(int n)
    if (e->vma_start <= newstart) {
      if (e->vma_end >= newstart + newn) {
        myproc()->brk += n;
-        switchuvm(myproc());
        return 0;
      }
@@ -311,7 +310,6 @@ growproc(int n)
  span.replace(prev, repl);
  myproc()->brk += n;
-  switchuvm(myproc());
  return 0;
 }

--- a/kernel/syscall.cc
+++ b/kernel/syscall.cc
@@ -58,7 +58,8 @@ argint64(int n, u64 *ip)
  case 2: *ip = myproc()->tf->rdx; break;
  case 3: *ip = myproc()->tf->rcx; break;
  case 4: *ip = myproc()->tf->r8;  break;
-  case 5: *ip = myproc()->tf->r9;  break;
+  // r9 is corrupted by sysentry
+  // case 5: *ip = myproc()->tf->r9;  break;
  default:
    cprintf("argint64: bad arg %d\n", n);
    return -1;

--- a/kernel/trap.cc
+++ b/kernel/trap.cc
@@ -10,6 +10,7 @@
 #include "proc.hh"
 #include "kmtrace.hh"
 #include "bits.hh"
+#include "kalloc.hh"
 struct intdesc idt[256] __attribute__((aligned(16)));
@@ -17,6 +18,29 @@ struct intdesc idt[256] __attribute__((aligned(16)));
 extern u64 trapentry[];
 void
+sysentry_c()
+{
+  writegs(KDSEG);
+  writemsr(MSR_GS_BASE, (u64)&cpus[cpunum()].cpu);
+  sti();
+  if(myproc()->killed) {
+    mtstart(trap, myproc());
+    exit();
+  }
+  trapframe *tf = (trapframe*) (myproc()->kstack + KSTACKSIZE - sizeof(*tf));
+  myproc()->tf = tf;
+  syscall();
+  if(myproc()->killed) {
+    mtstart(trap, myproc());
+    exit();
+  }
+}
+void
 trap(struct trapframe *tf)
 {
  writegs(KDSEG);
@@ -30,7 +54,6 @@ trap(struct trapframe *tf)
    panic("NMI");
  }
-  // XXX(sbw) sysenter/sysexit
  if(tf->trapno == T_SYSCALL){
    sti();
    if(myproc()->killed) {
@@ -183,9 +206,18 @@ inittrap(void)
 }
 void
+initnmi(void)
+{
+  void *nmistackbase = ksalloc(slab_stack);
+  mycpu()->ts.ist[1] = (u64) nmistackbase + KSTACKSIZE;
+  if (mycpu()->id == 0)
+    idt[T_NMI].ist = 1;
+}
+void
 initseg(void)
 {
-  extern void sysentry(void);
  volatile struct desctr dtr;
  struct cpu *c;
@@ -201,7 +233,6 @@ initseg(void)
  dtr.base = (u64)c->gdt;
  lgdt((void *)&dtr.limit);
-#if 0
  // When executing a syscall instruction the CPU sets the SS selector
  // to (star >> 32) + 8 and the CS selector to (star >> 32).
  // When executing a sysret instruction the CPU sets the SS selector
@@ -209,8 +240,7 @@ initseg(void)
  u64 star = ((((u64)UCSEG|0x3) - 16)<<48)|((u64)KCSEG<<32);
  writemsr(MSR_STAR, star);
  writemsr(MSR_LSTAR, (u64)&sysentry);
-  writemsr(MSR_SFMASK, FL_TF);
+  writemsr(MSR_SFMASK, FL_TF | FL_IF);
-#endif
 }
 // Pushcli/popcli are like cli/sti except that they are matched:

--- a/kernel/trapasm.S
+++ b/kernel/trapasm.S
@@ -18,13 +18,71 @@
 #define TRAPCODE(x) _TRAP(x, EC)
 .code64
-#if 0
 .globl sysentry
 .align 8
 sysentry:
-        jmp sysentry
+        // can syscall/sysret be used safely in the presence of NMIs?
-#endif
+        // we are executing with cpl=0 but without a valid stack.
+        // blow away %r9: syscalls can take at most 5 args
+        swapgs
+        movq    %gs:8, %r9  // myproc()
+        swapgs
+        movq    %ss:PROC_KSTACK_OFFSET(%r9), %r9
+        addq    $(KSTACKSIZE-TRAPFRAME_SIZE), %r9
+        // syscall number: %rax
+        // function arguments: %rdi, %rsi, %rdx, %rcx, %r8, %r9 (killed)
+        // skip padding3, ds
+        movq    %r15, %ss:0x10(%r9)
+        movq    %r14, %ss:0x18(%r9)
+        movq    %r13, %ss:0x20(%r9)
+        movq    %r12, %ss:0x28(%r9)
+        movq    %rbp, %ss:0x30(%r9)
+        movq    %rbx, %ss:0x38(%r9)
+        // skip r11 (0x40)
+        // skip r10 (0x48)
+        // skip r9 (0x50)
+        movq    %r8,  %ss:0x58(%r9)
+        movq    %rax, %ss:0x60(%r9)
+        movq    %r10, %ss:0x68(%r9)  // rcx saved by usys.S
+        movq    %rdx, %ss:0x70(%r9)
+        movq    %rsi, %ss:0x78(%r9)
+        movq    %rdi, %ss:0x80(%r9)
+        // skip trapno (0x88)
+        // skip err, padding2 (0x90)
+        movq    %rcx, %ss:0x98(%r9)  // rip saved by syscall
+        // skip cs, padding (0xa0)
+        movq    %r11, %ss:0xa8(%r9)  // eflags saved by syscall
+        movq    %rsp, %ss:0xb0(%r9)
+        movw    $KDSEG, %ax
+        movw    %ax, %ds
+        movw    %ax, %es
+        movq    %r9, %rsp
+        call    sysentry_c
+        // return using SYSRET
+        cli
+        movq    %rsp, %r11
+        movw    $UDSEG, %ax
+        movw    %ax, %ds
+        movw    %ax, %es
+        movq    %ss:0x10(%r11), %r15
+        movq    %ss:0x18(%r11), %r14
+        movq    %ss:0x20(%r11), %r13
+        movq    %ss:0x28(%r11), %r12
+        movq    %ss:0x30(%r11), %rbp
+        movq    %ss:0x38(%r11), %rbx
+        movq    %ss:0x60(%r11), %rax
+        movq    %ss:0x98(%r11), %rcx    // rip to be restored by sysret
+        movq    %ss:0xb0(%r11), %rsp
+        movq    %ss:0xa8(%r11), %r11    // eflags to be restored by sysret
+        sysretq
 trapcommon:
        pushq %rdi
        pushq %rsi

--- a/lib/usys.S
+++ b/lib/usys.S
 #include "syscall.h"
 #include "traps.h"
-#define SYSCALL(name) \
+#define SYSCALL_INT(name) \
  .globl name; \
  name: \
    movq $SYS_ ## name, %rax; \
    int $T_SYSCALL; \
    ret
+#define SYSCALL(name) \
+  .globl name; \
+  name: \
+    movq $SYS_ ## name, %rax; \
+    movq %rcx, %r10; \
+    syscall; \
+    ret
 SYSCALL(fork)
 SYSCALL(exit)
 SYSCALL(wait)
@@ -16,7 +24,7 @@ SYSCALL(read)
 SYSCALL(write)
 SYSCALL(close)
 SYSCALL(kill)
-SYSCALL(exec)
+SYSCALL_INT(exec)
 SYSCALL(open)
 SYSCALL(mknod)
 SYSCALL(unlink)

--- a/param.h
+++ b/param.h
@@ -11,12 +11,12 @@
 #define MAXNAME      16  // max string names
 #define NEPOCH        4
 #define CACHELINE    64  // cache line size
-#define CPUKSTACKS   (NPROC + NCPU)
+#define CPUKSTACKS   (NPROC + NCPU*2)
 #define QUANTUM      10  // scheduling time quantum and tick length (in msec)
 #define CILKSHIFT     4  // 2^WORKSHIFT work queue slots
 #define VICTIMAGE 1000000 // cycles a proc executes before an eligible victim
 #define VERBOSE       0  // print kernel diagnostics
-#define	SPINLOCK_DEBUG 1 // Debug spin locks
+#define SPINLOCK_DEBUG 1 // Debug spin locks
 #define RCU_TYPE_DEBUG 1
 #define LOCKSTAT      1
 #define VERIFYFREE    0  // Unreliable, e.g. vma's vmnode pointer gets reused