Boot APs, but panic in mpboot.

7a532024 · Silas Boyd-Wickizer · fbe2bead · 7a532024 · 7a532024 · 7a532024
--- a/Makefile
+++ b/Makefile
@@ -39,9 +39,9 @@ CFLAGS += $(shell $(CC) -fno-stack-protector -E -x c /dev/null >/dev/null 2>&1 &
 ASFLAGS = -m64 -gdwarf-2
 LDFLAGS += -m elf_x86_64

-kernel: boot.o $(OBJS) initcode
+kernel: boot.o $(OBJS) initcode bootother
 	$(LD) $(LDFLAGS) -T kernel.ld -z max-page-size=4096 -e start \
-		-o $@ boot.o $(OBJS) -b binary initcode
+		-o $@ boot.o $(OBJS) -b binary initcode bootother

 initcode: initcode.S
 	$(CC) $(CFLAGS) -nostdinc -I. -c initcode.S
@@ -49,6 +49,12 @@ initcode: initcode.S
 	$(OBJCOPY) -S -O binary initcode.out initcode
 	$(OBJDUMP) -S initcode.o > initcode.asm

+bootother: bootother.S
+	$(CC) $(CFLAGS) -nostdinc -I. -c bootother.S
+	$(LD) $(LDFLAGS) -N -e start -Ttext 0x7000 -o bootother.out bootother.o
+	$(OBJCOPY) -S -O binary bootother.out bootother
+	$(OBJDUMP) -S bootother.out > bootother.asm
+
 xv6memfs.img: bootblock kernelmemfs
 	dd if=/dev/zero of=xv6memfs.img count=10000
 	dd if=bootblock of=xv6memfs.img conv=notrunc
@@ -59,11 +65,11 @@ xv6memfs.img: bootblock kernelmemfs
 .PHONY: clean qemu ud0

 clean: 
-	rm -f *.o *.d *.asm *.sym initcode kernel
+	rm -f *.o *.d *.asm *.sym initcode kernel bootother

-QEMUOPTS = -smp $(CPUS) -m 512  -nographic
+QEMUOPTS = -smp $(CPUS) -m 512
 qemu: kernel
-	$(QEMU) $(QEMUOPTS) -kernel kernel
+	$(QEMU) $(QEMUOPTS) -kernel kernel #-S -s

 ud0: kernel
 	rsync -avP kernel amsterdam.csail.mit.edu:/tftpboot/ud0/kernel.xv6
--- a/boot.S
+++ b/boot.S
@@ -23,7 +23,7 @@
 #define DATAMAGIC 0x123456789abcdef0

 # STACK is the size of the bootstrap stack.
-#define STACK 8192
+#define STACK 4096

 # CGATEXTMEM is the physical address of the CGA text memory.
 #define CGATEXTMEM 0xB8000
@@ -107,66 +107,23 @@ start:
 	rep stosb
 	movl %edx, %edi

-	# Load GDT.
-	subl $8, %esp
-	movl $PADDR(bootgdt), 4(%esp)
-	movw $(8*NSEGS-1), 2(%esp)
-	lgdt 2(%esp)
-	addl $8, %esp
-
-	movl $KDSEG, %eax  // data segment selector
-	movw %ax, %ds
-	movw %ax, %es
-	movw %ax, %ss
-	movl $0, %eax  // null segment selector
-	movw %ax, %fs
-	movw %ax, %gs
+        call loadgdt

+       
 	# Enter new 32-bit code segment (already in 32-bit mode).
 	ljmp $KCSEG32, $PADDR(start32)  // code32 segment selector

 start32:
 	# Initialize page table.
 	call initpagetables
+        call init32e

-	# Initialize IA-32e mode.  See section 9.8.5 of 253668.pdf.
-	
-	# Set CR4.PAE and CR4.PSE = 1.
-	movl %cr4, %eax
-	orl $0x30, %eax
-	movl %eax, %cr4
-
-	# Load CR3 with physical base address of level 4 page table.
-	movl $PADDR(kpml4), %eax
-	movl %eax, %cr3
-	
-	# Enable IA-32e mode by setting IA32_EFER.LME = 1.
-	# Also turn on IA32_EFER.SCE (syscall enable).
-	movl $0xc0000080, %ecx
-	rdmsr
-	orl $0x101, %eax
-	wrmsr
-
-	# Enable paging by setting CR0.PG = 1.
-	movl %cr0, %eax
-	orl $0x80000000, %eax	
-	movl %eax, %cr0
-	nop
-	nop
-
+        movl $PADDR(start64), %edi
 	# Enter 64-bit mode.
-	ljmp $KCSEG, $PADDR(start64)  // code64 segment selector
+	ljmp $KCSEG, $PADDR(trap64)  // code64 segment selector

 .code64
 start64:
-	# The linker thinks we are running at start64, but we're actually
-	# running at PADDR(start64), so use an explicit calculation to
-	# load and jump to the correct address.
-	movq $KBASE, %rax
-	addq $PADDR(start64hi), %rax
-	jmp *%rax
-
-start64hi:
 	# Now executing at the real link address.
 	# Double-check that data segment is set up correctly.
 	movq $DATAMAGIC, %rax
@@ -177,14 +134,42 @@ start64hi:
 1:
 	# Load VA stack pointer
 	movabsq $(stack+STACK), %rsp
-	# Kill 1GB identity mapping
-	movq $0, kpml4
 	# Clear frame pointer for stack walks, and call into C code.
 	movq $0, %rbp
 	call cmain
 1:      jmp 1b

 .code32
+.globl apstart
+apstart:
+        call loadgdt
+	ljmp $KCSEG32, $PADDR(apstart32)  // code32 segment selector
+
+apstart32:
+        call init32e
+        movl $PADDR(apstart64), %edi
+	ljmp $KCSEG, $PADDR(trap64)  // code64 segment selector
+
+.code64       
+apstart64:
+	# Remember (from bootothers), that our kernel stack pointer is
+        # at the top of our temporary stack.
+        popq %rax
+        movq %rax, %rsp
+	movq $0, %rbp
+	call mpboot
+1:      jmp 1b
+        
+.code64
+trap64:
+	# The linker thinks we are running at trap64, but we're actually
+	# running at PADDR(trap64), so use an explicit calculation to
+	# load and jump to the correct address.  %rdi should hold the
+        # physical address of the jmp target.
+	addq $KBASE, %rdi
+	jmp *%rdi
+
+.code32
 .globl bootpanic
 bootpanic:
 	# This is 32-bit code but can be called from either
@@ -248,19 +233,6 @@ setcursor:
 	popl %edx
 	ret

-# Google color bar: spaces with background colors.
-colorbar:
-	.ascii " \x40 \x40 \x40 \x40 \x40 \x40 \x40 \x40 \x40 \x40"
-	.ascii " \x40 \x40 \x40 \x40 \x40 \x40 \x40 \x40 \x40 \x40"
-	.ascii " \x10 \x10 \x10 \x10 \x10 \x10 \x10 \x10 \x10 \x10"
-	.ascii " \x10 \x10 \x10 \x10 \x10 \x10 \x10 \x10 \x10 \x10"
-	.ascii " \x20 \x20 \x20 \x20 \x20 \x20 \x20 \x20 \x20 \x20"
-	.ascii " \x20 \x20 \x20 \x20 \x20 \x20 \x20 \x20 \x20 \x20"
-	.ascii "\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E"
-	.ascii "\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E"
-	.ascii "\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E"
-	.ascii "\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E\xDB\x0E"
-
 # Initial stack
 .comm stack, STACK

@@ -325,6 +297,50 @@ initpagetables:
 	popl %edi
 	ret

+# Initialize IA-32e mode.  See section 9.8.5 of 253668.pdf.
+init32e:
+        # Set CR4.PAE and CR4.PSE = 1.
+	movl %cr4, %eax
+	orl $0x30, %eax
+	movl %eax, %cr4
+
+	# Load CR3 with physical base address of level 4 page table.
+	movl $PADDR(kpml4), %eax
+	movl %eax, %cr3
+	
+	# Enable IA-32e mode by setting IA32_EFER.LME = 1.
+	# Also turn on IA32_EFER.SCE (syscall enable).
+	movl $0xc0000080, %ecx
+	rdmsr
+	orl $0x101, %eax
+	wrmsr
+
+	# Enable paging by setting CR0.PG = 1.
+	movl %cr0, %eax
+	orl $0x80000000, %eax	
+	movl %eax, %cr0
+	nop
+	nop
+
+        ret
+
+loadgdt:
+	subl $8, %esp
+	movl $PADDR(bootgdt), 4(%esp)
+	movw $(8*NSEGS-1), 2(%esp)
+	lgdt 2(%esp)
+	addl $8, %esp
+
+	movl $KDSEG, %eax  // data segment selector
+	movw %ax, %ds
+	movw %ax, %es
+	movw %ax, %ss
+	movl $0, %eax  // null segment selector
+	movw %ax, %fs
+	movw %ax, %gs
+
+        ret
+        
 # For sanity check above.
 .data
 datamagic:

--- a/bootother.S
+++ b/bootother.S
-#include "asm.h"
-#include "memlayout.h"
-
 # Each non-boot CPU ("AP") is started up in response to a STARTUP
 # IPI from the boot CPU.  Section B.4.2 of the Multi-Processor
 # Specification says that the AP will start in real mode with CS:IP
@@ -20,66 +17,50 @@
 #   - it uses the address at start-4 for the %esp
 #   - it jumps to the address at start-8 instead of calling bootmain

-#define SEG_KCODE 1
-#define SEG_KDATA 2
-
-#define CR0_PE    1
-
-#define RELOC1(x) ((x) + KERNBASE)    // same as V2P, but without casts
-
+#define SEG_ASM(type,base,lim)                                  \
+        .word (((lim) >> 12) & 0xffff), ((base) & 0xffff);      \
+        .byte (((base) >> 16) & 0xff), (0x90 | (type)),         \
+                (0xC0 | (((lim) >> 28) & 0xf)), (((base) >> 24) & 0xff)
+        
+#define PROT_MODE_CSEG 0x8
+#define PROT_MODE_DSEG 0x10
+#define CR0_PE_ON      0x1
+        
 .code16           
 .globl start
 start:
-  cli
-
-  ## movw $0xb800, %ax
-  ## movw %ax, %es
-  ## movw $65, %es:(0x0)
-
-  xorw    %ax,%ax
-  movw    %ax,%ds
-  movw    %ax,%es
-  movw    %ax,%ss
-
-  lgdt    gdtdesc
-  movl    %cr0, %eax
-  orl     $CR0_PE, %eax
-  movl    %eax, %cr0
-  ljmpl  $(SEG_KCODE<< 3), $(start32+KERNBASE)
+        cli
+        cld
+        
+	lgdt gdtdesc
+	movl %cr0, %eax
+	orl $CR0_PE_ON, %eax
+	movl %eax, %cr0
+        # Jump to next instruction, but in 32-bit code segment.
+        # Switches processor into 32-bit mode.
+	ljmp $PROT_MODE_CSEG, $protcseg

 .code32
-start32:
-  # movw $65, 0xb8002
-
-  movw    $(SEG_KDATA<<3), %ax
-
-  movw    %ax, %ds
-  movw    %ax, %es
-  movw    %ax, %ss
-  movw    $0, %ax
-  movw    %ax, %fs
-  movw    %ax, %gs
-
-  # switch to the stack allocated by bootothers()
-  movl    RELOC1(start-4), %esp
-
-  # call mpmain()
-  call	*(RELOC1(start)-8)
-
-  movw    $0x8a00, %ax
-  movw    %ax, %dx
-  outw    %ax, %dx
-  movw    $0x8ae0, %ax
-  outw    %ax, %dx
-spin:
-  jmp     spin
-
-.p2align 2
+protcseg:
+	movw $PROT_MODE_DSEG, %ax
+	movw %ax, %ds
+	movw %ax, %es
+	movw %ax, %fs
+	movw %ax, %gs
+	movw %ax, %ss
+
+        movl $start-12, %esp
+	movl start-4, %ecx
+	jmp *%ecx
+
+.align 4
 gdt:
-  SEG_NULLASM
-  SEG_ASM(STA_X|STA_R, -KERNBASE, 0xffffffff)
-  SEG_ASM(STA_W, -KERNBASE, 0xffffffff)
+	.quad 0 # null seg
+	SEG_ASM(0xa, 0x0, 0xffffffff) # code seg
+	SEG_ASM(0x2, 0x0, 0xffffffff) # data seg

+.align 16
 gdtdesc:
-  .word   (gdtdesc - gdt - 1)
-  .long   gdt
+	.word	0x17 # sizeof(gdt) - 1
+gdtaddr:
+	.long	gdt  # address gdt
--- a/kernel.h
+++ b/kernel.h
@@ -55,6 +55,7 @@ void            kmfree(void*);

 // lapic.c
 int             cpunum(void);
+void            lapicstartap(u8, u32 addr);

 // mp.c
 extern int      ncpu;

--- a/lapic.c
+++ b/lapic.c
@@ -167,8 +167,8 @@ void
 lapicstartap(u8 apicid, u32 addr)
 {
  int i;
-  u16 *wrv;
-  
+  volatile u16 *wrv;
+
  // "The BSP must initialize CMOS shutdown code to 0AH
  // and the warm reset vector (DWORD based at 40:67) to point at
  // the AP startup code prior to the [universal startup algorithm]."
@@ -181,9 +181,9 @@ lapicstartap(u8 apicid, u32 addr)
  // "Universal startup algorithm."
  // Send INIT (level-triggered) interrupt to reset other CPU.
  lapicw(ICRHI, apicid<<24);
-  lapicw(ICRLO, INIT | LEVEL | ASSERT);
+  lapicw(ICRLO, apicid | INIT | LEVEL | ASSERT);
  microdelay(200);
-  lapicw(ICRLO, INIT | LEVEL);
+  lapicw(ICRLO, apicid |INIT | LEVEL);
  microdelay(100);    // should be 10ms, but too slow in Bochs!
  
  // Send startup IPI (twice!) to enter bootstrap code.

--- a/main.c
+++ b/main.c
 #include "types.h"
+#include "param.h"
 #include "multiboot.h"
 #include "kernel.h"
+#include "cpu.h"

 extern void inituart(void);
 extern void initcga(void);
@@ -16,10 +18,60 @@ extern void initproc(void);
 extern void initbio(void);
 extern void inituser(void);

+static volatile int bstate;
+
+
+
+#if 1
+// Common CPU setup code.
+// Bootstrap CPU comes here from mainc().
+// Other CPUs jump here from bootother.S.
 void
-cmain(void)
+mpboot(void)
 {
+  // XXX(sbw) load VA for gdt, etc
+  panic("mpboot");
+}
+#endif
+
+// Start the non-boot processors.
+static void
+bootothers(void)
+{
+  extern u8 _binary_bootother_start[], _binary_bootother_size[];
+  extern void (*apstart)(void);
+  struct cpu *c;
+  char *stack;
+  u8 *code;
+
+  // Write bootstrap code to unused memory at 0x7000.
+  // The linker has placed the image of bootother.S in
+  // _binary_bootother_start.
+  code = p2v(0x7000);
+  memmove(code, _binary_bootother_start, (u64)_binary_bootother_size);

+  for(c = cpus; c < cpus+ncpu; c++){
+    if(c == cpus+cpunum())  // We've started already.
+      continue;
+
+    // Tell bootother.S what stack to use and the address of apstart;
+    // it expects to find these two addresses stored just before
+    // its first instruction.
+    stack = kalloc();
+    *(u32*)(code-4) = (u32)v2p(&apstart);
+    *(u64*)(code-12) = (u64)stack + KSTACKSIZE;
+
+    bstate = 0;
+    lapicstartap(c->id, v2p(code));
+    // Wait for cpu to finish mpmain()
+    while(bstate == 0)
+      ;
+  }
+}
+
+void
+cmain(void)
+{
  inituart();
  initcga();
  initconsole();
@@ -33,6 +85,7 @@ cmain(void)
  initrcu();       // initialize rcu module
  initproc();      // process table
  initbio();       // buffer cache
+
 #if 0
  fileinit();      // file table
  iinit();         // inode cache
@@ -40,6 +93,9 @@ cmain(void)
 #endif

  inituser();      // first user process
+  bootothers();    // start other processors
+  
+  // XXX(sbw) kill ident. mapping
  
  cprintf("ncpu %d\n", ncpu);
  panic("end");