ditch bcache refcounts, use rcu

d564ca3d · Nickolai Zeldovich · ae6af083 · d564ca3d · d564ca3d · d564ca3d
--- a/bio.c
+++ b/bio.c
@@ -54,7 +54,7 @@ evict(uint key, void *bp)
 {
  struct buf *b = bp;
  acquire(&b->lock);
-  if ((b->flags & (B_BUSYR | B_BUSYW | B_VALID)) == 0)
+  if ((b->flags & (B_BUSY | B_VALID)) == 0)
    return b;
  release(&b->lock);
  return 0;
@@ -65,7 +65,7 @@ evict_valid(uint key, void *bp)
 {
  struct buf *b = bp;
  acquire(&b->lock);
-  if ((b->flags & (B_BUSYR | B_BUSYW)) == 0)
+  if ((b->flags & (B_BUSY)) == 0)
    return b;
  release(&b->lock);
  return 0;
@@ -75,7 +75,7 @@ evict_valid(uint key, void *bp)
 // If not found, allocate fresh block.
 // In either case, return locked buffer.
 static struct buf*
-bget(uint dev, uint sector, int writer)
+bget(uint dev, uint sector, int *writer)
 {
  struct buf *b;
@@ -85,21 +85,24 @@ bget(uint dev, uint sector, int writer)
  rcu_begin_read();
  b = ns_lookup(bufns, sector);
  if (b) {
-    acquire(&b->lock);
    if (b->dev != dev)
      panic("dev mismatch");
-    if (!(b->flags & (B_BUSYW | (writer ? B_BUSYR : 0)))) {
+    if (*writer || !(b->flags & B_VALID)) {
-      b->flags |= B_BUSYR | (writer ? B_BUSYW : 0);
+      acquire(&b->lock);
-      __sync_fetch_and_add(&b->readbusy, 1);
+      if (b->flags & B_BUSY) {
+	cv_sleep(&b->cv, &b->lock);
 	release(&b->lock);
 	rcu_end_read();
-      return b;
+	goto loop;
      }
-    cv_sleep(&b->cv, &b->lock);
+      b->flags |= B_BUSY;
      release(&b->lock);
-    rcu_end_read();
+      *writer = 1;
-    goto loop;
+    }
+    // rcu_end_read() happens in brelse
+    return b;
  }
  rcu_end_read();
@@ -109,8 +112,7 @@ bget(uint dev, uint sector, int writer)
    victim = ns_enumerate(bufns, evict_valid);
  if (victim == 0)
    panic("bget all busy");
-  victim->flags |= B_BUSYR | B_BUSYW;
+  victim->flags |= B_BUSY;
-  __sync_fetch_and_add(&victim->readbusy, 1);
  ns_remove(bufns, victim->sector, victim);
  release(&victim->lock);
  rcu_delayed(victim, kmfree);
@@ -118,15 +120,17 @@ bget(uint dev, uint sector, int writer)
  b = kmalloc(sizeof(*b));
  b->dev = dev;
  b->sector = sector;
-  b->flags = B_BUSYR | B_BUSYW;
+  b->flags = B_BUSY;
-  b->readbusy = 1;
+  *writer = 1;
  snprintf(b->lockname, sizeof(b->lockname), "cv:buf:%d", b->sector);
  initlock(&b->lock, b->lockname+3);
  initcondvar(&b->cv, b->lockname);
+  rcu_begin_read();
  if (ns_insert(bufns, b->sector, b) < 0) {
    rcu_delayed(b, kmfree);
    goto loop;
  }
+  // rcu_end_read() happens in brelse
  return b;
 }
@@ -136,9 +140,15 @@ bread(uint dev, uint sector, int writer)
 {
  struct buf *b;
-  b = bget(dev, sector, writer);
+  int origwriter = writer;
+  b = bget(dev, sector, &writer);
  if(!(b->flags & B_VALID))
    iderw(b);
+  if (writer && !origwriter) {
+    acquire(&b->lock);
+    b->flags &= ~B_BUSY;
+    release(&b->lock);
+  }
  return b;
 }
@@ -146,7 +156,7 @@ bread(uint dev, uint sector, int writer)
 void
 bwrite(struct buf *b)
 {
-  if((b->flags & B_BUSYW) == 0)
+  if((b->flags & B_BUSY) == 0)
    panic("bwrite");
  b->flags |= B_DIRTY;
  if (writeback)
@@ -155,14 +165,17 @@ bwrite(struct buf *b)
 // Release the buffer b.
 void
-brelse(struct buf *b)
+brelse(struct buf *b, int writer)
 {
+  if (writer) {
    acquire(&b->lock);
-  if((b->flags & (B_BUSYR | B_BUSYW)) == 0)
+    if((b->flags & B_BUSY) == 0)
      panic("brelse");
-  int lastreader = __sync_sub_and_fetch(&b->readbusy, 1);
+    b->flags &= ~B_BUSY;
-  b->flags &= ~(B_BUSYW | ((lastreader==0) ? B_BUSYR : 0));
    release(&b->lock);
    cv_wakeup(&b->cv);
+  }
+  // rcu_begin_read() happens in bread
+  rcu_end_read();
 }
--- a/buf.h
+++ b/buf.h
@@ -2,7 +2,6 @@ struct buf {
  int flags;
  uint dev;
  uint sector;
-  int readbusy;
  struct buf *prev; // LRU cache list
  struct buf *next;
  struct buf *qnext; // disk queue
@@ -11,8 +10,7 @@ struct buf {
  struct spinlock lock;
  uchar data[512];
 };
-#define B_BUSYR 0x1  // buffer is locked by some process
+#define B_BUSY  0x1  // buffer is locked by some process
-#define B_BUSYW 0x2  // buffer is locked by some process
+#define B_VALID 0x2  // buffer has been read from disk
-#define B_VALID 0x4  // buffer has been read from disk
+#define B_DIRTY 0x4  // buffer needs to be written to disk
-#define B_DIRTY 0x8  // buffer needs to be written to disk
--- a/condvar.c
+++ b/condvar.c
@@ -52,6 +52,9 @@ cv_sleep(struct condvar *cv, struct spinlock *lk)
 void
 cv_wakeup(struct condvar *cv)
 {
+  if (!cv->waiters)
+    return;
  acquire(&cv->lock);
  while(cv->waiters) {
    struct proc *p = cv->waiters;

--- a/defs.h
+++ b/defs.h
@@ -15,7 +15,7 @@ struct ns;
 // bio.c
 void            binit(void);
 struct buf*     bread(uint, uint, int writer);
-void            brelse(struct buf*);
+void            brelse(struct buf*, int writer);
 void            bwrite(struct buf*);
 // condvar.c

--- a/fs.c
+++ b/fs.c
@@ -34,7 +34,7 @@ readsb(int dev, struct superblock *sb)
  bp = bread(dev, 1, 0);
  memmove(sb, bp->data, sizeof(*sb));
-  brelse(bp);
+  brelse(bp, 0);
 }
 // Zero a block.
@@ -46,7 +46,7 @@ bzero(int dev, int bno)
  bp = bread(dev, bno, 1);
  memset(bp->data, 0, BSIZE);
  bwrite(bp);
-  brelse(bp);
+  brelse(bp, 1);
 }
 // Blocks. 
@@ -68,11 +68,11 @@ balloc(uint dev)
      if((bp->data[bi/8] & m) == 0){  // Is block free?
        bp->data[bi/8] |= m;  // Mark block in use on disk.
        bwrite(bp);
-        brelse(bp);
+        brelse(bp, 1);
        return b + bi;
      }
    }
-    brelse(bp);
+    brelse(bp, 1);
  }
  panic("balloc: out of blocks");
 }
@@ -95,7 +95,7 @@ bfree(int dev, uint b)
    panic("freeing free block");
  bp->data[bi/8] &= ~m;  // Mark block free on disk.
  bwrite(bp);
-  brelse(bp);
+  brelse(bp, 1);
 }
 // Inodes.
@@ -160,10 +160,10 @@ ialloc(uint dev, short type)
  readsb(dev, &sb);
  for(inum = 1; inum < sb.ninodes; inum++){  // loop over inode blocks
-    bp = bread(dev, IBLOCK(inum), 1);
+    bp = bread(dev, IBLOCK(inum), 0);
    dip = (struct dinode*)bp->data + inum%IPB;
    int seemsfree = (dip->type == 0);
-    brelse(bp);
+    brelse(bp, 0);
    if(seemsfree){
      // maybe this inode is free. look at it via the
      // inode cache to make sure.
@@ -201,7 +201,7 @@ iupdate(struct inode *ip)
  dip->gen = ip->gen;
  memmove(dip->addrs, ip->addrs, sizeof(ip->addrs));
  bwrite(bp);
-  brelse(bp);
+  brelse(bp, 1);
 }
 static void *
@@ -277,7 +277,7 @@ iget(uint dev, uint inum)
  ip->size = dip->size;
  ip->gen = dip->gen;
  memmove(ip->addrs, dip->addrs, sizeof(ip->addrs));
-  brelse(bp);
+  brelse(bp, 0);
  ip->flags |= I_VALID;
  iunlock(ip);
@@ -400,7 +400,7 @@ bmap(struct inode *ip, uint bn)
      a[bn] = addr = balloc(ip->dev);
      bwrite(bp);
    }
-    brelse(bp);
+    brelse(bp, 1);
    return addr;
  }
@@ -431,7 +431,7 @@ itrunc(struct inode *ip)
      if(a[j])
        bfree(ip->dev, a[j]);
    }
-    brelse(bp);
+    brelse(bp, 0);
    bfree(ip->dev, ip->addrs[NDIRECT]);
    ip->addrs[NDIRECT] = 0;
  }
@@ -474,7 +474,7 @@ readi(struct inode *ip, char *dst, uint off, uint n)
    bp = bread(ip->dev, bmap(ip, off/BSIZE), 0);
    m = min(n - tot, BSIZE - off%BSIZE);
    memmove(dst, bp->data + off%BSIZE, m);
-    brelse(bp);
+    brelse(bp, 0);
  }
  return n;
 }
@@ -503,7 +503,7 @@ writei(struct inode *ip, char *src, uint off, uint n)
    m = min(n - tot, BSIZE - off%BSIZE);
    memmove(bp->data + off%BSIZE, src, m);
    bwrite(bp);
-    brelse(bp);
+    brelse(bp, 1);
  }
  if(n > 0 && off > ip->size){
@@ -547,11 +547,11 @@ dirlookup(struct inode *dp, char *name, uint *poff)
        if(poff)
          *poff = off + (uchar*)de - bp->data;
        inum = de->inum;
-        brelse(bp);
+        brelse(bp, 0);
        return iget(dp->dev, inum);
      }
    }
-    brelse(bp);
+    brelse(bp, 0);
  }
  return 0;
 }

--- a/ide.c
+++ b/ide.c
@@ -130,7 +130,7 @@ iderw(struct buf *b)
 {
  struct buf **pp;
-  if(!(b->flags & (B_BUSYR | B_BUSYW)))
+  if(!(b->flags & B_BUSY))
    panic("iderw: buf not busy");
  if((b->flags & (B_VALID|B_DIRTY)) == B_VALID)
    panic("iderw: nothing to do");

--- a/main.c
+++ b/main.c
@@ -62,6 +62,7 @@ mainc(void)
  ideinit();       // disk
  if(!ismp)
    timerinit();   // uniprocessor timer
+  nc_init();
  userinit();      // first user process
  bootothers();    // start other processors
  kvmalloc();      // new kernel page table wo. bottom mapped