The unified diff between revisions [7460dbcf..] and [e6beb7c9..] is displayed below. It can also be downloaded as a raw diff.
#
#
# patch "src/fs/coss/store_coss.h"
# from [866cd796dde6cce04a8cd07d69e5724e7cb1aa39]
# to [f54c5c80a387b623ee3abf42ac565585a41df44f]
#
# patch "src/fs/coss/store_dir_coss.c"
# from [d4ba5a7e85033b9d9a931c5c6c7370579daa1e00]
# to [fb849ed24cdd3b4f19ee54a48d3d6d2ba24ba36c]
#
# patch "src/fs/coss/store_io_coss.c"
# from [2c3038b076f7281b9a40222f3b57730ee3ce9edf]
# to [d903b5fc046ce7be75b5c1584e88fec70a8c0a22]
#
============================================================
--- src/fs/coss/store_coss.h 866cd796dde6cce04a8cd07d69e5724e7cb1aa39
+++ src/fs/coss/store_coss.h f54c5c80a387b623ee3abf42ac565585a41df44f
@@ -195,7 +195,7 @@ extern STSYNC storeCossSync;
extern STOBJUNLINK storeCossUnlink;
extern STSYNC storeCossSync;
-extern void storeCossAdd(SwapDir *, StoreEntry *);
+extern void storeCossAdd(SwapDir * sd, StoreEntry * e, int curstripe);
extern void storeCossRemove(SwapDir *, StoreEntry *);
extern void storeCossStartMembuf(SwapDir * SD);
extern void membufsDump(CossInfo *cs, StoreEntry *e);
============================================================
--- src/fs/coss/store_dir_coss.c d4ba5a7e85033b9d9a931c5c6c7370579daa1e00
+++ src/fs/coss/store_dir_coss.c fb849ed24cdd3b4f19ee54a48d3d6d2ba24ba36c
@@ -59,6 +59,12 @@ struct _RebuildState {
unsigned int clean:1;
} flags;
struct _store_rebuild_data counts;
+ struct {
+ int new;
+ int reloc;
+ int fresher;
+ int unknown;
+ } cosscounts;
};
static char *storeCossDirSwapLogFile(SwapDir *, const char *);
@@ -198,14 +204,14 @@ void
}
void
-storeCossAdd(SwapDir * sd, StoreEntry * e)
+storeCossAdd(SwapDir * sd, StoreEntry * e, int curstripe)
{
CossInfo *cs = (CossInfo *) sd->fsdata;
- CossStripe *cstripe = &cs->stripes[cs->curstripe];
+ CossStripe *cstripe = &cs->stripes[curstripe];
CossIndexNode *coss_node = memPoolAlloc(coss_index_pool);
assert(!e->repl.data);
/* Make sure the object exists in the current stripe, it should do! */
- assert(cs->curstripe == storeCossFilenoToStripe(cs, e->swap_filen));
+ assert(curstripe == storeCossFilenoToStripe(cs, e->swap_filen));
e->repl.data = coss_node;
dlinkAddTail(e, &coss_node->node, &cstripe->objlist);
cs->count += 1;
@@ -215,11 +221,14 @@ storeCossRebuildComplete(void *data)
storeCossRebuildComplete(void *data)
{
RebuildState *rb = data;
- SwapDir *sd = rb->sd;
- storeCossStartMembuf(sd);
+ SwapDir *SD = rb->sd;
+ storeCossStartMembuf(SD);
store_dirs_rebuilding--;
- storeCossDirCloseTmpSwapLog(rb->sd);
+ storeCossDirCloseTmpSwapLog(SD);
storeRebuildComplete(&rb->counts);
+ debug(47, 1) ("COSS: %s: Rebuild Completed\n", SD->path);
+ debug(47, 1) (" %d objects scanned, %d objects relocated, %d objects fresher, %d objects ignored\n",
+ rb->counts.scancount, rb->cosscounts.reloc, rb->cosscounts.fresher, rb->cosscounts.unknown);
cbdataFree(rb);
}
@@ -883,6 +892,7 @@ static void storeDirCoss_ParseStripeBuff
static void storeDirCoss_ReadStripe(RebuildState *rb);
static void storeDirCoss_ParseStripeBuffer(RebuildState *rb);
+static void storeCoss_ConsiderStoreEntry(RebuildState *rb, const cache_key *key, StoreEntry *e);
static void
storeDirCoss_ReadStripeComplete(int fd, const char *buf, int r_len, int r_errflag, void *my_data)
@@ -891,10 +901,10 @@ storeDirCoss_ReadStripeComplete(int fd,
SwapDir *SD = rb->sd;
CossInfo *cs = SD->fsdata;
- debug(47, 1) ("COSS: %s: stripe %d, read %d bytes, status %d\n", SD->path, cs->rebuild.curstripe, r_len, r_errflag);
+ debug(47, 2) ("COSS: %s: stripe %d, read %d bytes, status %d\n", SD->path, cs->rebuild.curstripe, r_len, r_errflag);
cs->rebuild.reading = 0;
if (r_errflag != DISK_OK) {
- debug(47, 1) ("COSS: %s: stripe %d: error! Ignoring objects in this stripe.\n", SD->path, cs->rebuild.curstripe);
+ debug(47, 2) ("COSS: %s: stripe %d: error! Ignoring objects in this stripe.\n", SD->path, cs->rebuild.curstripe);
goto nextstripe;
}
cs->rebuild.buflen = r_len;
@@ -905,8 +915,9 @@ nextstripe:
cs->rebuild.curstripe++;
if (cs->rebuild.curstripe >= cs->numstripes) {
/* Completed the rebuild - move onto the next phase */
- debug(47, 1) ("COSS: %s: completed reading the stripes.\n", SD->path);
- fatal("Done!");
+ debug(47, 2) ("COSS: %s: completed reading the stripes.\n", SD->path);
+ storeCossRebuildComplete(rb);
+ return;
} else {
/* Next stripe */
storeDirCoss_ReadStripe(rb);
@@ -922,7 +933,7 @@ storeDirCoss_ReadStripe(RebuildState *rb
assert(cs->rebuild.reading == 0);
cs->rebuild.reading = 1;
/* Use POSIX AIO for now */
- debug(47, 1) ("COSS: %s: reading stripe %d\n", SD->path, cs->rebuild.curstripe);
+ debug(47, 2) ("COSS: %s: reading stripe %d\n", SD->path, cs->rebuild.curstripe);
a_file_read(&cs->aq, cs->fd, cs->rebuild.buf, COSS_MEMBUF_SZ, cs->rebuild.curstripe * COSS_MEMBUF_SZ, storeDirCoss_ReadStripeComplete, rb);
}
@@ -939,7 +950,7 @@ storeDirCoss_StartDiskRebuild(RebuildSta
cs->rebuild.curstripe = 0;
cs->rebuild.buf = xmalloc(COSS_MEMBUF_SZ);
- debug(47, 1) ("COSS: %s: Beginning disk rebuild.\n", SD->path);
+ debug(47, 2) ("COSS: %s: Beginning disk rebuild.\n", SD->path);
storeDirCoss_ReadStripe(rb);
}
@@ -957,18 +968,16 @@ storeDirCoss_ParseStripeBuffer(RebuildSt
int tmp;
squid_off_t *l, len;
int blocksize = cs->blksz_mask + 1;
- StoreEntry *e = NULL;
StoreEntry tmpe;
cache_key key[MD5_DIGEST_CHARS];
+ sfileno filen;
assert(cs->rebuild.rebuilding == 1);
assert(cs->numstripes > 0);
assert(cs->rebuild.buf != NULL);
- debug(47, 1) ("COSS: debug: blocksize is %d\n", blocksize);
-
if (cs->rebuild.buflen == 0) {
- debug(47, 1) ("COSS: %s: stripe %d: read 0 bytes, skipping stripe\n", SD->path, cs->rebuild.curstripe);
+ debug(47, 3) ("COSS: %s: stripe %d: read 0 bytes, skipping stripe\n", SD->path, cs->rebuild.curstripe);
return;
}
@@ -979,10 +988,11 @@ storeDirCoss_ParseStripeBuffer(RebuildSt
/* XXX there's no bounds checking on the buffer being passed into storeSwapMetaUnpack! */
tlv_list = storeSwapMetaUnpack(cs->rebuild.buf + j, &bl);
if (tlv_list == NULL) {
- debug(47, 1) ("COSS: %s: stripe %d: offset %d gives NULL swapmeta data; end of stripe\n", SD->path, cs->rebuild.curstripe, j);
+ debug(47, 3) ("COSS: %s: stripe %d: offset %d gives NULL swapmeta data; end of stripe\n", SD->path, cs->rebuild.curstripe, j);
return;
}
- debug(47, 1) ("COSS: %s: stripe %d: filen %d: header size %d\n", SD->path, cs->rebuild.curstripe, j / blocksize + (cs->rebuild.curstripe * COSS_MEMBUF_SZ / blocksize), bl);
+ filen = j / blocksize + (cs->rebuild.curstripe * COSS_MEMBUF_SZ / blocksize);
+ debug(47, 3) ("COSS: %s: stripe %d: filen %d: header size %d\n", SD->path, cs->rebuild.curstripe, filen, bl);
/* COSS objects will have an object size written into the metadata */
bzero(&tmpe, sizeof(tmpe));
@@ -990,11 +1000,11 @@ storeDirCoss_ParseStripeBuffer(RebuildSt
for (t = tlv_list; t; t = t->next) {
switch(t->type) {
case STORE_META_URL:
- debug(47, 1) (" URL: %s\n", (char *)t->value);
+ debug(47, 3) (" URL: %s\n", (char *)t->value);
break;
case STORE_META_OBJSIZE:
l = t->value;
- debug(47, 1) ("Size: %lld (len %d)\n", *l, t->length);
+ debug(47, 3) ("Size: %lld (len %d)\n", *l, t->length);
break;
case STORE_META_KEY:
assert(t->length == MD5_DIGEST_CHARS);
@@ -1037,7 +1047,7 @@ storeDirCoss_ParseStripeBuffer(RebuildSt
}
/* Make sure we have an object; if we don't then it may be an indication of trouble */
if (l == NULL) {
- debug(47, 1) ("COSS: %s: stripe %d: Object with no size; end of stripe\n", SD->path, cs->rebuild.curstripe);
+ debug(47, 3) ("COSS: %s: stripe %d: Object with no size; end of stripe\n", SD->path, cs->rebuild.curstripe);
storeSwapTLVFree(tlv_list);
return;
}
@@ -1046,32 +1056,35 @@ storeDirCoss_ParseStripeBuffer(RebuildSt
* we've just been informed about
*/
if (cs->rebuild.buflen - j < len) {
- debug(47, 1) ("COSS: %s: stripe %d: Not enough data in this stripe for this object, bye bye.\n", SD->path, cs->rebuild.curstripe);
+ debug(47, 3) ("COSS: %s: stripe %d: Not enough data in this stripe for this object, bye bye.\n", SD->path, cs->rebuild.curstripe);
storeSwapTLVFree(tlv_list);
return;
}
/* Houston, we have an object */
if (storeKeyNull(key)) {
- debug(47, 1) ("COSS: %s: stripe %d: null data, next!\n", SD->path, cs->rebuild.curstripe);
+ debug(47, 3) ("COSS: %s: stripe %d: null data, next!\n", SD->path, cs->rebuild.curstripe);
goto nextobject;
}
+ rb->counts.scancount++;
tmpe.hash.key = key;
/* Check sizes */
if (tmpe.swap_file_sz == 0) {
tmpe.swap_file_sz = len;
}
if (tmpe.swap_file_sz != len) {
- debug(47, 1) ("COSS: %s: stripe %d: file size mismatch (%d != %d)\n", SD->path, cs->rebuild.curstripe, (int) tmpe.swap_file_sz, (int) len);
+ debug(47, 3) ("COSS: %s: stripe %d: file size mismatch (%d != %d)\n", SD->path, cs->rebuild.curstripe, (int) tmpe.swap_file_sz, (int) len);
goto nextobject;
}
if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
- debug(47, 1) ("COSS: %s: stripe %d: private key flag set, ignoring.\n", SD->path, cs->rebuild.curstripe);
+ debug(47, 3) ("COSS: %s: stripe %d: private key flag set, ignoring.\n", SD->path, cs->rebuild.curstripe);
rb->counts.badflags++;
goto nextobject;
}
- /* Next; check for clashes: we might need to overwrite this object! */
- e = storeGet(key);
+ /* Time to consider the object! */
+ tmpe.swap_filen = filen;
+ tmpe.swap_dirn = SD->index;
+ storeCoss_ConsiderStoreEntry(rb, key, &tmpe);
nextobject:
/* Free the TLV data */
@@ -1086,3 +1099,107 @@ nextobject:
j = tmp;
}
}
+
+
+static void
+storeCoss_AddStoreEntry(RebuildState *rb, const cache_key *key, StoreEntry *e)
+{
+ StoreEntry *ne;
+ SwapDir *SD = rb->sd;
+ CossInfo *cs = SD->fsdata;
+ rb->counts.objcount++;
+ /* The Passed-in store entry is temporary; don't bloody use it directly! */
+ ne = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
+ ne->store_status = STORE_OK;
+ storeSetMemStatus(ne, NOT_IN_MEMORY);
+ ne->swap_status = SWAPOUT_DONE;
+ ne->swap_filen = e->swap_filen;
+ ne->swap_dirn = SD->index;
+ ne->swap_file_sz = e->swap_file_sz;
+ ne->lock_count = 0;
+ ne->lastref = e->lastref;
+ ne->timestamp = e->timestamp;
+ ne->expires = e->expires;
+ ne->lastmod = e->lastmod;
+ ne->refcount = e->refcount;
+ ne->flags = e->flags;
+ EBIT_SET(ne->flags, ENTRY_CACHABLE);
+ EBIT_CLR(ne->flags, RELEASE_REQUEST);
+ EBIT_CLR(ne->flags, KEY_PRIVATE);
+ ne->ping_status = PING_NONE;
+ EBIT_CLR(ne->flags, ENTRY_VALIDATED);
+ storeHashInsert(ne, key); /* do it after we clear KEY_PRIVATE */
+ storeCossAdd(SD, ne, cs->rebuild.curstripe);
+
+ storeEntryDump(ne, 5);
+}
+
+static void
+storeCoss_DeleteStoreEntry(RebuildState *rb, const cache_key *key, StoreEntry *e)
+{
+ SwapDir *SD = rb->sd;
+ assert(rb->counts.objcount >= 0);
+ rb->counts.objcount--;
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ storeCossRemove(SD, e);
+ storeRelease(e);
+}
+
+/*
+ * Consider inserting the given StoreEntry into the given
+ * COSS directory.
+ *
+ * The rules for doing this is reasonably simple:
+ *
+ * If the object doesn't exist in the cache then we simply
+ * add it to the current stripe list
+ *
+ * If the object does exist in the cache then we compare
+ * "freshness"; if the newer object is fresher then we
+ * remove it from its stripe and re-add it to the current
+ * stripe.
+ */
+static void
+storeCoss_ConsiderStoreEntry(RebuildState *rb, const cache_key *key, StoreEntry *e)
+{
+ SwapDir *SD = rb->sd;
+ StoreEntry *oe;
+
+ /* Check for clashes */
+ oe = storeGet(key);
+ if (oe == NULL) {
+ rb->cosscounts.new++;
+ /* no clash! woo, can add and forget */
+ storeCoss_AddStoreEntry(rb, key, e);
+ return;
+ }
+ assert(oe->swap_dirn == SD->index);
+
+ /* Dang, its a clash. See if its fresher */
+
+ /* Fresher? Its a new object: deallocate the old one, reallocate the new one */
+ if (e->lastref > oe->lastref) {
+ debug(47, 3) ("COSS: fresher object for filen %d found (%d -> %d)\n", oe->swap_filen, (int) oe->timestamp, (int) e->timestamp);
+ rb->cosscounts.fresher++;
+ storeCoss_DeleteStoreEntry(rb, key, oe);
+ oe = NULL;
+ storeCoss_AddStoreEntry(rb, key, e);
+ return;
+ }
+
+ /*
+ * Not fresher? Its the same object then we /should/ probably relocate it; I'm
+ * not sure what should be done here.
+ */
+ if (oe->timestamp == e->timestamp && oe->expires == e->expires) {
+ debug(47, 3) ("COSS: filen %d -> %d (since they're the same!)\n", oe->swap_filen, e->swap_filen);
+ rb->cosscounts.reloc++;
+ storeCoss_DeleteStoreEntry(rb, key, oe);
+ oe = NULL;
+ storeCoss_AddStoreEntry(rb, key, e);
+ return;
+ }
+ debug(47, 3) ("COSS: filen %d: ignoring this one for some reason\n", e->swap_filen);
+ rb->cosscounts.unknown++;
+}
============================================================
--- src/fs/coss/store_io_coss.c 2c3038b076f7281b9a40222f3b57730ee3ce9edf
+++ src/fs/coss/store_io_coss.c d903b5fc046ce7be75b5c1584e88fec70a8c0a22
@@ -179,6 +179,7 @@ storeCossCreate(SwapDir * SD, StoreEntry
{
CossState *cstate;
storeIOState *sio;
+ CossInfo *cs = SD->fsdata;
coss_stats.create.ops++;
sio = cbdataAlloc(storeIOState);
@@ -216,7 +217,7 @@ storeCossCreate(SwapDir * SD, StoreEntry
/* Now add it into the index list */
e->swap_filen = sio->swap_filen;
e->swap_dirn = sio->swap_dirn;
- storeCossAdd(SD, e);
+ storeCossAdd(SD, e, cs->curstripe);
storeCossMemBufLock(SD, sio);
coss_stats.create.success++;
@@ -305,7 +306,7 @@ storeCossOpen(SwapDir * SD, StoreEntry *
* Do the index magic to keep the disk and memory LRUs identical
* by adding the object into the link list on the current stripe
*/
- storeCossAdd(SD, e);
+ storeCossAdd(SD, e, cs->curstripe);
}
coss_stats.open.success++;
return sio;