The unified diff between revisions [20e9456f..] and [53770148..] is displayed below. It can also be downloaded as a raw diff.
#
#
# add_file "doc/README.diskload"
# content [a508e639e8882f32ac170fb695a3550052212abe]
#
# patch "configure"
# from [90ce6b51a826374b92dae5ce5d4705b701257b80]
# to [d11a62d61ba798a867f99bc7254e96dafa31e860]
#
# patch "configure.in"
# from [905c235650c23ccd4f42fb2ef884037330f06161]
# to [cdb446006a4d51b315d0115d3e3ec35b1e7b2627]
#
# patch "src/client_side.c"
# from [882d42c7767308a3036106324457a687d15d52b0]
# to [f2f0d2e1e191ffeec9c101a29a66353d9b9cdc0d]
#
# patch "src/enums.h"
# from [b48c01f14f67bf878e17c8fd8a3d1aac10f67ab9]
# to [cdbb572d3e510c8b6c856c755c8ce78577f6b838]
#
# patch "src/fs/aufs/store_dir_aufs.c"
# from [99f2f1eebf5dd0b8e29deb65369dcd0c44bb0cd6]
# to [434e3a5f048cd6111395e88fb1397144b4f2cc14]
#
# patch "src/fs/coss/store_dir_coss.c"
# from [4ed0002caad3b46a4fc8fc9e1a5c64c4926e93bc]
# to [e0a68cfd02a72baa7b392a368136d5a204c53495]
#
# patch "src/fs/diskd/store_dir_diskd.c"
# from [b338483de8d86584e890188328fb92711677fc7f]
# to [29790dc96bd0f60cad3f36d201be8a32655cf28d]
#
# patch "src/fs/null/store_null.c"
# from [fa5f237c80376f490212dc9b5431584958991925]
# to [eda38ac409bcb678d4d853ce97e8eecb25ee57e7]
#
# patch "src/fs/ufs/store_dir_ufs.c"
# from [4793d2c314a1b2a0085408592b65bb858ab30058]
# to [7fea6eabaa15ef88ff1cbbc3b2a8f95da3419aa6]
#
# patch "src/pconn.c"
# from [5f83c46e535d572c2a6d6f55389e635926c3feee]
# to [8c9ab2458a98fedd3b53d1e42fdef52ea0235e66]
#
# patch "src/protos.h"
# from [822ab2931a4c6df49be0b07f482586d3a7bf04cb]
# to [aa65dda706bf16d368332b30fcdf514d4a6ed0b3]
#
# patch "src/store_dir.c"
# from [e795049d82fdee67d27f34a53aed344c9c60f488]
# to [299ef0c831c2b8f781cfe9992ca0912e5baa5792]
#
# patch "src/store_io.c"
# from [b3189954bb7b33712f066ccf62783d2b0506f710]
# to [8a31eb00c0678314dbf38b0fa41d2bbd3f96f3b9]
#
# patch "src/structs.h"
# from [09fd7df0ee0cc66a8be6be822cf34b2203aac563]
# to [2cc0257c851aa9f3899204538a4037888a0465cf]
#
# patch "src/typedefs.h"
# from [0a1930b4adf4cddddda7f44324e88fb53311162f]
# to [af3aabd123fce77fbe2ce6bfc45c942e516c8640]
#
============================================================
--- doc/README.diskload a508e639e8882f32ac170fb695a3550052212abe
+++ doc/README.diskload a508e639e8882f32ac170fb695a3550052212abe
@@ -0,0 +1,37 @@
+
+The disk load calculation code has been modified slightly - it now
+attempts to shed load during both periods of extraordinarily busy
+disk activity.
+
+Each filesystem implements slightly different semantics for knowing
+when its busy:
+
+* aufs: doesn't have a concept of 'per-storedir' IO as the thread
+ pool is shared between all aufs storedir; loadav is the number
+ of currently busy threads out of the number of maximum threads.
+
+* ufs: since it is synchronous the "busy" heuristic is simply the
+ number of currently open files in the storedir. This allows for
+ the usage of the storedirs to be relatively even but it doesn't
+ help manage disk load.
+
+* diskd: each storedir has its own SYSV SHM/MSG queue; so the load
+ average is calculated based on the number of currently queued
+ messages compared to magic1 and magic2.
+
+* coss: each coss storedir has its own async-io queue (currently
+ 128 entries deep); load average the percentage of these in use
+
+* null: the cache is always busy.
+
+The main problem with this; Squid will invalidate cached objects
+if there's a problem swapping them in; so heavy disk load can cause
+both your hit rate to drop and your cache to start losing objects.
+
+This isn't easily solvable! We'd need to be able to compare what
+we've retrieved against what we have (kind of like an IMS) to see
+whether we've fetched something "fresher". If we have then we should
+invalidate what we have in the cache. If the object we've fetched is
+just as fresh as what we have in the cache then we should hold onto
+our cached object. This will require a little more thought.
+
============================================================
--- configure 90ce6b51a826374b92dae5ce5d4705b701257b80
+++ configure d11a62d61ba798a867f99bc7254e96dafa31e860
@@ -1026,7 +1026,7 @@ PACKAGE=squid
# Define the identity of the package.
PACKAGE=squid
-VERSION=2.5.STABLE13-CVS-CACHEBOY
+VERSION=2.5.STABLE13-CACHEBOY_1.0.PRE1
cat >> confdefs.h <<EOF
#define PACKAGE "$PACKAGE"
EOF
============================================================
--- configure.in 905c235650c23ccd4f42fb2ef884037330f06161
+++ configure.in cdb446006a4d51b315d0115d3e3ec35b1e7b2627
@@ -9,7 +9,7 @@ AC_CONFIG_AUX_DIR(cfgaux)
dnl
AC_INIT(src/main.c)
AC_CONFIG_AUX_DIR(cfgaux)
-AM_INIT_AUTOMAKE(squid, 2.5.STABLE13-CVS-CACHEBOY)
+AM_INIT_AUTOMAKE(squid, 2.5.STABLE13-CACHEBOY_1.0.PRE1)
AM_CONFIG_HEADER(include/autoconf.h)
AC_REVISION($Revision: 1.251.2.107 $)dnl
AC_PREFIX_DEFAULT(/usr/local/squid)
============================================================
--- src/client_side.c 882d42c7767308a3036106324457a687d15d52b0
+++ src/client_side.c f2f0d2e1e191ffeec9c101a29a66353d9b9cdc0d
@@ -1043,14 +1043,18 @@ clientInterpretRequestHeaders(clientHttp
if (request->range)
request->flags.range = 1;
}
+ /* If any conneciton has been pinned to this client, force keep alive */
if(http->conn->pinned) {
- request->flags.auth = 1;
- request->flags.pinned = 1;
request->flags.must_keepalive = 1;
+ /* If this specific host/port has been pinned to this client, flag */
+ if(pconnLookup(request->host,request->port,&request->client_addr,request->client_port)) {
+ request->flags.auth = 1;
+ request->flags.pinned = 1;
+ }
}
- if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION))
+ else if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION))
request->flags.auth = 1;
- if (request->login[0] != '\0')
+ else if (request->login[0] != '\0')
request->flags.auth = 1;
if (httpHeaderHas(req_hdr, HDR_VIA)) {
String s = httpHeaderGetList(req_hdr, HDR_VIA);
@@ -1459,7 +1463,11 @@ clientBuildReplyHeader(clientHttpRequest
||
(strncasecmp(value, "Negotiate", 9) == 0 &&
(value[9] == '\0' || value[9] == ' '))) {
- httpHeaderPutStr(hdr, HDR_PROXY_SUPPORT, "Session-Based-Authentication");
+
+ if(!request->flags.accelerated) {
+ httpHeaderPutStr(hdr, HDR_PROXY_SUPPORT, "Session-Based-Authentication");
+ httpHeaderPutStr(hdr, HDR_CONNECTION, "Proxy-support");
+ }
request->flags.pinned=1;
request->flags.must_keepalive = 1;
http->conn->pinned=1;
============================================================
--- src/enums.h b48c01f14f67bf878e17c8fd8a3d1aac10f67ab9
+++ src/enums.h cdbb572d3e510c8b6c856c755c8ce78577f6b838
@@ -745,4 +745,10 @@ enum {
#endif
+typedef enum {
+ ST_OP_NONE,
+ ST_OP_OPEN,
+ ST_OP_CREATE
+} store_op_t;
+
#endif /* SQUID_ENUMS_H */
============================================================
--- src/fs/aufs/store_dir_aufs.c 99f2f1eebf5dd0b8e29deb65369dcd0c44bb0cd6
+++ src/fs/aufs/store_dir_aufs.c 434e3a5f048cd6111395e88fb1397144b4f2cc14
@@ -106,6 +106,7 @@ static STCHECKOBJ storeAufsDirCheckObj;
static STDUMP storeAufsDirDump;
static STMAINTAINFS storeAufsDirMaintain;
static STCHECKOBJ storeAufsDirCheckObj;
+static STCHECKLOADAV storeAufsDirCheckLoadAv;
static STREFOBJ storeAufsDirRefObj;
static STUNREFOBJ storeAufsDirUnrefObj;
static QS rev_int_sort;
@@ -1591,26 +1592,23 @@ storeAufsDirMaintain(SwapDir * SD)
* object is able to be stored on this filesystem. AUFS filesystems will
* happily store anything as long as the LRU time isn't too small.
*/
-int
+char
storeAufsDirCheckObj(SwapDir * SD, const StoreEntry * e)
{
- int loadav;
- int ql;
+ return 1;
+}
-#if OLD_UNUSED_CODE
- if (storeAufsDirExpiredReferenceAge(SD) < 300) {
- debug(47, 3) ("storeAufsDirCheckObj: NO: LRU Age = %d\n",
- storeAufsDirExpiredReferenceAge(SD));
- /* store_check_cachable_hist.no.lru_age_too_low++; */
- return -1;
- }
-#endif
- ql = aioQueueSize();
- if (ql == 0)
- loadav = 0;
- loadav = ql * 1000 / MAGIC1;
- debug(47, 9) ("storeAufsDirCheckObj: load=%d\n", loadav);
- return loadav;
+int
+storeAufsDirCheckLoadAv(SwapDir *SD, store_op_t op)
+{
+ int loadav, ql;
+
+ ql = aioQueueSize();
+ if (ql == 0) {
+ return 1;
+ }
+ loadav = ql * 1000 / MAGIC1;
+ return loadav;
}
/*
@@ -1896,6 +1894,7 @@ storeAufsDirParse(SwapDir * sd, int inde
sd->statfs = storeAufsDirStats;
sd->maintainfs = storeAufsDirMaintain;
sd->checkobj = storeAufsDirCheckObj;
+ sd->checkload = storeAufsDirCheckLoadAv;
sd->refobj = storeAufsDirRefObj;
sd->unrefobj = storeAufsDirUnrefObj;
sd->callback = aioCheckCallbacks;
============================================================
--- src/fs/coss/store_dir_coss.c 4ed0002caad3b46a4fc8fc9e1a5c64c4926e93bc
+++ src/fs/coss/store_dir_coss.c e0a68cfd02a72baa7b392a368136d5a204c53495
@@ -703,20 +703,24 @@ storeCossDirShutdown(SwapDir * SD)
* not store everything. We don't check for maxobjsize here since its
* done by the upper layers.
*/
-int
+char
storeCossDirCheckObj(SwapDir * SD, const StoreEntry * e)
{
+ /* Check if the object is a special object, we can't cache these */
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
+ return 0;
+ return 1;
+}
+
+int
+storeCossDirCheckLoadAv(SwapDir *SD, store_op_t op)
+{
#if !USE_AUFSOPS
CossInfo *cs = (CossInfo *) SD->fsdata;
#endif
int loadav;
int ql = 0;
- /* Check if the object is a special object, we can't cache these */
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
- return -1;
-
- /* Otherwise, we're ok */
/* Return load, cs->aq.aq_numpending out of MAX_ASYNCOP */
#if USE_AUFSOPS
ql = aioQueueSize();
@@ -814,6 +818,7 @@ storeCossDirParse(SwapDir * sd, int inde
sd->statfs = storeCossDirStats;
sd->maintainfs = NULL;
sd->checkobj = storeCossDirCheckObj;
+ sd->checkload = storeCossDirCheckLoadAv;
sd->refobj = NULL; /* LRU is done in storeCossRead */
sd->unrefobj = NULL;
sd->callback = storeCossDirCallback;
============================================================
--- src/fs/diskd/store_dir_diskd.c b338483de8d86584e890188328fb92711677fc7f
+++ src/fs/diskd/store_dir_diskd.c 29790dc96bd0f60cad3f36d201be8a32655cf28d
@@ -110,6 +110,7 @@ static STCHECKOBJ storeDiskdDirCheckObj;
static STDUMP storeDiskdDirDump;
static STMAINTAINFS storeDiskdDirMaintain;
static STCHECKOBJ storeDiskdDirCheckObj;
+static STCHECKLOADAV storeDiskdDirCheckLoadAv;
static STREFOBJ storeDiskdDirRefObj;
static STUNREFOBJ storeDiskdDirUnrefObj;
static QS rev_int_sort;
@@ -1819,15 +1820,24 @@ storeDiskdDirMaintain(SwapDir * SD)
* object is able to be stored on this filesystem. DISKD filesystems will
* happily store anything as long as the LRU time isn't too small.
*/
-int
+char
storeDiskdDirCheckObj(SwapDir * SD, const StoreEntry * e)
{
diskdinfo_t *diskdinfo = SD->fsdata;
/* Check the queue length */
if (diskdinfo->away >= diskdinfo->magic1)
- return -1;
+ return 0;
+ return 1;
+}
+
+int
+storeDiskdDirCheckLoadAv(SwapDir *SD, store_op_t op)
+{
+ diskdinfo_t *diskdinfo = SD->fsdata;
/* Calculate the storedir load relative to magic2 on a scale of 0 .. 1000 */
/* the parse function guarantees magic2 is positivie */
+ if (diskdinfo->away >= diskdinfo->magic1)
+ return -1;
return diskdinfo->away * 1000 / diskdinfo->magic2;
}
@@ -2199,6 +2209,7 @@ storeDiskdDirParse(SwapDir * sd, int ind
sd->statfs = storeDiskdDirStats;
sd->maintainfs = storeDiskdDirMaintain;
sd->checkobj = storeDiskdDirCheckObj;
+ sd->checkload = storeDiskdDirCheckLoadAv;
sd->refobj = storeDiskdDirRefObj;
sd->unrefobj = storeDiskdDirUnrefObj;
sd->callback = storeDiskdDirCallback;
============================================================
--- src/fs/null/store_null.c fa5f237c80376f490212dc9b5431584958991925
+++ src/fs/null/store_null.c eda38ac409bcb678d4d853ce97e8eecb25ee57e7
@@ -87,9 +87,18 @@ storeNullDirRebuildComplete(void *unused
storeRebuildComplete(&counts);
}
-static int
+static char
storeNullDirCheckObj(SwapDir * SD, const StoreEntry * e)
{
+ return 0;
+}
+
+/*
+ * We should never, in theory, see an open/create, but just in case..
+ */
+static int
+storeNullDirCheckLoadAv(SwapDir *SD, store_op_t op)
+{
return -1;
}
@@ -113,6 +122,7 @@ storeNullDirParse(SwapDir * sd, int inde
sd->statfs = storeNullDirStats;
sd->init = storeNullDirInit;
sd->checkobj = storeNullDirCheckObj;
+ sd->checkload = storeNullDirCheckLoadAv;
sd->log.clean.start = storeNullDirWriteCleanStart;
sd->log.clean.done = storeNullDirWriteCleanDone;
parse_cachedir_options(sd, NULL, 0);
============================================================
--- src/fs/ufs/store_dir_ufs.c 4793d2c314a1b2a0085408592b65bb858ab30058
+++ src/fs/ufs/store_dir_ufs.c 7fea6eabaa15ef88ff1cbbc3b2a8f95da3419aa6
@@ -104,6 +104,7 @@ static STCHECKOBJ storeUfsDirCheckObj;
static STDUMP storeUfsDirDump;
static STMAINTAINFS storeUfsDirMaintain;
static STCHECKOBJ storeUfsDirCheckObj;
+static STCHECKLOADAV storeUfsDirCheckLoadAv;
static STREFOBJ storeUfsDirRefObj;
static STUNREFOBJ storeUfsDirUnrefObj;
static QS rev_int_sort;
@@ -1613,9 +1614,20 @@ storeUfsDirMaintain(SwapDir * SD)
* object is able to be stored on this filesystem. UFS filesystems will
* happily store anything as long as the LRU time isn't too small.
*/
-int
+char
storeUfsDirCheckObj(SwapDir * SD, const StoreEntry * e)
{
+ return 1;
+}
+
+/*
+ * storeUfsDirCheckLoadAv
+ *
+ * Return load average from 0 to 1000.
+ */
+int
+storeUfsDirCheckLoadAv(SwapDir *SD, store_op_t op)
+{
ufsinfo_t *ufsinfo = SD->fsdata;
return 500 + ufsinfo->open_files / 2;
}
@@ -1906,6 +1918,7 @@ storeUfsDirParse(SwapDir * sd, int index
sd->statfs = storeUfsDirStats;
sd->maintainfs = storeUfsDirMaintain;
sd->checkobj = storeUfsDirCheckObj;
+ sd->checkload = storeUfsDirCheckLoadAv;
sd->refobj = storeUfsDirRefObj;
sd->unrefobj = storeUfsDirUnrefObj;
sd->callback = NULL;
============================================================
--- src/pconn.c 5f83c46e535d572c2a6d6f55389e635926c3feee
+++ src/pconn.c 8c9ab2458a98fedd3b53d1e42fdef52ea0235e66
@@ -232,10 +232,7 @@ pconnPop(const char *peer, u_short port,
struct _pconn *p;
hash_link *hptr;
int fd = -1;
- LOCAL_ARRAY(char, key, PCONN_KEYLEN);
- assert(table != NULL);
- pconnKey(key, peer, port, client_address, client_port);
- hptr = hash_lookup(table, key);
+ hptr = pconnLookup( peer, port, client_address, client_port);
if (hptr != NULL) {
p = (struct _pconn *) hptr;
assert(p->nfds > 0);
@@ -247,6 +244,15 @@ pconnPop(const char *peer, u_short port,
return fd;
}
+hash_link *
+pconnLookup(const char *peer, u_short port, struct in_addr *client_address, u_short client_port)
+{
+ LOCAL_ARRAY(char, key, PCONN_KEYLEN);
+ assert(table != NULL);
+ pconnKey(key, peer, port, client_address, client_port);
+ return hash_lookup(table, key);
+}
+
void
pconnHistCount(int what, int i)
{
============================================================
--- src/protos.h 822ab2931a4c6df49be0b07f482586d3a7bf04cb
+++ src/protos.h aa65dda706bf16d368332b30fcdf514d4a6ed0b3
@@ -1151,6 +1151,7 @@ extern int pconnPop(const char *peer, u_
extern void pconnPush(int, const char *peer, u_short port, struct in_addr *client_address, u_short client_port);
extern int pconnPop(const char *peer, u_short port, struct in_addr *client_address, u_short client_port);
+extern hash_link *pconnLookup(const char *peer, u_short port, struct in_addr *client_address, u_short client_port);
extern void pconnInit(void);
extern int asnMatchIp(void *, struct in_addr);
============================================================
--- src/store_dir.c e795049d82fdee67d27f34a53aed344c9c60f488
+++ src/store_dir.c 299ef0c831c2b8f781cfe9992ca0912e5baa5792
@@ -156,7 +156,10 @@ storeDirSelectSwapDirRoundRobin(const St
if (!storeDirValidSwapDirSize(dirn, objsize))
continue;
/* check for error or overload condition */
- load = sd->checkobj(sd, e);
+ if (sd->checkobj(sd, e) == 0) {
+ continue;
+ }
+ load = sd->checkload(sd, ST_OP_CREATE);
if (load < 0 || load > 1000) {
continue;
}
@@ -197,7 +200,10 @@ storeDirSelectSwapDirLeastLoad(const Sto
for (i = 0; i < Config.cacheSwap.n_configured; i++) {
SD = &Config.cacheSwap.swapDirs[i];
SD->flags.selected = 0;
- load = SD->checkobj(SD, e);
+ if (SD->checkobj(SD, e) == 0) {
+ continue;
+ }
+ load = SD->checkload(SD, ST_OP_CREATE);
if (load < 0 || load > 1000) {
continue;
}
============================================================
--- src/store_io.c b3189954bb7b33712f066ccf62783d2b0506f710
+++ src/store_io.c 8a31eb00c0678314dbf38b0fa41d2bbd3f96f3b9
@@ -7,6 +7,12 @@ static struct {
int create_fail;
int success;
} create;
+ struct {
+ int calls;
+ int success;
+ int open_fail;
+ int loadav_fail;
+ } open;
} store_io_stats;
OBJH storeIOStats;
@@ -61,8 +67,23 @@ storeOpen(StoreEntry * e, STFNCB * file_
storeOpen(StoreEntry * e, STFNCB * file_callback, STIOCB * callback,
void *callback_data)
{
+ int load;
+ storeIOState *sio;
+
SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn];
- return SD->obj.open(SD, e, file_callback, callback, callback_data);
+ store_io_stats.open.calls++;
+ load = SD->checkload(SD, ST_OP_OPEN);
+ if (load < 0 || load > 1000) {
+ store_io_stats.open.loadav_fail++;
+ return NULL;
+ }
+ sio = SD->obj.open(SD, e, file_callback, callback, callback_data);
+ if (sio == NULL) {
+ store_io_stats.open.open_fail++;
+ } else {
+ store_io_stats.open.success++;
+ }
+ return sio;
}
void
@@ -114,4 +135,8 @@ storeIOStats(StoreEntry * sentry)
storeAppendPrintf(sentry, "create.select_fail %d\n", store_io_stats.create.select_fail);
storeAppendPrintf(sentry, "create.create_fail %d\n", store_io_stats.create.create_fail);
storeAppendPrintf(sentry, "create.success %d\n", store_io_stats.create.success);
+ storeAppendPrintf(sentry, "open.calls %d\n", store_io_stats.open.calls);
+ storeAppendPrintf(sentry, "open.success %d\n", store_io_stats.open.success);
+ storeAppendPrintf(sentry, "open.loadav_fail %d\n", store_io_stats.open.loadav_fail);
+ storeAppendPrintf(sentry, "open.open_fail %d\n", store_io_stats.open.open_fail);
}
============================================================
--- src/structs.h 09fd7df0ee0cc66a8be6be822cf34b2203aac563
+++ src/structs.h 2cc0257c851aa9f3899204538a4037888a0465cf
@@ -1616,6 +1616,7 @@ struct _SwapDir {
STSTATFS *statfs; /* Dump fs statistics */
STMAINTAINFS *maintainfs; /* Replacement maintainence */
STCHECKOBJ *checkobj; /* Check if the fs will store an object */
+ STCHECKLOADAV *checkload; /* Check if the fs is getting overloaded .. */
/* These two are notifications */
STREFOBJ *refobj; /* Reference this object */
STUNREFOBJ *unrefobj; /* Unreference this object */
============================================================
--- src/typedefs.h 0a1930b4adf4cddddda7f44324e88fb53311162f
+++ src/typedefs.h af3aabd123fce77fbe2ce6bfc45c942e516c8640
@@ -288,7 +288,8 @@ typedef void STMAINTAINFS(SwapDir *);
typedef int STDBLCHECK(SwapDir *, StoreEntry *);
typedef void STSTATFS(SwapDir *, StoreEntry *);
typedef void STMAINTAINFS(SwapDir *);
-typedef int STCHECKOBJ(SwapDir *, const StoreEntry *);
+typedef int STCHECKLOADAV(SwapDir *, store_op_t op);
+typedef char STCHECKOBJ(SwapDir *, const StoreEntry *);
typedef void STREFOBJ(SwapDir *, StoreEntry *);
typedef void STUNREFOBJ(SwapDir *, StoreEntry *);
typedef void STSETUP(storefs_entry_t *);