=== modified file 'src/ClientRequestContext.h' --- src/ClientRequestContext.h 2012-11-30 11:08:47 +0000 +++ src/ClientRequestContext.h 2012-12-05 12:04:18 +0000 @@ -18,71 +18,76 @@ class ClientRequestContext : public RefCountable { public: void *operator new(size_t); void operator delete(void *); ClientRequestContext(ClientHttpRequest *); ~ClientRequestContext(); bool httpStateIsValid(); void hostHeaderVerify(); void hostHeaderIpVerify(const ipcache_addrs* ia, const DnsLookupDetails &dns); void hostHeaderVerifyFailed(const char *A, const char *B); void clientAccessCheck(); void clientAccessCheck2(); void clientAccessCheckDone(const allow_t &answer); void clientRedirectStart(); void clientRedirectDone(const HelperReply &reply); + void clientStoreIdStart(); + void clientStoreIdDone(const HelperReply &reply); void checkNoCache(); void checkNoCacheDone(const allow_t &answer); #if USE_ADAPTATION void adaptationAccessCheck(); #endif #if USE_SSL /** * Initiates and start the acl checklist to check if the a CONNECT * request must be bumped. \retval true if the acl check scheduled, false if no ssl-bump required */ bool sslBumpAccessCheck(); /// The callback function for ssl-bump access check list void sslBumpAccessCheckDone(const allow_t &answer); #endif ClientHttpRequest *http; ACLChecklist *acl_checklist; /* need ptr back so we can unreg if needed */ int redirect_state; + int store_id_state; /** * URL-rewrite/redirect helper may return BH for internal errors. * We attempt to recover by trying the lookup again, but limit the * number of retries to prevent lag and lockups. * This tracks the number of previous failures for the current context. */ uint8_t redirect_fail_count; + uint8_t store_id_fail_count; bool host_header_verify_done; bool http_access_done; bool adapted_http_access_done; #if USE_ADAPTATION bool adaptation_acl_check_done; #endif bool redirect_done; + bool store_id_done; bool no_cache_done; bool interpreted_req_hdrs; bool tosToClientDone; bool nfmarkToClientDone; #if USE_SSL bool sslBumpCheckDone; #endif ErrorState *error; ///< saved error page for centralized/delayed processing bool readNextRequest; ///< whether Squid should read after error handling private: CBDATA_CLASS(ClientRequestContext); }; #endif /* SQUID_CLIENTREQUESTCONTEXT_H */ === modified file 'src/HelperReply.cc' --- src/HelperReply.cc 2012-11-28 01:13:21 +0000 +++ src/HelperReply.cc 2013-02-03 03:31:19 +0000 @@ -1,64 +1,71 @@ /* * DEBUG: section 84 Helper process maintenance * AUTHOR: Amos Jeffries */ #include "squid.h" #include "ConfigParser.h" #include "HelperReply.h" #include "helper.h" #include "rfc1738.h" #include "SquidString.h" +#include "Debug.h" HelperReply::HelperReply(char *buf, size_t len) : result(HelperReply::Unknown), whichServer(NULL) { parse(buf,len); } void HelperReply::parse(char *buf, size_t len) { + debugs(84, 3, "Parsing helper buffer"); // check we have something to parse if (!buf || len < 1) { // for now ensure that legacy handlers are not presented with NULL strings. + debugs(84, 3, "Buff lenth is smaller then 1 or no at all "); other_.init(1,1); other_.terminate(); return; } char *p = buf; // optimization: do not consider parsing result code if the response is short. // URL-rewriter may return relative URLs or empty response for a large portion // of its replies. if (len >= 2) { + debugs(84, 3, "Buff lenth is larger then 2"); // some helper formats (digest auth, URL-rewriter) just send a data string // we must also check for the ' ' character after the response token (if anything) if (!strncmp(p,"OK",2) && (len == 2 || p[2] == ' ')) { + debugs(84, 3, "helper Result = OK"); result = HelperReply::Okay; p+=2; } else if (!strncmp(p,"ERR",3) && (len == 3 || p[3] == ' ')) { + debugs(84, 3, "helper Result = ERR"); result = HelperReply::Error; p+=3; } else if (!strncmp(p,"BH",2) && (len == 2 || p[2] == ' ')) { + debugs(84, 3, "helper Result = BrokenHelper"); result = HelperReply::BrokenHelper; p+=2; } else if (!strncmp(p,"TT ",3)) { // NTLM challenge token result = HelperReply::TT; p+=3; // followed by an auth token char *w1 = strwordtok(NULL, &p); if (w1 != NULL) { MemBuf authToken; authToken.init(); authToken.append(w1, strlen(w1)); notes.add("token",authToken.content()); } else { // token field is mandatory on this response code result = HelperReply::BrokenHelper; notes.add("message","Missing 'token' data"); } } else if (!strncmp(p,"AF ",3)) { === modified file 'src/HttpRequest.cc' --- src/HttpRequest.cc 2012-12-02 23:10:23 +0000 +++ src/HttpRequest.cc 2013-02-03 06:16:47 +0000 @@ -671,20 +671,38 @@ bool HttpRequest::canHandle1xx() const { // old clients do not support 1xx unless they sent Expect: 100-continue // (we reject all other HDR_EXPECT values so just check for HDR_EXPECT) if (http_ver <= HttpVersion(1,0) && !header.has(HDR_EXPECT)) return false; // others must support 1xx control messages return true; } ConnStateData * HttpRequest::pinnedConnection() { if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned) return clientConnectionManager.get(); return NULL; } + +/** + * The method returns the current storeID of the request. + * returns canonicalUrl of the request in a case StoreID not present. + * Always returns null terminated char* . + */ +const char * +HttpRequest::storeId() +{ + if (store_id.size() != 0){ + debugs(73, 3, "sent back store_id:" << store_id); + + return store_id.termedBuf(); + } + debugs(73, 3, "sent back canonicalUrl:" << urlCanonical(this) ); + + return urlCanonical(this); +} === modified file 'src/HttpRequest.h' --- src/HttpRequest.h 2012-11-30 22:57:54 +0000 +++ src/HttpRequest.h 2013-02-03 05:54:09 +0000 @@ -147,40 +147,45 @@ char host[SQUIDHOSTNAMELEN]; int host_is_numeric; #if USE_ADAPTATION mutable Adaptation::History::Pointer adaptHistory_; ///< per-HTTP transaction info #endif #if ICAP_CLIENT mutable Adaptation::Icap::History::Pointer icapHistory_; ///< per-HTTP transaction info #endif public: Ip::Address host_addr; #if USE_AUTH Auth::UserRequest::Pointer auth_user_request; #endif unsigned short port; String urlpath; char *canonical; + /** + * The ID string used internally by Squid to uniquely de-duplicate this requests + * URL with other URLs stored objects. + */ + String store_id; RequestFlags flags; HttpHdrRange *range; time_t ims; int imslen; Ip::Address client_addr; #if FOLLOW_X_FORWARDED_FOR Ip::Address indirect_client_addr; #endif /* FOLLOW_X_FORWARDED_FOR */ Ip::Address my_addr; HierarchyLogEntry hier; int dnsWait; ///< sum of DNS lookup delays in milliseconds, for %dt @@ -225,40 +230,49 @@ virtual bool expectingBody(const HttpRequestMethod& unused, int64_t&) const; bool bodyNibbled() const; // the request has a [partially] consumed body int prefixLen(); void swapOut(StoreEntry * e); void pack(Packer * p); static void httpRequestPack(void *obj, Packer *p); static HttpRequest * CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method); static HttpRequest * CreateFromUrl(char * url); ConnStateData *pinnedConnection(); /** + * Returns the current StoreID for the request as a nul-terminated char*. + * Always returns the current id for the request + * (either the request canonical url or modified ID by the helper). + * dosn't return NULL since this became the glue between most of + * store MD5 hash and other important code that a cache object based on. + */ + const char *storeId(); + + /** * The client connection manager, if known; * Used for any response actions needed directly to the client. * ie 1xx forwarding or connection pinning state changes */ CbcPointer clientConnectionManager; int64_t getRangeOffsetLimit(); /* the result of this function gets cached in rangeOffsetLimit */ private: const char *packableURI(bool full_uri) const; mutable int64_t rangeOffsetLimit; /* caches the result of getRangeOffsetLimit */ protected: virtual void packFirstLineInto(Packer * p, bool full_uri) const; virtual bool sanityCheckStartLine(MemBuf *buf, const size_t hdr_len, http_status *error); virtual void hdrCacheInit(); === modified file 'src/SquidConfig.h' --- src/SquidConfig.h 2012-10-29 04:59:58 +0000 +++ src/SquidConfig.h 2012-12-04 18:10:46 +0000 @@ -186,57 +186,59 @@ char *swap; CustomLog *accesslogs; #if ICAP_CLIENT CustomLog *icaplogs; #endif int rotateNumber; } Log; char *adminEmail; char *EmailFrom; char *EmailProgram; char *effectiveUser; char *visible_appname_string; char *effectiveGroup; struct { #if USE_DNSHELPER char *dnsserver; #endif wordlist *redirect; + wordlist *store_id; #if USE_UNLINKD char *unlinkd; #endif char *diskd; #if USE_SSL char *ssl_password; #endif } Program; #if USE_DNSHELPER HelperChildConfig dnsChildren; #endif HelperChildConfig redirectChildren; + HelperChildConfig storeIdChildren; time_t authenticateGCInterval; time_t authenticateTTL; time_t authenticateIpTTL; struct { char *surrogate_id; } Accel; char *appendDomain; size_t appendDomainLen; char *pidFilename; char *netdbFilename; char *mimeTablePathname; char *etcHostsPath; char *visibleHostname; char *uniqueHostname; wordlist *hostnameAliases; char *errHtmlText; struct { char *host; @@ -301,40 +303,41 @@ int buffered_logs; int common_log; int log_mime_hdrs; int log_fqdn; int announce; int mem_pools; int test_reachability; int half_closed_clients; int refresh_all_ims; #if USE_HTTP_VIOLATIONS int reload_into_ims; #endif int offline; int redir_rewrites_host; int prefer_direct; int nonhierarchical_direct; int strip_query_terms; int redirector_bypass; + int store_id_bypass; int ignore_unknown_nameservers; int client_pconns; int server_pconns; int error_pconns; #if USE_CACHE_DIGESTS int digest_generation; #endif int ie_refresh; int vary_ignore_expire; int pipeline_prefetch; int surrogate_is_remote; int request_entities; int detect_broken_server_pconns; int balance_on_multiple_ip; int relaxed_header_parser; int check_hostnames; int allow_underscore; int via; @@ -364,40 +367,41 @@ class ACL *aclList; struct { acl_access *http; acl_access *adapted_http; acl_access *icp; acl_access *miss; acl_access *NeverDirect; acl_access *AlwaysDirect; acl_access *ASlists; acl_access *noCache; acl_access *log; #if SQUID_SNMP acl_access *snmp; #endif #if USE_HTTP_VIOLATIONS acl_access *brokenPosts; #endif acl_access *redirector; + acl_access *store_id; acl_access *reply; AclAddress *outgoing_address; #if USE_HTCP acl_access *htcp; acl_access *htcp_clr; #endif #if USE_SSL acl_access *ssl_bump; #endif #if FOLLOW_X_FORWARDED_FOR acl_access *followXFF; #endif /* FOLLOW_X_FORWARDED_FOR */ #if ICAP_CLIENT acl_access* icap; #endif } accessList; AclDenyInfoList *denyInfoList; === modified file 'src/cache_cf.cc' --- src/cache_cf.cc 2013-01-30 15:39:37 +0000 +++ src/cache_cf.cc 2013-02-03 04:46:54 +0000 @@ -684,40 +684,47 @@ Config.onoff.announce = 0; } if (Config.onoff.httpd_suppress_version_string) visible_appname_string = (char *)appname_string; else visible_appname_string = (char const *)APP_FULLNAME; #if USE_DNSHELPER if (Config.dnsChildren.n_max < 1) fatal("No DNS helpers allocated"); #endif if (Config.Program.redirect) { if (Config.redirectChildren.n_max < 1) { Config.redirectChildren.n_max = 0; wordlistDestroy(&Config.Program.redirect); } } + if (Config.Program.store_id) { + if (Config.storeIdChildren.n_max < 1) { + Config.storeIdChildren.n_max = 0; + wordlistDestroy(&Config.Program.store_id); + } + } + if (Config.appendDomain) if (*Config.appendDomain != '.') fatal("append_domain must begin with a '.'"); if (Config.errHtmlText == NULL) Config.errHtmlText = xstrdup(null_string); #if !HAVE_SETRLIMIT || !defined(RLIMIT_NOFILE) if (Config.max_filedescriptors > 0) { debugs(0, DBG_IMPORTANT, "WARNING: max_filedescriptors disabled. Operating System setrlimit(RLIMIT_NOFILE) is missing."); } #elif USE_SELECT || USE_SELECT_WIN32 if (Config.max_filedescriptors > FD_SETSIZE) { debugs(0, DBG_IMPORTANT, "WARNING: max_filedescriptors limited to " << FD_SETSIZE << " by select() algorithm."); } #endif storeConfigure(); snprintf(ThisCache, sizeof(ThisCache), "%s (%s)", @@ -746,40 +753,43 @@ else Config.appendDomainLen = 0; if (Config.connect_retries > 10) { debugs(0,DBG_CRITICAL, "WARNING: connect_retries cannot be larger than 10. Resetting to 10."); Config.connect_retries = 10; } requirePathnameExists("MIME Config Table", Config.mimeTablePathname); #if USE_DNSHELPER requirePathnameExists("cache_dns_program", Config.Program.dnsserver); #endif #if USE_UNLINKD requirePathnameExists("unlinkd_program", Config.Program.unlinkd); #endif requirePathnameExists("logfile_daemon", Log::TheConfig.logfile_daemon); if (Config.Program.redirect) requirePathnameExists("redirect_program", Config.Program.redirect->key); + if (Config.Program.store_id) + requirePathnameExists("store_id_program", Config.Program.store_id->key); + requirePathnameExists("Icon Directory", Config.icons.directory); if (Config.errorDirectory) requirePathnameExists("Error Directory", Config.errorDirectory); #if USE_HTTP_VIOLATIONS { const RefreshPattern *R; for (R = Config.Refresh; R; R = R->next) { if (!R->flags.override_expire) continue; debugs(22, DBG_IMPORTANT, "WARNING: use of 'override-expire' in 'refresh_pattern' violates HTTP"); break; } for (R = Config.Refresh; R; R = R->next) { === modified file 'src/cf.data.pre' --- src/cf.data.pre 2012-12-13 22:31:55 +0000 +++ src/cf.data.pre 2013-02-03 05:56:38 +0000 @@ -4329,40 +4329,166 @@ See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. DOC_END NAME: url_rewrite_bypass redirector_bypass TYPE: onoff LOC: Config.onoff.redirector_bypass DEFAULT: off DOC_START When this is 'on', a request will not go through the redirector if all redirectors are busy. If this is 'off' and the redirector queue grows too large, Squid will exit with a FATAL error and ask you to increase the number of redirectors. You should only enable this if the redirectors are not critical to your caching system. If you use redirectors for access control, and you enable this option, users may have access to pages they should not be allowed to request. DOC_END COMMENT_START + OPTIONS FOR STORE ID + ----------------------------------------------------------------------------- +COMMENT_END + +NAME: store_id_program storeurl_rewrite_program +TYPE: wordlist +LOC: Config.Program.store_id +DEFAULT: none +DOC_START + Specify the location of the executable StoreID helper to use. + Since they can perform almost any function there isn't one included. + + For each requested URL, the helper will receive one line with the format + + [channel-ID ] URL client_ip "/" fqdn user method [ kv-pairs] + + + After processing the request the helper must reply using the following format: + + [channel-ID ] result [ kv-pairs] + + The result code can be: + + OK store-id="..." + Use the StoreID supplied in 'store-id='. + + ERR + The default is to use HTTP request URL as the store ID. + + BH + An internal error occured in the helper, preventing + a result being identified. + + + In the future, the interface protocol will be extended with + key=value pairs ("kv-pairs" shown above). Helper programs + should be prepared to receive and possibly ignore additional + whitespace-separated tokens on each input line. + + When using the concurrency= option the protocol is changed by + introducing a query channel tag in front of the request/response. + The query channel tag is a number between 0 and concurrency-1. + This value must be echoed back unchanged to Squid as the first part + of the response relating to its request. + + WARNING: + StoreID forces squid to use a specific String to identify and verify a + object\resource. The default is the url of the resource and If not + planned carefully a bad StoreID response from the helper can result + in a mismatch between the user request to the reply. + Note that when using StoreID the refresh_patterns will apply to the + StoreID returned from the helper and not the original URL. + + By default, a StoreID helper is not used. +DOC_END + +NAME: store_id_children storeurl_rewrite_children +TYPE: HelperChildConfig +DEFAULT: 20 startup=0 idle=1 concurrency=0 +LOC: Config.storeIdChildren +DOC_START + The maximum number of StoreID helper processes to spawn. If you limit + it too few Squid will have to wait for them to process a backlog of + requests, slowing it down. If you allow too many they will use RAM + and other system resources noticably. + + The startup= and idle= options allow some measure of skew in your + tuning. + + startup= + + Sets a minimum of how many processes are to be spawned when Squid + starts or reconfigures. When set to zero the first request will + cause spawning of the first child process to handle it. + + Starting too few will cause an initial slowdown in traffic as Squid + attempts to simultaneously spawn enough processes to cope. + + idle= + + Sets a minimum of how many processes Squid is to try and keep available + at all times. When traffic begins to rise above what the existing + processes can handle this many more will be spawned up to the maximum + configured. A minimum setting of 1 is required. + + concurrency= + + The number of requests each storeID helper can handle in + parallel. Defaults to 0 which indicates the helper + is a old-style single threaded program. + + When this directive is set to a value >= 1 then the protocol + used to communicate with the helper is modified to include + an ID in front of the request/response. The ID from the request + must be echoed back with the response to that request. +DOC_END + +NAME: store_id_access storeurl_rewrite_access +TYPE: acl_access +DEFAULT: none +LOC: Config.accessList.store_id +DOC_START + If defined, this access list specifies which requests are + sent to the StoreID processes. By default all requests + are sent. + + This clause supports both fast and slow acl types. + See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. +DOC_END + +NAME: store_id_bypass storeurl_rewrite_bypass +TYPE: onoff +LOC: Config.onoff.store_id_bypass +DEFAULT: on +DOC_START + When this is 'on', a request will not go through the + helper if all helpers are busy. If this is 'off' + and the helper queue grows too large, Squid will exit + with a FATAL error and ask you to increase the number of + helpers. You should only enable this if the helperss + are not critical to your caching system. If you use + helpers for critical caching components, and you enable this + option, users may not get objects from cache. +DOC_END + +COMMENT_START OPTIONS FOR TUNING THE CACHE ----------------------------------------------------------------------------- COMMENT_END NAME: cache no_cache TYPE: acl_access DEFAULT: none LOC: Config.accessList.noCache DOC_START A list of ACL elements which, if matched and denied, cause the request to not be satisfied from the cache and the reply to not be cached. In other words, use this to force certain objects to never be cached. You must use the words 'allow' or 'deny' to indicate whether items matching the ACL should be allowed or denied into the cache. Default is to allow all to be cached. This clause supports both fast and slow acl types. See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. === modified file 'src/client_side_reply.cc' --- src/client_side_reply.cc 2013-01-30 15:39:37 +0000 +++ src/client_side_reply.cc 2013-02-03 07:50:51 +0000 @@ -241,41 +241,41 @@ /* This function is wrong - the client parameters don't include the * header offset */ void clientReplyContext::triggerInitialStoreRead() { /* when confident, 0 becomes reqofs, and then this factors into * startSendProcess */ assert(reqofs == 0); StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data); storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this); } /* there is an expired entry in the store. * setup a temporary buffer area and perform an IMS to the origin */ void clientReplyContext::processExpired() { - char *url = http->uri; + const char *url = storeId(); StoreEntry *entry = NULL; debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'"); assert(http->storeEntry()->lastmod >= 0); /* * check if we are allowed to contact other servers * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return * a stale entry *if* it matches client requirements */ if (http->onlyIfCached()) { processOnlyIfCachedMiss(); return; } http->request->flags.refresh = true; #if STORE_CLIENT_LIST_DEBUG /* Prevent a race with the store client memory free routines */ assert(storeClientIsThisAClient(sc, this)); #endif @@ -480,42 +480,42 @@ if (result.length == 0) { /* the store couldn't get enough data from the file for us to id the * object */ /* treat as a miss */ http->logType = LOG_TCP_MISS; processMiss(); return; } assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); /* update size of the request */ reqsize = result.length + reqofs; /* * Got the headers, now grok them */ assert(http->logType == LOG_TCP_HIT); - if (strcmp(e->mem_obj->url, urlCanonical(r)) != 0) { - debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->url << "' != '" << urlCanonical(r) << "'"); + if (strcmp(e->mem_obj->url, http->request->storeId()) != 0) { + debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->url << "' != '" << r->storeId() << "'"); processMiss(); return; } switch (varyEvaluateMatch(e, r)) { case VARY_NONE: /* No variance detected. Continue as normal */ break; case VARY_MATCH: /* This is the correct entity for this request. Continue */ debugs(88, 2, "clientProcessHit: Vary MATCH!"); break; case VARY_OTHER: /* This is not the correct entity for this request. We need * to requery the cache. */ removeClientStoreReference(&sc, http); @@ -863,41 +863,41 @@ } void clientReplyContext::purgeFoundObject(StoreEntry *entry) { assert (entry && !entry->isNull()); if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) { http->logType = LOG_TCP_DENIED; ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL, http->getConn()->clientConnection->remote, http->request); startError(err); return; } StoreIOBuffer localTempBuffer; /* Swap in the metadata */ http->storeEntry(entry); http->storeEntry()->lock(); - http->storeEntry()->createMemObject(http->uri, http->log_uri); + http->storeEntry()->createMemObject(storeId(), http->log_uri); http->storeEntry()->mem_obj->method = http->request->method; sc = storeClientListAdd(http->storeEntry(), this); http->logType = LOG_TCP_HIT; reqofs = 0; localTempBuffer.offset = http->out.offset; localTempBuffer.length = next()->readBuffer.length; localTempBuffer.data = next()->readBuffer.data; storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this); } void @@ -1727,42 +1727,48 @@ } void clientReplyContext::doGetMoreData() { /* We still have to do store logic processing - vary, cache hit etc */ if (http->storeEntry() != NULL) { /* someone found the object in the cache for us */ StoreIOBuffer localTempBuffer; http->storeEntry()->lock(); if (http->storeEntry()->mem_obj == NULL) { /* * This if-block exists because we don't want to clobber * a preexiting mem_obj->method value if the mem_obj * already exists. For example, when a HEAD request * is a cache hit for a GET response, we want to keep * the method as GET. */ - http->storeEntry()->createMemObject(http->uri, http->log_uri); + http->storeEntry()->createMemObject(storeId(), http->log_uri); http->storeEntry()->mem_obj->method = http->request->method; + /** + * Here is the place where the snowball of an object + * creation starts. Here we can see if the object was + * created using URL or alternative StoreID from helper. + */ + debugs(88, 3, "What mem-obj->url contains ?: "<< http->storeEntry()->mem_obj->url ); } sc = storeClientListAdd(http->storeEntry(), this); #if USE_DELAY_POOLS sc->setDelayId(DelayId::DelayClient(http)); #endif assert(http->logType == LOG_TCP_HIT); reqofs = 0; /* guarantee nothing has been sent yet! */ assert(http->out.size == 0); assert(http->out.offset == 0); if (Ip::Qos::TheConfig.isHitTosActive()) { Ip::Qos::doTosLocalHit(http->getConn()->clientConnection); } if (Ip::Qos::TheConfig.isHitNfmarkActive()) { Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection); } @@ -2159,41 +2165,41 @@ holdingBuffer = result; processReplyAccess(); return; } /* Using this breaks the client layering just a little! */ void clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags) { assert(http != NULL); /* * For erroneous requests, we might not have a h->request, * so make a fake one. */ if (http->request == NULL) http->request = HTTPMSGLOCK(new HttpRequest(m, AnyP::PROTO_NONE, null_string)); - StoreEntry *e = storeCreateEntry(http->uri, http->log_uri, reqFlags, m); + StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m); sc = storeClientListAdd(e, this); #if USE_DELAY_POOLS sc->setDelayId(DelayId::DelayClient(http)); #endif reqofs = 0; reqsize = 0; /* I don't think this is actually needed! -- adrian */ /* http->reqbuf = http->norm_reqbuf; */ // assert(http->reqbuf == http->norm_reqbuf); /* The next line is illegal because we don't know if the client stream * buffers have been set up */ // storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf, // SendMoreData, this); /* So, we mark the store logic as complete */ === modified file 'src/client_side_reply.h' --- src/client_side_reply.h 2012-10-29 04:59:58 +0000 +++ src/client_side_reply.h 2013-02-03 07:48:30 +0000 @@ -74,40 +74,41 @@ int storeNotOKTransferDone() const; /// replaces current response store entry with the given one void setReplyToStoreEntry(StoreEntry *e); /// builds error using clientBuildError() and calls setReplyToError() below void setReplyToError(err_type, http_status, const HttpRequestMethod&, char const *, Ip::Address &, HttpRequest *, const char *, #if USE_AUTH Auth::UserRequest::Pointer); #else void * unused); #endif /// creates a store entry for the reply and appends err to it void setReplyToError(const HttpRequestMethod& method, ErrorState *err); void createStoreEntry(const HttpRequestMethod& m, RequestFlags flags); void removeStoreReference(store_client ** scp, StoreEntry ** ep); void removeClientStoreReference(store_client **scp, ClientHttpRequest *http); void startError(ErrorState * err); void processExpired(); clientStream_status_t replyStatus(); void processMiss(); void traceReply(clientStreamNode * node); + const char *storeId() { return (http->store_id.size() > 0 ? http->store_id.termedBuf() : http->uri); } http_status purgeStatus; /* state variable - replace with class to handle storeentries at some point */ int lookingforstore; virtual void created (StoreEntry *newEntry); ClientHttpRequest *http; int headers_sz; store_client *sc; /* The store_client we're using */ StoreIOBuffer tempBuffer; /* For use in validating requests via IMS */ int old_reqsize; /* ... again, for the buffer */ size_t reqsize; size_t reqofs; char tempbuf[HTTP_REQBUF_SZ]; /* a temporary buffer if we need working storage */ #if USE_CACHE_DIGESTS const char *lookup_type; /* temporary hack: storeGet() result: HIT/MISS/NONE */ #endif === modified file 'src/client_side_request.cc' --- src/client_side_request.cc 2013-01-30 15:39:37 +0000 +++ src/client_side_request.cc 2013-02-03 07:40:17 +0000 @@ -115,65 +115,68 @@ ClientRequestContext *result = cbdataAlloc(ClientRequestContext); return result; } void ClientRequestContext::operator delete (void *address) { ClientRequestContext *t = static_cast(address); cbdataFree(t); } /* Local functions */ /* other */ static void clientAccessCheckDoneWrapper(allow_t, void *); #if USE_SSL static void sslBumpAccessCheckDoneWrapper(allow_t, void *); #endif static int clientHierarchical(ClientHttpRequest * http); static void clientInterpretRequestHeaders(ClientHttpRequest * http); static HLPCB clientRedirectDoneWrapper; +static HLPCB clientStoreIdDoneWrapper; static void checkNoCacheDoneWrapper(allow_t, void *); SQUIDCEXTERN CSR clientGetMoreData; SQUIDCEXTERN CSS clientReplyStatus; SQUIDCEXTERN CSD clientReplyDetach; static void checkFailureRatio(err_type, hier_code); ClientRequestContext::~ClientRequestContext() { /* * Release our "lock" on our parent, ClientHttpRequest, if we * still have one */ if (http) cbdataReferenceDone(http); delete error; debugs(85,3, HERE << this << " ClientRequestContext destructed"); } -ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), error(NULL), readNextRequest(false) +ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), store_id_state(REDIRECT_NONE),error(NULL), readNextRequest(false) { http_access_done = false; redirect_done = false; redirect_fail_count = 0; + store_id_done = false; + store_id_fail_count = 0; no_cache_done = false; interpreted_req_hdrs = false; #if USE_SSL sslBumpCheckDone = false; #endif debugs(85,3, HERE << this << " ClientRequestContext constructed"); } CBDATA_CLASS_INIT(ClientHttpRequest); void * ClientHttpRequest::operator new (size_t size) { assert (size == sizeof (ClientHttpRequest)); CBDATA_INIT_TYPE(ClientHttpRequest); ClientHttpRequest *result = cbdataAlloc(ClientHttpRequest); return result; } void @@ -903,40 +906,78 @@ if (answer == ACCESS_ALLOWED) redirectStart(http, clientRedirectDoneWrapper, context); else { HelperReply nilReply; context->clientRedirectDone(nilReply); } } void ClientRequestContext::clientRedirectStart() { debugs(33, 5, HERE << "'" << http->uri << "'"); if (Config.accessList.redirector) { acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http); acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this); } else redirectStart(http, clientRedirectDoneWrapper, this); } +/** + * This methods handles Access checks result of StoreId access list. + * Will handle as "ERR" (no change) in a case Access is not allowed. + */ +static void +clientStoreIdAccessCheckDone(allow_t answer, void *data) +{ + ClientRequestContext *context = static_cast(data); + ClientHttpRequest *http = context->http; + context->acl_checklist = NULL; + + if (answer == ACCESS_ALLOWED) + storeIdStart(http, clientStoreIdDoneWrapper, context); + else { + debugs(85, 3, "access denied expected ERR reply handling: " << answer); + HelperReply nilReply; + nilReply.result = HelperReply::Error; + context->clientStoreIdDone(nilReply); + } +} + +/** + * Start locating an alternative storeage ID string (if any) from admin + * configured helper program. This is an asynchronous operation terminating in + * ClientRequestContext::clientStoreIdDone() when completed. + */ +void +ClientRequestContext::clientStoreIdStart() +{ + debugs(33, 5,"'" << http->uri << "'"); + + if (Config.accessList.store_id) { + acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http); + acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this); + } else + storeIdStart(http, clientStoreIdDoneWrapper, this); +} + static int clientHierarchical(ClientHttpRequest * http) { const char *url = http->uri; HttpRequest *request = http->request; HttpRequestMethod method = request->method; const wordlist *p = NULL; // intercepted requests MUST NOT (yet) be sent to peers unless verified if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.spoofClientIp)) return 0; /* * IMS needs a private key, so we can use the hierarchy for IMS only if our * neighbors support private keys */ if (request->flags.ims && !neighbors_do_private_keys) return 0; @@ -1180,40 +1221,51 @@ (request->flags.noCache ? "SET" : "NOT SET")); debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " << (request->flags.cachable ? "SET" : "NOT SET")); debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " << (request->flags.hierarchical ? "SET" : "NOT SET")); } void clientRedirectDoneWrapper(void *data, const HelperReply &result) { ClientRequestContext *calloutContext = (ClientRequestContext *)data; if (!calloutContext->httpStateIsValid()) return; calloutContext->clientRedirectDone(result); } void +clientStoreIdDoneWrapper(void *data, const HelperReply &result) +{ + ClientRequestContext *calloutContext = (ClientRequestContext *)data; + + if (!calloutContext->httpStateIsValid()) + return; + + calloutContext->clientStoreIdDone(result); +} + +void ClientRequestContext::clientRedirectDone(const HelperReply &reply) { HttpRequest *old_request = http->request; debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply); assert(redirect_state == REDIRECT_PENDING); redirect_state = REDIRECT_DONE; // copy the URL rewriter response Notes to the HTTP request for logging // do it early to ensure that no matter what the outcome the notes are present. // TODO put them straight into the transaction state record (ALE?) eventually if (!old_request->helperNotes) old_request->helperNotes = new Notes; old_request->helperNotes->add(reply.notes); switch (reply.result) { case HelperReply::Unknown: case HelperReply::TT: // Handler in redirect.cc should have already mapped Unknown // IF it contained valid entry for the old URL-rewrite helper protocol debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply); @@ -1296,40 +1348,96 @@ debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " << old_request->method << " " << urlNote->firstValue() << " " << old_request->http_ver); delete new_request; } } } } break; } /* FIXME PIPELINE: This is innacurate during pipelining */ if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection)) fd_note(http->getConn()->clientConnection->fd, http->uri); assert(http->uri); http->doCallouts(); } +/** + * This method handles the different replies from StoreID helper. + */ +void +ClientRequestContext::clientStoreIdDone(const HelperReply &reply) +{ + HttpRequest *old_request = http->request; + debugs(85, 5, "'" << http->uri << "' result=" << reply); + assert(store_id_state == REDIRECT_PENDING); + store_id_state = REDIRECT_DONE; + + // copy the helper response Notes to the HTTP request for logging + // do it early to ensure that no matter what the outcome the notes are present. + // TODO put them straight into the transaction state record (ALE?) eventually + if (!old_request->helperNotes) + old_request->helperNotes = new Notes; + old_request->helperNotes->add(reply.notes); + + switch(reply.result) { + case HelperReply::Unknown: + case HelperReply::TT: + // Handler in redirect.cc should have already mapped Unknown + // IF it contained valid entry for the old helper protocol + debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply); + break; + + case HelperReply::BrokenHelper: + debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply << ", attempt #" << (store_id_fail_count+1) << " of 2"); + if (store_id_fail_count < 2) { // XXX: make this configurable ? + ++store_id_fail_count; + // reset state flag to try StoreID again from scratch. + store_id_done = false; + } + break; + + case HelperReply::Error: + // no change to be done. + break; + + case HelperReply::Okay: { + Note::Pointer urlNote = reply.notes.find("store-id"); + + // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write. + if (urlNote != NULL && strcmp(urlNote->firstValue(), http->uri) ) { + // Debug section required for some very specific cases. + debugs(85, 9, "Setting storeID with: " << urlNote->firstValue() ); + http->request->store_id = urlNote->firstValue(); + http->store_id = urlNote->firstValue(); + } + } + break; + } + + http->doCallouts(); +} + /** Test cache allow/deny configuration * Sets flags.cachable=1 if caching is not denied. */ void ClientRequestContext::checkNoCache() { if (Config.accessList.noCache) { acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http); acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this); } else { /* unless otherwise specified, we try to cache. */ checkNoCacheDone(ACCESS_ALLOWED); } } static void checkNoCacheDoneWrapper(allow_t answer, void *data) { ClientRequestContext *calloutContext = (ClientRequestContext *) data; @@ -1625,40 +1733,52 @@ if (!calloutContext->redirect_done) { calloutContext->redirect_done = true; assert(calloutContext->redirect_state == REDIRECT_NONE); if (Config.Program.redirect) { debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()"); calloutContext->redirect_state = REDIRECT_PENDING; calloutContext->clientRedirectStart(); return; } } if (!calloutContext->adapted_http_access_done) { debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()"); calloutContext->adapted_http_access_done = true; calloutContext->clientAccessCheck2(); return; } + if (!calloutContext->store_id_done) { + calloutContext->store_id_done = true; + assert(calloutContext->store_id_state == REDIRECT_NONE); + + if (Config.Program.store_id) { + debugs(83, 3,"Doing calloutContext->clientStoreIdStart()"); + calloutContext->store_id_state = REDIRECT_PENDING; + calloutContext->clientStoreIdStart(); + return; + } + } + if (!calloutContext->interpreted_req_hdrs) { debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()"); calloutContext->interpreted_req_hdrs = 1; clientInterpretRequestHeaders(this); } if (!calloutContext->no_cache_done) { calloutContext->no_cache_done = true; if (Config.accessList.noCache && request->flags.cachable) { debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()"); calloutContext->checkNoCache(); return; } } } // if !calloutContext->error if (!calloutContext->tosToClientDone) { calloutContext->tosToClientDone = true; if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) { @@ -1678,42 +1798,42 @@ ch.src_addr = request->client_addr; ch.my_addr = request->my_addr; nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch); if (mark) Ip::Qos::setSockNfmark(getConn()->clientConnection, mark); } } #if USE_SSL // We need to check for SslBump even if the calloutContext->error is set // because bumping may require delaying the error until after CONNECT. if (!calloutContext->sslBumpCheckDone) { calloutContext->sslBumpCheckDone = true; if (calloutContext->sslBumpAccessCheck()) return; /* else no ssl bump required*/ } #endif if (calloutContext->error) { - const char *url = urlCanonical(request); - StoreEntry *e= storeCreateEntry(url, url, request->flags, request->method); + const char *uri = request->storeId(); + StoreEntry *e= storeCreateEntry(uri, uri, request->flags, request->method); #if USE_SSL if (sslBumpNeeded()) { // set final error but delay sending until we bump Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e); errorAppendEntry(e, calloutContext->error); calloutContext->error = NULL; getConn()->setServerBump(srvBump); e->unlock(); } else #endif { // send the error to the client now clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data; clientReplyContext *repContext = dynamic_cast(node->data.getRaw()); assert (repContext); repContext->setReplyToStoreEntry(e); errorAppendEntry(e, calloutContext->error); calloutContext->error = NULL; if (calloutContext->readNextRequest) getConn()->flags.readMore = true; // resume any pipeline reads. === modified file 'src/client_side_request.h' --- src/client_side_request.h 2013-01-30 15:39:37 +0000 +++ src/client_side_request.h 2013-02-03 04:46:54 +0000 @@ -81,40 +81,41 @@ void processRequest(); void httpStart(); bool onlyIfCached()const; bool gotEnough() const; _SQUID_INLINE_ StoreEntry *storeEntry() const; void storeEntry(StoreEntry *); _SQUID_INLINE_ StoreEntry *loggingEntry() const; void loggingEntry(StoreEntry *); _SQUID_INLINE_ ConnStateData * getConn() const; _SQUID_INLINE_ void setConn(ConnStateData *); /** Details of the client socket which produced us. * Treat as read-only for the lifetime of this HTTP request. */ Comm::ConnectionPointer clientConnection; HttpRequest *request; /* Parsed URL ... */ char *uri; char *log_uri; + String store_id; /* Store ID for transactions where the request member is nil */ struct { int64_t offset; int64_t size; size_t headers_sz; } out; HttpHdrRangeIter range_iter; /* data for iterating thru range specs */ size_t req_sz; /* raw request size on input, not current request size */ log_type logType; struct timeval start_time; AccessLogEntry::Pointer al; ///< access.log entry struct { bool accel; //bool intercepted; //XXX: it's apparently never used. //bool spoof_client_ip; //XXX: it's apparently never used. bool internal; bool done_copying; === modified file 'src/redirect.cc' --- src/redirect.cc 2012-11-30 11:08:47 +0000 +++ src/redirect.cc 2013-02-03 07:46:27 +0000 @@ -49,43 +49,46 @@ #include "auth/UserRequest.h" #endif #if USE_SSL #include "ssl/support.h" #endif /// url maximum lengh + extra informations passed to redirector #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024) typedef struct { void *data; char *orig_url; Ip::Address client_addr; const char *client_ident; const char *method_s; HLPCB *handler; } redirectStateData; static HLPCB redirectHandleReply; +static HLPCB storeIdHandleReply; static void redirectStateFree(redirectStateData * r); static helper *redirectors = NULL; +static helper *storeIds = NULL; static OBJH redirectStats; +static OBJH storeIdStats; static int n_bypassed = 0; CBDATA_TYPE(redirectStateData); static void redirectHandleReply(void *data, const HelperReply &reply) { redirectStateData *r = static_cast(data); debugs(61, 5, HERE << "reply=" << reply); // XXX: This function is now kept only to check for and display the garbage use-case // and to map the old helper response format(s) into new format result code and key=value pairs // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format if (reply.result == HelperReply::Unknown) { // BACKWARD COMPATIBILITY 2012-06-15: // Some nasty old helpers send back the entire input line including extra format keys. // This is especially bad for simple perl search-replace filter scripts. // // * trim all but the first word off the response. // * warn once every 50 responses that this will stop being fixed-up soon. @@ -139,88 +142,122 @@ } void *cbdata; if (cbdataReferenceValidDone(r->data, &cbdata)) r->handler(cbdata, newReply); redirectStateFree(r); return; } } } void *cbdata; if (cbdataReferenceValidDone(r->data, &cbdata)) r->handler(cbdata, reply); redirectStateFree(r); } static void +storeIdHandleReply(void *data, const HelperReply &reply) +{ + redirectStateData *r = static_cast(data); + debugs(61, 5,"StoreId helper: reply=" << reply); + + // XXX: This function is now kept only to check for and display the garbage use-case + // and to map the old helper response format(s) into new format result code and key=value pairs + // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format + void *cbdata; + if (cbdataReferenceValidDone(r->data, &cbdata)) + r->handler(cbdata, reply); + + redirectStateFree(r); +} + +static void redirectStateFree(redirectStateData * r) { safe_free(r->orig_url); cbdataFree(r); } static void redirectStats(StoreEntry * sentry) { if (redirectors == NULL) { storeAppendPrintf(sentry, "No redirectors defined\n"); return; } helperStats(sentry, redirectors, "Redirector Statistics"); if (Config.onoff.redirector_bypass) storeAppendPrintf(sentry, "\nNumber of requests bypassed " "because all redirectors were busy: %d\n", n_bypassed); } +static void +storeIdStats(StoreEntry * sentry) +{ + if (storeIds == NULL) { + storeAppendPrintf(sentry, "No StoreId helpers defined\n"); + return; + } + + helperStats(sentry, storeIds, "StoreId helper Statistics"); + + if (Config.onoff.store_id_bypass) + storeAppendPrintf(sentry, "\nNumber of requests bypassed " + "because all StoreId helpers were busy: %d\n", n_bypassed); +} + /**** PUBLIC FUNCTIONS ****/ void redirectStart(ClientHttpRequest * http, HLPCB * handler, void *data) { ConnStateData * conn = http->getConn(); redirectStateData *r = NULL; const char *fqdn; char buf[MAX_REDIRECTOR_REQUEST_STRLEN]; int sz; http_status status; char claddr[MAX_IPSTRLEN]; char myaddr[MAX_IPSTRLEN]; assert(http); assert(handler); debugs(61, 5, "redirectStart: '" << http->uri << "'"); if (Config.onoff.redirector_bypass && redirectors->stats.queue_size) { /* Skip redirector if there is one request queued */ ++n_bypassed; HelperReply bypassReply; bypassReply.result = HelperReply::Okay; bypassReply.notes.add("message","URL rewrite/redirect queue too long. Bypassed."); handler(data, bypassReply); return; } + /** TODO: create a standalone method to initialize + * the cbdata\redirectStateData for all the helpers. + */ r = cbdataAlloc(redirectStateData); r->orig_url = xstrdup(http->uri); if (conn != NULL) r->client_addr = conn->log_addr; else r->client_addr.SetNoAddr(); r->client_ident = NULL; #if USE_AUTH if (http->request->auth_user_request != NULL) { r->client_ident = http->request->auth_user_request->username(); debugs(61, 5, HERE << "auth-user=" << (r->client_ident?r->client_ident:"NULL")); } #endif // HttpRequest initializes with null_string. So we must check both defined() and size() if (!r->client_ident && http->request->extacl_user.defined() && http->request->extacl_user.size()) { r->client_ident = http->request->extacl_user.termedBuf(); debugs(61, 5, HERE << "acl-user=" << (r->client_ident?r->client_ident:"NULL")); } @@ -277,67 +314,222 @@ http->getConn() != NULL && http->getConn()->clientConnection != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr, http->request, NULL, #if USE_AUTH http->getConn() != NULL && http->getConn()->auth_user_request != NULL ? http->getConn()->auth_user_request : http->request->auth_user_request); #else NULL); #endif node = (clientStreamNode *)http->client_stream.tail->data; clientStreamRead(node, http, node->readBuffer); return; } debugs(61,6, HERE << "sending '" << buf << "' to the helper"); helperSubmit(redirectors, buf, redirectHandleReply, r); } +/** + * Handles the StoreID feature helper starting. + * For now it cannot be done using the redirectStart method. + */ +void +storeIdStart(ClientHttpRequest * http, HLPCB * handler, void *data) +{ + ConnStateData * conn = http->getConn(); + redirectStateData *r = NULL; + const char *fqdn; + char buf[MAX_REDIRECTOR_REQUEST_STRLEN]; + int sz; + http_status status; + char claddr[MAX_IPSTRLEN]; + char myaddr[MAX_IPSTRLEN]; + assert(http); + assert(handler); + debugs(61, 5, "storeIdStart: '" << http->uri << "'"); + + if (Config.onoff.store_id_bypass && storeIds->stats.queue_size) { + /* Skip StoreID Helper if there is one request queued */ + ++n_bypassed; + HelperReply bypassReply; + + bypassReply.result = HelperReply::Okay; + + bypassReply.notes.add("message","StoreId helper queue too long. Bypassed."); + handler(data, bypassReply); + return; + } + + r = cbdataAlloc(redirectStateData); + r->orig_url = xstrdup(http->uri); + if (conn != NULL) + r->client_addr = conn->log_addr; + else + r->client_addr.SetNoAddr(); + r->client_ident = NULL; +#if USE_AUTH + if (http->request->auth_user_request != NULL) { + r->client_ident = http->request->auth_user_request->username(); + debugs(61, 5,"auth-user=" << (r->client_ident?r->client_ident:"NULL")); + } +#endif + + // HttpRequest initializes with null_string. So we must check both defined() and size() + if (!r->client_ident && http->request->extacl_user.defined() && http->request->extacl_user.size()) { + r->client_ident = http->request->extacl_user.termedBuf(); + debugs(61, 5,"acl-user=" << (r->client_ident?r->client_ident:"NULL")); + } + + if (!r->client_ident && conn != NULL && conn->clientConnection != NULL && conn->clientConnection->rfc931[0]) { + r->client_ident = conn->clientConnection->rfc931; + debugs(61, 5,"ident-user=" << (r->client_ident?r->client_ident:"NULL")); + } + +#if USE_SSL + + if (!r->client_ident && conn != NULL && Comm::IsConnOpen(conn->clientConnection)) { + r->client_ident = sslGetUserEmail(fd_table[conn->clientConnection->fd].ssl); + debugs(61, 5,"ssl-user=" << (r->client_ident?r->client_ident:"NULL")); + } +#endif + + if (!r->client_ident) + r->client_ident = dash_str; + + r->method_s = RequestMethodStr(http->request->method); + + r->handler = handler; + + r->data = cbdataReference(data); + + if ((fqdn = fqdncache_gethostbyaddr(r->client_addr, 0)) == NULL) + fqdn = dash_str; + + sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s %s/%s %s %s myip=%s myport=%d\n", + r->orig_url, + r->client_addr.NtoA(claddr,MAX_IPSTRLEN), + fqdn, + r->client_ident[0] ? rfc1738_escape(r->client_ident) : dash_str, + r->method_s, + http->request->my_addr.NtoA(myaddr,MAX_IPSTRLEN), + http->request->my_addr.GetPort()); + + if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) { + if (sz<=0) { + status = HTTP_INTERNAL_SERVER_ERROR; + debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to StoreId helper. Request ABORTED."); + } else { + status = HTTP_REQUEST_URI_TOO_LARGE; + debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to redirector exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED."); + } + + clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data; + clientReplyContext *repContext = dynamic_cast(node->data.getRaw()); + assert (repContext); + Ip::Address tmpnoaddr; + tmpnoaddr.SetNoAddr(); + repContext->setReplyToError(ERR_GATEWAY_FAILURE, status, + http->request->method, NULL, + http->getConn() != NULL && http->getConn()->clientConnection != NULL ? + http->getConn()->clientConnection->remote : tmpnoaddr, + http->request, + NULL, +#if USE_AUTH + http->getConn() != NULL && http->getConn()->auth_user_request != NULL ? + http->getConn()->auth_user_request : http->request->auth_user_request); +#else + NULL); +#endif + + node = (clientStreamNode *)http->client_stream.tail->data; + clientStreamRead(node, http, node->readBuffer); + return; + } + + debugs(61,6,"sending '" << buf << "' to the helper"); + helperSubmit(storeIds, buf, storeIdHandleReply, r); +} + static void redirectRegisterWithCacheManager(void) { Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1); + Mgr::RegisterAction("store_id", "StoreId helper Stats", storeIdStats, 0, 1); /* registering the new StoreID statistics in Mgr*/ } void redirectInit(void) { static int init = 0; redirectRegisterWithCacheManager(); - if (!Config.Program.redirect) + /** FIXME: Temporary unified helpers startup + * When and if needed for more helpers a separated startup + * method will be added for each and one of them. + */ + if (!Config.Program.redirect && !Config.Program.store_id) return; - if (redirectors == NULL) - redirectors = new helper("redirector"); + if (Config.Program.redirect){ - redirectors->cmdline = Config.Program.redirect; + if (redirectors == NULL) + redirectors = new helper("redirector"); - redirectors->childs.updateLimits(Config.redirectChildren); + redirectors->cmdline = Config.Program.redirect; + + redirectors->childs.updateLimits(Config.redirectChildren); + + redirectors->ipc_type = IPC_STREAM; + + helperOpenServers(redirectors); + } - redirectors->ipc_type = IPC_STREAM; + if (Config.Program.store_id){ - helperOpenServers(redirectors); + if (storeIds == NULL) + storeIds = new helper("store_id"); + + storeIds->cmdline = Config.Program.store_id; + + storeIds->childs.updateLimits(Config.storeIdChildren); + + storeIds->ipc_type = IPC_STREAM; + + helperOpenServers(storeIds); + } if (!init) { init = 1; CBDATA_INIT_TYPE(redirectStateData); } } void redirectShutdown(void) { - if (!redirectors) + /** FIXME: Temporary unified helpers Shutdown + * When and if needed for more helpers a separated shutdown + * method will be added for each and one of them. + */ + if (!storeIds && !redirectors) return; - helperShutdown(redirectors); + if (redirectors) + helperShutdown(redirectors); + + if (storeIds) + helperShutdown(storeIds); if (!shutting_down) return; delete redirectors; redirectors = NULL; + + delete storeIds; + storeIds = NULL; + } === modified file 'src/redirect.h' --- src/redirect.h 2012-11-04 12:27:49 +0000 +++ src/redirect.h 2013-02-03 04:07:46 +0000 @@ -23,22 +23,23 @@ * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "helper.h" class ClientHttpRequest; void redirectInit(void); void redirectShutdown(void); void redirectStart(ClientHttpRequest *, HLPCB *, void *); +void storeIdStart(ClientHttpRequest *, HLPCB *, void *); #endif /* SQUID_REDIRECT_H_ */ === modified file 'src/store.cc' --- src/store.cc 2013-01-14 11:44:31 +0000 +++ src/store.cc 2013-02-03 07:07:51 +0000 @@ -790,52 +790,55 @@ if (vary.defined()) { /* Again, we own this structure layout */ rep->header.putStr(HDR_X_ACCELERATOR_VARY, vary.termedBuf()); vary.clean(); } #endif pe->replaceHttpReply(rep); pe->timestampsSet(); pe->makePublic(); pe->complete(); pe->unlock(); } newkey = storeKeyPublicByRequest(mem_obj->request); - } else + } else{ + debugs(20, 3,"used with StoreID: " << mem_obj->url); newkey = storeKeyPublic(mem_obj->url, mem_obj->method); - + } if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { debugs(20, 3, "StoreEntry::setPublicKey: Making old '" << mem_obj->url << "' private."); e2->setPrivateKey(); e2->release(); if (mem_obj->request) newkey = storeKeyPublicByRequest(mem_obj->request); - else + else{ newkey = storeKeyPublic(mem_obj->url, mem_obj->method); + } + } if (key) hashDelete(); EBIT_CLR(flags, KEY_PRIVATE); hashInsert(newkey); if (swap_filen > -1) storeDirSwapLog(this, SWAP_LOG_ADD); } StoreEntry * storeCreateEntry(const char *url, const char *log_url, const RequestFlags &flags, const HttpRequestMethod& method) { StoreEntry *e = NULL; MemObject *mem = NULL; debugs(20, 3, "storeCreateEntry: '" << url << "'"); @@ -1677,47 +1680,49 @@ --hot_obj_count; } mem_status = new_status; } const char * StoreEntry::url() const { if (this == NULL) return "[null_entry]"; else if (mem_obj == NULL) return "[null_mem_obj]"; else return mem_obj->url; } void StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl) { + debugs(20, 3, HERE << "A mem-obj created using :" << aUrl); + if (mem_obj) return; if (hidden_mem_obj) { debugs(20, 3, HERE << "restoring " << hidden_mem_obj); mem_obj = hidden_mem_obj; - hidden_mem_obj = NULL; + hidden_mem_obj = NULL;; mem_obj->resetUrls(aUrl, aLogUrl); return; } mem_obj = new MemObject(aUrl, aLogUrl); } /* this just sets DELAY_SENDING */ void StoreEntry::buffer() { EBIT_SET(flags, DELAY_SENDING); } /* this just clears DELAY_SENDING and Invokes the handlers */ void StoreEntry::flush() { if (EBIT_TEST(flags, DELAY_SENDING)) { EBIT_CLR(flags, DELAY_SENDING); === modified file 'src/store_key_md5.cc' --- src/store_key_md5.cc 2012-09-01 14:38:36 +0000 +++ src/store_key_md5.cc 2013-02-03 06:43:47 +0000 @@ -101,71 +101,76 @@ } const cache_key * storeKeyPrivate(const char *url, const HttpRequestMethod& method, int id) { static cache_key digest[SQUID_MD5_DIGEST_LENGTH]; SquidMD5_CTX M; assert(id > 0); debugs(20, 3, "storeKeyPrivate: " << RequestMethodStr(method) << " " << url); SquidMD5Init(&M); SquidMD5Update(&M, (unsigned char *) &id, sizeof(id)); SquidMD5Update(&M, (unsigned char *) &method, sizeof(method)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); SquidMD5Final(digest, &M); return digest; } const cache_key * storeKeyPublic(const char *url, const HttpRequestMethod& method) { + debugs(20, 3, "using: " << RequestMethodStr(method) << " " << url); static cache_key digest[SQUID_MD5_DIGEST_LENGTH]; unsigned char m = (unsigned char) method.id(); SquidMD5_CTX M; SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); SquidMD5Final(digest, &M); + debugs(20, 3, "created public key: " << digest << " for: " << url); return digest; } const cache_key * storeKeyPublicByRequest(HttpRequest * request) { return storeKeyPublicByRequestMethod(request, request->method); } const cache_key * storeKeyPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method) { static cache_key digest[SQUID_MD5_DIGEST_LENGTH]; unsigned char m = (unsigned char) method.id(); - const char *url = urlCanonical(request); + const char *url = request->storeId(); /* storeId returns the right storeID\canonical URL for the md5 calc */ + debugs(20, 3, "using: " << RequestMethodStr(method) << " " << url); SquidMD5_CTX M; SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); - if (request->vary_headers) + if (request->vary_headers){ SquidMD5Update(&M, (unsigned char *) request->vary_headers, strlen(request->vary_headers)); - + debugs(20, 3, "updating public key by vary headers: " << request->vary_headers << " for: " << url); + } SquidMD5Final(digest, &M); + debugs(20, 3, "created public key: " << digest << " for: " << url); return digest; } cache_key * storeKeyDup(const cache_key * key) { cache_key *dup = (cache_key *)memAllocate(MEM_MD5_DIGEST); memcpy(dup, key, SQUID_MD5_DIGEST_LENGTH); return dup; } cache_key * storeKeyCopy(cache_key * dst, const cache_key * src) { memcpy(dst, src, SQUID_MD5_DIGEST_LENGTH); return dst; } void === modified file 'src/store_swapmeta.cc' --- src/store_swapmeta.cc 2012-09-01 14:38:36 +0000 +++ src/store_swapmeta.cc 2013-02-03 04:28:01 +0000 @@ -50,42 +50,51 @@ while ((t = n) != NULL) { n = t->next; xfree(t->value); delete t; } } /* * Build a TLV list for a StoreEntry */ tlv * storeSwapMetaBuild(StoreEntry * e) { tlv *TLV = NULL; /* we'll return this */ tlv **T = &TLV; const char *url; const char *vary; assert(e->mem_obj != NULL); const int64_t objsize = e->mem_obj->expectedReplySize(); assert(e->swap_status == SWAPOUT_WRITING); - url = e->url(); - debugs(20, 3, "storeSwapMetaBuild: " << url ); + + /** Prevents some nasty crash + * In a case a request dosn't exists and storeId() being accessed + * */ + if (e->mem_obj->request) + url = e->mem_obj->request->storeId(); + else + url = e->url(); + + debugs(20, 3, "storeSwapMetaBuild URL: " << url ); + tlv *t = StoreMeta::Factory (STORE_META_KEY,SQUID_MD5_DIGEST_LENGTH, e->key); if (!t) { storeSwapTLVFree(TLV); return NULL; } T = StoreMeta::Add(T, t); t = StoreMeta::Factory(STORE_META_STD_LFS,STORE_HDR_METASIZE,&e->timestamp); if (!t) { storeSwapTLVFree(TLV); return NULL; } T = StoreMeta::Add(T, t); t = StoreMeta::Factory(STORE_META_URL, strlen(url) + 1, url); if (!t) { storeSwapTLVFree(TLV);