=== modified file 'src/BodyPipe.cc' --- src/BodyPipe.cc 2010-03-20 01:53:34 +0000 +++ src/BodyPipe.cc 2010-06-08 19:32:12 +0000 @@ -180,6 +180,15 @@ return bodySize() - thePutSize; // bodySize() asserts that size is known } +void BodyPipe::expectProductionEndAfter(uint64_t size) +{ + const uint64_t expectedSize = thePutSize + size; + if (bodySizeKnown()) + Must(bodySize() == expectedSize); + else + theBodySize = expectedSize; +} + void BodyPipe::clearProducer(bool atEof) { === modified file 'src/BodyPipe.h' --- src/BodyPipe.h 2009-04-09 22:31:13 +0000 +++ src/BodyPipe.h 2010-06-08 19:32:12 +0000 @@ -101,6 +101,7 @@ bool needsMoreData() const { return bodySizeKnown() && unproducedSize() > 0; } uint64_t unproducedSize() const; // size of still unproduced data bool stillProducing(const Producer *producer) const { return theProducer == producer; } + void expectProductionEndAfter(uint64_t extraSize); ///< sets or checks body size // called by consumers bool setConsumerIfNotLate(Consumer *aConsumer); === modified file 'src/ChunkedCodingParser.cc' --- src/ChunkedCodingParser.cc 2010-03-20 01:53:34 +0000 +++ src/ChunkedCodingParser.cc 2010-06-08 19:32:12 +0000 @@ -21,6 +21,7 @@ theChunkSize = theLeftBodySize = 0; doNeedMoreData = false; theIn = theOut = NULL; + useOriginBody = -1; } bool ChunkedCodingParser::parse(MemBuf *rawData, MemBuf *parsedContent) @@ -74,6 +75,10 @@ return; } + // to allow chunk extensions in any chunk, remove this size check + if (size == 0) // is this the last-chunk? + parseChunkExtension(p, theIn->content() + crlfBeg); + theIn->consume(crlfEnd); theChunkSize = theLeftBodySize = size; debugs(94,7, "found chunk: " << theChunkSize); @@ -228,3 +233,35 @@ return false; } +// chunk-extension= *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) +void ChunkedCodingParser::parseChunkExtension(const char *startExt, const char *endExt) +{ + // chunk-extension starts at startExt and ends with LF at endEx + for (const char *p = startExt; p < endExt;) { + + while (*p == ' ' || *p == '\t') ++p; // skip spaces before ';' + + if (*p++ != ';') // each ext name=value pair is preceded with ';' + return; + + while (*p == ' ' || *p == '\t') ++p; // skip spaces before name + + if (p >= endExt) + return; // malformed extension: ';' without ext name=value pair + + const int extSize = endExt - p; + // TODO: we need debugData() stream manipulator to dump data + debugs(94,7, "Found chunk extension; size=" << extSize); + + // TODO: support implied *LWS around '=' + if (extSize > 18 && strncmp(p, "use-original-body=", 18) == 0) { + (void)StringToInt64(p+18, useOriginBody, &p, 10); + debugs(94, 3, HERE << "use-original-body=" << useOriginBody); + return; // remove to support more than just use-original-body + } else { + debugs(94, 5, HERE << "skipping unknown chunk extension"); + // TODO: support quoted-string chunk-ext-val + while (p < endExt && *p != ';') ++p; // skip until the next ';' + } + } +} === modified file 'src/ChunkedCodingParser.h' --- src/ChunkedCodingParser.h 2009-01-21 03:47:47 +0000 +++ src/ChunkedCodingParser.h 2010-06-08 19:32:12 +0000 @@ -80,6 +80,7 @@ void parseTrailerHeader(); void parseMessageEnd(); + void parseChunkExtension(const char *, const char * ); bool findCrlf(size_t &crlfBeg, size_t &crlfEnd); private: @@ -96,6 +97,9 @@ uint64_t theChunkSize; uint64_t theLeftBodySize; bool doNeedMoreData; + +public: + int64_t useOriginBody; }; #endif /* SQUID_CHUNKEDCODINGPARSER_H */ === modified file 'src/adaptation/icap/Config.cc' --- src/adaptation/icap/Config.cc 2010-05-27 00:51:44 +0000 +++ src/adaptation/icap/Config.cc 2010-06-08 19:32:12 +0000 @@ -46,6 +46,7 @@ Adaptation::Icap::Config Adaptation::Icap::TheConfig; Adaptation::Icap::Config::Config(): preview_enable(0), preview_size(0), + allow206_enable(0), connect_timeout_raw(0), io_timeout_raw(0), reuse_connections(0), client_username_header(NULL), client_username_encode(0), repeat(NULL) { === modified file 'src/adaptation/icap/Config.h' --- src/adaptation/icap/Config.h 2009-08-23 09:30:49 +0000 +++ src/adaptation/icap/Config.h 2010-06-08 19:32:12 +0000 @@ -57,6 +57,7 @@ int default_options_ttl; int preview_enable; int preview_size; + int allow206_enable; time_t connect_timeout_raw; time_t io_timeout_raw; int reuse_connections; === modified file 'src/adaptation/icap/Elements.cc' --- src/adaptation/icap/Elements.cc 2009-08-23 09:30:49 +0000 +++ src/adaptation/icap/Elements.cc 2010-06-08 19:32:12 +0000 @@ -11,6 +11,7 @@ const XactOutcome xoError = "ICAP_ERR_OTHER"; const XactOutcome xoOpt = "ICAP_OPT"; const XactOutcome xoEcho = "ICAP_ECHO"; +const XactOutcome xoPartEcho = "ICAP_PART_ECHO"; const XactOutcome xoModified = "ICAP_MOD"; const XactOutcome xoSatisfied = "ICAP_SAT"; === modified file 'src/adaptation/icap/Elements.h' --- src/adaptation/icap/Elements.h 2009-08-23 09:30:49 +0000 +++ src/adaptation/icap/Elements.h 2010-06-08 19:32:12 +0000 @@ -67,6 +67,7 @@ extern const XactOutcome xoError; ///< all kinds of transaction errors extern const XactOutcome xoOpt; ///< OPTION transaction extern const XactOutcome xoEcho; ///< preserved virgin message (ICAP 204) +extern const XactOutcome xoPartEcho; ///< preserved virgin msg part (ICAP 206) extern const XactOutcome xoModified; ///< replaced virgin msg with adapted extern const XactOutcome xoSatisfied; ///< request satisfaction === modified file 'src/adaptation/icap/ModXact.cc' --- src/adaptation/icap/ModXact.cc 2010-05-27 12:44:59 +0000 +++ src/adaptation/icap/ModXact.cc 2010-06-08 19:32:12 +0000 @@ -726,6 +726,10 @@ handle204NoContent(); break; + case 206: + handle206PartialContent(); + break; + default: debugs(93, 5, HERE << "ICAP status " << icapReply->sline.status); handleUnknownScode(); @@ -797,8 +801,9 @@ // server must not respond before the end of preview: we may send ieof Must(preview.enabled() && preview.done() && !preview.ieof()); - // 100 "Continue" cancels our preview commitment, not 204s outside preview - if (!state.allowedPostview204) + // 100 "Continue" cancels our Preview commitment, + // but not commitment to handle 204 or 206 outside Preview + if (!state.allowedPostview204 && !state.allowedPostview206) stopBackup(); state.parsing = State::psIcapHeader; // eventually @@ -823,6 +828,23 @@ prepEchoing(); } +void Adaptation::Icap::ModXact::handle206PartialContent() +{ + if (state.writing == State::writingPaused) { + Must(preview.enabled()); + Must(state.allowedPreview206); + debugs(93, 7, HERE << "206 inside preview"); + } else { + Must(state.writing > State::writingPaused); + Must(state.allowedPostview206); + debugs(93, 7, HERE << "206 outside preview"); + } + state.parsing = State::psHttpHeader; + state.sending = State::sendingAdapted; + state.readyForUob = true; + checkConsuming(); +} + // Called when we receive a 204 No Content response and // when we are trying to bypass a service failure. // We actually start sending (echoig or not) in startSending. @@ -898,6 +920,37 @@ } } +/// Called when we received use-original-body chunk extension in 206 response. +/// We actually start sending (echoing or not) in startSending(). +void Adaptation::Icap::ModXact::prepPartialBodyEchoing(uint64_t pos) +{ + Must(virginBodySending.active()); + Must(virgin.header->body_pipe != NULL); + + setOutcome(xoPartEcho); + + debugs(93, 7, HERE << "will echo virgin body suffix from " << + virgin.header->body_pipe << " offset " << pos ); + + // check that use-original-body=N does not point beyond buffered data + const uint64_t virginDataEnd = virginConsumed + + virgin.body_pipe->buf().contentSize(); + Must(pos <= virginDataEnd); + virginBodySending.progress(static_cast(pos)); + + state.sending = State::sendingVirgin; + checkConsuming(); + + if (virgin.header->body_pipe->bodySizeKnown()) + adapted.body_pipe->expectProductionEndAfter(virgin.header->body_pipe->bodySize() - pos); + + debugs(93, 7, HERE << "will echo virgin body suffix to " << + adapted.body_pipe); + + // Start echoing data + echoMore(); +} + void Adaptation::Icap::ModXact::handleUnknownScode() { stopParsing(); @@ -997,6 +1050,13 @@ } if (parsed) { + if (state.readyForUob && bodyParser->useOriginBody >= 0) { + prepPartialBodyEchoing( + static_cast(bodyParser->useOriginBody)); + stopParsing(); + return; + } + stopParsing(); stopSending(true); // the parser succeeds only if all parsed data fits return; @@ -1235,19 +1295,11 @@ if (preview.enabled()) { buf.Printf("Preview: %d\r\n", (int)preview.ad()); - if (virginBody.expected()) // there is a body to preview - virginBodySending.plan(); - else + if (!virginBody.expected()) // there is no body to preview finishNullOrEmptyBodyPreview(httpBuf); } - if (shouldAllow204()) { - debugs(93,5, HERE << "will allow 204s outside of preview"); - state.allowedPostview204 = true; - buf.Printf("Allow: 204\r\n"); - if (virginBody.expected()) // there is a body to echo - virginBodySending.plan(); - } + makeAllowHeader(buf); if (TheConfig.send_client_ip && request) { Ip::Address client_addr; @@ -1277,6 +1329,44 @@ httpBuf.clean(); } +// decides which Allow values to write and updates the request buffer +void Adaptation::Icap::ModXact::makeAllowHeader(MemBuf &buf) +{ + const bool allow204in = preview.enabled(); // TODO: add shouldAllow204in() + const bool allow204out = state.allowedPostview204 = shouldAllow204(); + const bool allow206in = state.allowedPreview206 = shouldAllow206in(); + const bool allow206out = state.allowedPostview206 = shouldAllow206out(); + + debugs(93,9, HERE << "Allows: " << allow204in << allow204out << + allow206in << allow206out); + + const bool allow204 = allow204in || allow204out; + const bool allow206 = allow206in || allow206out; + + if (!allow204 && !allow206) + return; // nothing to do + + if (virginBody.expected()) // if there is a virgin body, plan to send it + virginBodySending.plan(); + + // writing Preview:... means we will honor 204 inside preview + // writing Allow/204 means we will honor 204 outside preview + // writing Allow:206 means we will honor 206 inside preview + // writing Allow:204,206 means we will honor 206 outside preview + const char *allowHeader = NULL; + if (allow204out && allow206) + allowHeader = "Allow: 204, 206\r\n"; + else if (allow204out) + allowHeader = "Allow: 204\r\n"; + else if (allow206) + allowHeader = "Allow: 206\r\n"; + + if (allowHeader) { // may be nil if only allow204in is true + buf.append(allowHeader, strlen(allowHeader)); + debugs(93,5, HERE << "Will write " << allowHeader); + } +} + void Adaptation::Icap::ModXact::makeUsernameHeader(const HttpRequest *request, MemBuf &buf) { if (request->auth_user_request != NULL) { @@ -1379,6 +1469,25 @@ return canBackupEverything(); } +// decides whether to allow 206 responses in some mode +bool Adaptation::Icap::ModXact::shouldAllow206any() +{ + return TheConfig.allow206_enable && service().allows206() && + virginBody.expected(); // no need for 206 without a body +} + +// decides whether to allow 206 responses in preview mode +bool Adaptation::Icap::ModXact::shouldAllow206in() +{ + return shouldAllow206any() && preview.enabled(); +} + +// decides whether to allow 206 responses outside of preview +bool Adaptation::Icap::ModXact::shouldAllow206out() +{ + return shouldAllow206any() && canBackupEverything(); +} + // used by shouldAllow204 and decideOnRetries bool Adaptation::Icap::ModXact::canBackupEverything() const { @@ -1461,6 +1570,9 @@ if (!doneSending() && state.sending != State::sendingUndecided) buf.Printf("S(%d)", state.sending); + if (state.readyForUob) + buf.append("6", 1); + if (canStartBypass) buf.append("Y", 1); === modified file 'src/adaptation/icap/ModXact.h' --- src/adaptation/icap/ModXact.h 2009-08-23 09:30:49 +0000 +++ src/adaptation/icap/ModXact.h 2010-06-08 19:32:12 +0000 @@ -195,6 +195,7 @@ bool virginBodyEndReached(const VirginBodyAct &act) const; void makeRequestHeaders(MemBuf &buf); + void makeAllowHeader(MemBuf &buf); void makeUsernameHeader(const HttpRequest *request, MemBuf &buf); void addLastRequestChunk(MemBuf &buf); void openChunk(MemBuf &buf, size_t chunkSize, bool ieof); @@ -205,6 +206,9 @@ void decideOnPreview(); void decideOnRetries(); bool shouldAllow204(); + bool shouldAllow206any(); + bool shouldAllow206in(); + bool shouldAllow206out(); bool canBackupEverything() const; void prepBackup(size_t expectedSize); @@ -225,6 +229,7 @@ bool validate200Ok(); void handle200Ok(); void handle204NoContent(); + void handle206PartialContent(); void handleUnknownScode(); void bypassFailure(); @@ -233,6 +238,7 @@ void disableBypass(const char *reason, bool includeGroupBypass); void prepEchoing(); + void prepPartialBodyEchoing(uint64_t pos); void echoMore(); virtual bool doneAll() const; @@ -281,6 +287,9 @@ bool serviceWaiting; // waiting for ICAP service options bool allowedPostview204; // mmust handle 204 No Content outside preview + bool allowedPostview206; // must handle 206 Partial Content outside preview + bool allowedPreview206; // must handle 206 Partial Content inside preview + bool readyForUob; ///< got a 206 response and expect a use-origin-body // will not write anything [else] to the ICAP server connection bool doneWriting() const { return writing == writingReallyDone; } @@ -288,7 +297,8 @@ // will not use virgin.body_pipe bool doneConsumingVirgin() const { return writing >= writingAlmostDone - && (sending == sendingAdapted || sending == sendingDone); + && ((sending == sendingAdapted && !readyForUob) || + sending == sendingDone); } // parsed entire ICAP response from the ICAP server === modified file 'src/adaptation/icap/OptXact.cc' --- src/adaptation/icap/OptXact.cc 2010-03-20 01:53:34 +0000 +++ src/adaptation/icap/OptXact.cc 2010-06-08 19:32:47 +0000 @@ -8,6 +8,7 @@ #include "adaptation/icap/OptXact.h" #include "adaptation/icap/Options.h" +#include "adaptation/icap/Config.h" #include "base/TextException.h" #include "SquidTime.h" #include "HttpRequest.h" @@ -49,6 +50,8 @@ buf.Printf("OPTIONS " SQUIDSTRINGPH " ICAP/1.0\r\n", SQUIDSTRINGPRINT(uri)); const String host = s.cfg().host; buf.Printf("Host: " SQUIDSTRINGPH ":%d\r\n", SQUIDSTRINGPRINT(host), s.cfg().port); + if (TheConfig.allow206_enable) + buf.Printf("Allow: 206\r\n"); buf.append(ICAP::crlf, 2); // XXX: HttpRequest cannot fully parse ICAP Request-Line === modified file 'src/adaptation/icap/Options.cc' --- src/adaptation/icap/Options.cc 2010-03-20 01:53:34 +0000 +++ src/adaptation/icap/Options.cc 2010-06-08 19:32:12 +0000 @@ -8,6 +8,7 @@ Adaptation::Icap::Options::Options(): error("unconfigured"), max_connections(-1), allow204(false), + allow206(false), preview(-1), theTTL(-1) { theTransfers.preview.name = "Transfer-Preview"; @@ -104,6 +105,9 @@ if (h->hasListMember(HDR_ALLOW, "204", ',')) allow204 = true; + if (h->hasListMember(HDR_ALLOW, "206", ',')) + allow206 = true; + cfgIntHeader(h, "Preview", preview); cfgTransferList(h, theTransfers.preview); === modified file 'src/adaptation/icap/Options.h' --- src/adaptation/icap/Options.h 2009-08-23 09:30:49 +0000 +++ src/adaptation/icap/Options.h 2010-06-08 19:32:12 +0000 @@ -81,6 +81,7 @@ String serviceId; int max_connections; bool allow204; + bool allow206; int preview; protected: === modified file 'src/adaptation/icap/ServiceRep.cc' --- src/adaptation/icap/ServiceRep.cc 2010-05-27 11:16:08 +0000 +++ src/adaptation/icap/ServiceRep.cc 2010-06-08 19:32:12 +0000 @@ -125,6 +125,14 @@ return true; // in the future, we may have ACLs to prevent 204s } +bool Adaptation::Icap::ServiceRep::allows206() const +{ + Must(hasOptions()); + if (theOptions->allow206) + return true; // in the future, we may have ACLs to prevent 206s + return false; +} + static void ServiceRep_noteTimeToUpdate(void *data) === modified file 'src/adaptation/icap/ServiceRep.h' --- src/adaptation/icap/ServiceRep.h 2010-05-27 00:51:44 +0000 +++ src/adaptation/icap/ServiceRep.h 2010-06-08 19:32:12 +0000 @@ -103,6 +103,7 @@ bool wantsUrl(const String &urlPath) const; bool wantsPreview(const String &urlPath, size_t &wantedSize) const; bool allows204() const; + bool allows206() const; void noteFailure(); // called by transactions to report service failure === modified file 'src/cf.data.pre' --- src/cf.data.pre 2010-06-03 07:49:20 +0000 +++ src/cf.data.pre 2010-06-08 19:32:12 +0000 @@ -5964,6 +5964,28 @@ basis by OPTIONS requests. DOC_END +NAME: icap_206_enable +TYPE: onoff +IFDEF: ICAP_CLIENT +COMMENT: on|off +LOC: Adaptation::Icap::TheConfig.allow206_enable +DEFAULT: on +DOC_START + 206 (Partial Content) responses is an ICAP extension that allows the + ICAP agents to optionally combine adapted and original HTTP message + content. The decision to combine is postponed until the end of the + ICAP response. Squid supports Partial Content extension by default. + + Activation of the Partial Content extension is negotiated with each + ICAP service during OPTIONS exchange. Most ICAP servers should handle + negotation correctly even if they do not support the extension, but + some might fail. To disable Partial Content support for all ICAP + services and to avoid any negotiation, set this option to "off". + + Example: + icap_206_enable off +DOC_END + NAME: icap_default_options_ttl TYPE: int IFDEF: ICAP_CLIENT