ModXact.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 93 ICAP (RFC 3507) Client */
10 
11 #include "squid.h"
12 #include "AccessLogEntry.h"
13 #include "adaptation/Answer.h"
14 #include "adaptation/History.h"
15 #include "adaptation/icap/Client.h"
16 #include "adaptation/icap/Config.h"
21 #include "adaptation/Initiator.h"
22 #include "auth/UserRequest.h"
23 #include "base/TextException.h"
24 #include "base64.h"
25 #include "comm.h"
26 #include "comm/Connection.h"
27 #include "error/Detail.h"
30 #include "HttpHeaderTools.h"
31 #include "HttpReply.h"
32 #include "MasterXaction.h"
33 #include "parser/Tokenizer.h"
34 #include "sbuf/Stream.h"
35 
36 // flow and terminology:
37 // HTTP| --> receive --> encode --> write --> |network
38 // end | <-- send <-- parse <-- read <-- |end
39 
40 // TODO: replace gotEncapsulated() with something faster; we call it often
41 
44 
45 static const size_t TheBackupLimit = BodyPipe::MaxCapacity;
46 
48 
50 {
51  memset(this, 0, sizeof(*this));
52 }
53 
56  AsyncJob("Adaptation::Icap::ModXact"),
57  Adaptation::Icap::Xaction("Adaptation::Icap::ModXact", aService),
58  virginConsumed(0),
60  canStartBypass(false), // too early
64  adaptHistoryId(-1),
65  trailerParser(nullptr),
66  alMaster(alp)
67 {
68  assert(virginHeader);
69 
70  virgin.setHeader(virginHeader); // sets virgin.body_pipe if needed
71  virgin.setCause(virginCause); // may be NULL
72 
73  // adapted header and body are initialized when we parse them
74 
75  // writing and reading ends are handled by Adaptation::Icap::Xaction
76 
77  // encoding
78  // nothing to do because we are using temporary buffers
79 
80  // parsing; TODO: do not set until we parse, see ICAPOptXact
81  icapReply = new HttpReply;
82  icapReply->protoPrefix = "ICAP/"; // TODO: make an IcapReply class?
83 
84  debugs(93,7, "initialized." << status());
85 }
86 
87 // initiator wants us to start
89 {
91 
92  // reserve an adaptation history slot (attempts are known at this time)
93  Adaptation::History::Pointer ah = virginRequest().adaptLogHistory();
94  if (ah != NULL)
95  adaptHistoryId = ah->recordXactStart(service().cfg().key, icap_tr_start, attempts > 1);
96 
97  estimateVirginBody(); // before virgin disappears!
98 
99  canStartBypass = service().cfg().bypass;
100 
101  // it is an ICAP violation to send request to a service w/o known OPTIONS
102  // and the service may is too busy for us: honor Max-Connections and such
103  if (service().up() && service().availableForNew())
104  startWriting();
105  else
106  waitForService();
107 }
108 
110 {
111  const char *comment;
112  Must(!state.serviceWaiting);
113 
114  if (!service().up()) {
115  AsyncCall::Pointer call = JobCallback(93,5,
117 
118  service().callWhenReady(call);
119  comment = "to be up";
120  } else {
121  //The service is unavailable because of max-connection or other reason
122 
123  if (service().cfg().onOverload != srvWait) {
124  // The service is overloaded, but waiting to be available prohibited by
125  // user configuration (onOverload is set to "block" or "bypass")
126  if (service().cfg().onOverload == srvBlock)
127  disableBypass("not available", true);
128  else //if (service().cfg().onOverload == srvBypass)
129  canStartBypass = true;
130 
131  disableRetries();
132  disableRepeats("ICAP service is not available");
133 
134  debugs(93, 7, "will not wait for the service to be available" <<
135  status());
136 
137  throw TexcHere("ICAP service is not available");
138  }
139 
140  AsyncCall::Pointer call = JobCallback(93,5,
142  service().callWhenAvailable(call, state.waitedForService);
143  comment = "to be available";
144  }
145 
146  debugs(93, 7, "will wait for the service " << comment << status());
147  state.serviceWaiting = true; // after callWhenReady() which may throw
148  state.waitedForService = true;
149 }
150 
152 {
153  Must(state.serviceWaiting);
154  state.serviceWaiting = false;
155 
156  if (!service().up()) {
157  disableRetries();
158  disableRepeats("ICAP service is unusable");
159  throw TexcHere("ICAP service is unusable");
160  }
161 
162  if (service().availableForOld())
163  startWriting();
164  else
165  waitForService();
166 }
167 
169 {
170  Must(state.serviceWaiting);
171  state.serviceWaiting = false;
172 
173  if (service().up() && service().availableForOld())
174  startWriting();
175  else
176  waitForService();
177 }
178 
180 {
181  state.writing = State::writingConnect;
182 
183  decideOnPreview(); // must be decided before we decideOnRetries
184  decideOnRetries();
185 
186  openConnection();
187 }
188 
190 {
191  Must(state.writing == State::writingConnect);
192 
193  startReading(); // wait for early errors from the ICAP server
194 
195  MemBuf requestBuf;
196  requestBuf.init();
197 
198  makeRequestHeaders(requestBuf);
199  debugs(93, 9, "will write" << status() << ":\n" <<
200  (requestBuf.terminate(), requestBuf.content()));
201 
202  // write headers
203  state.writing = State::writingHeaders;
204  icap_tio_start = current_time;
205  scheduleWrite(requestBuf);
206 }
207 
209 {
210  debugs(93, 5, "Wrote " << sz << " bytes");
211 
212  if (state.writing == State::writingHeaders)
213  handleCommWroteHeaders();
214  else
215  handleCommWroteBody();
216 }
217 
219 {
220  Must(state.writing == State::writingHeaders);
221 
222  // determine next step
223  if (preview.enabled()) {
224  if (preview.done())
225  decideWritingAfterPreview("zero-size");
226  else
227  state.writing = State::writingPreview;
228  } else if (virginBody.expected()) {
229  state.writing = State::writingPrime;
230  } else {
231  stopWriting(true);
232  return;
233  }
234 
235  writeMore();
236 }
237 
239 {
240  debugs(93, 5, "checking whether to write more" << status());
241 
242  if (writer != NULL) // already writing something
243  return;
244 
245  switch (state.writing) {
246 
247  case State::writingInit: // waiting for service OPTIONS
248  Must(state.serviceWaiting);
249  return;
250 
251  case State::writingConnect: // waiting for the connection to establish
252  case State::writingHeaders: // waiting for the headers to be written
253  case State::writingPaused: // waiting for the ICAP server response
254  case State::writingReallyDone: // nothing more to write
255  return;
256 
257  case State::writingAlmostDone: // was waiting for the last write
258  stopWriting(false);
259  return;
260 
261  case State::writingPreview:
262  writePreviewBody();
263  return;
264 
265  case State::writingPrime:
266  writePrimeBody();
267  return;
268 
269  default:
270  throw TexcHere("Adaptation::Icap::ModXact in bad writing state");
271  }
272 }
273 
275 {
276  debugs(93, 8, "will write Preview body from " <<
277  virgin.body_pipe << status());
278  Must(state.writing == State::writingPreview);
279  Must(virgin.body_pipe != NULL);
280 
281  const size_t sizeMax = (size_t)virgin.body_pipe->buf().contentSize();
282  const size_t size = min(preview.debt(), sizeMax);
283  writeSomeBody("preview body", size);
284 
285  // change state once preview is written
286 
287  if (preview.done())
288  decideWritingAfterPreview("body");
289 }
290 
293 {
294  if (preview.ieof()) // nothing more to write
295  stopWriting(true);
296  else if (state.parsing == State::psIcapHeader) // did not get a reply yet
297  state.writing = State::writingPaused; // wait for the ICAP server reply
298  else
299  stopWriting(true); // ICAP server reply implies no post-preview writing
300 
301  debugs(93, 6, "decided on writing after " << kind << " preview" <<
302  status());
303 }
304 
306 {
307  Must(state.writing == State::writingPrime);
308  Must(virginBodyWriting.active());
309 
310  const size_t size = (size_t)virgin.body_pipe->buf().contentSize();
311  writeSomeBody("prime virgin body", size);
312 
313  if (virginBodyEndReached(virginBodyWriting)) {
314  debugs(93, 5, "wrote entire body");
315  stopWriting(true);
316  }
317 }
318 
319 void Adaptation::Icap::ModXact::writeSomeBody(const char *label, size_t size)
320 {
321  Must(!writer && state.writing < state.writingAlmostDone);
322  Must(virgin.body_pipe != NULL);
323  debugs(93, 8, "will write up to " << size << " bytes of " <<
324  label);
325 
326  MemBuf writeBuf; // TODO: suggest a min size based on size and lastChunk
327 
328  writeBuf.init(); // note: we assume that last-chunk will fit
329 
330  const size_t writableSize = virginContentSize(virginBodyWriting);
331  const size_t chunkSize = min(writableSize, size);
332 
333  if (chunkSize) {
334  debugs(93, 7, "will write " << chunkSize <<
335  "-byte chunk of " << label);
336 
337  openChunk(writeBuf, chunkSize, false);
338  writeBuf.append(virginContentData(virginBodyWriting), chunkSize);
339  closeChunk(writeBuf);
340 
341  virginBodyWriting.progress(chunkSize);
342  virginConsume();
343  } else {
344  debugs(93, 7, "has no writable " << label << " content");
345  }
346 
347  const bool wroteEof = virginBodyEndReached(virginBodyWriting);
348  bool lastChunk = wroteEof;
349  if (state.writing == State::writingPreview) {
350  preview.wrote(chunkSize, wroteEof); // even if wrote nothing
351  lastChunk = lastChunk || preview.done();
352  }
353 
354  if (lastChunk) {
355  debugs(93, 8, "will write last-chunk of " << label);
356  addLastRequestChunk(writeBuf);
357  }
358 
359  debugs(93, 7, "will write " << writeBuf.contentSize()
360  << " raw bytes of " << label);
361 
362  if (writeBuf.hasContent()) {
363  scheduleWrite(writeBuf); // comm will free the chunk
364  } else {
365  writeBuf.clean();
366  }
367 }
368 
370 {
371  const bool ieof = state.writing == State::writingPreview && preview.ieof();
372  openChunk(buf, 0, ieof);
373  closeChunk(buf);
374 }
375 
376 void Adaptation::Icap::ModXact::openChunk(MemBuf &buf, size_t chunkSize, bool ieof)
377 {
378  buf.appendf((ieof ? "%x; ieof\r\n" : "%x\r\n"), (int) chunkSize);
379 }
380 
382 {
383  buf.append(ICAP::crlf, 2); // chunk-terminating CRLF
384 }
385 
387 {
388  const HttpRequest *request = virgin.cause ?
389  virgin.cause : dynamic_cast<const HttpRequest*>(virgin.header);
390  Must(request);
391  return *request;
392 }
393 
394 // did the activity reached the end of the virgin body?
396 {
397  return
398  !act.active() || // did all (assuming it was originally planned)
399  !virgin.body_pipe->expectMoreAfter(act.offset()); // will not have more
400 }
401 
402 // the size of buffered virgin body data available for the specified activity
403 // if this size is zero, we may be done or may be waiting for more data
405 {
406  Must(act.active());
407  // asbolute start of unprocessed data
408  const uint64_t dataStart = act.offset();
409  // absolute end of buffered data
410  const uint64_t dataEnd = virginConsumed + virgin.body_pipe->buf().contentSize();
411  Must(virginConsumed <= dataStart && dataStart <= dataEnd);
412  return static_cast<size_t>(dataEnd - dataStart);
413 }
414 
415 // pointer to buffered virgin body data available for the specified activity
417 {
418  Must(act.active());
419  const uint64_t dataStart = act.offset();
420  Must(virginConsumed <= dataStart);
421  return virgin.body_pipe->buf().content() + static_cast<size_t>(dataStart-virginConsumed);
422 }
423 
425 {
426  debugs(93, 9, "consumption guards: " << !virgin.body_pipe << isRetriable <<
427  isRepeatable << canStartBypass << protectGroupBypass);
428 
429  if (!virgin.body_pipe)
430  return; // nothing to consume
431 
432  if (isRetriable)
433  return; // do not consume if we may have to retry later
434 
435  BodyPipe &bp = *virgin.body_pipe;
436  const bool wantToPostpone = isRepeatable || canStartBypass || protectGroupBypass;
437 
438  // Why > 2? HttpState does not use the last bytes in the buffer
439  // because Client::delayRead() is arguably broken. See
440  // HttpStateData::maybeReadVirginBody for more details.
441  if (wantToPostpone && bp.buf().spaceSize() > 2) {
442  // Postponing may increase memory footprint and slow the HTTP side
443  // down. Not postponing may increase the number of ICAP errors
444  // if the ICAP service fails. We may also use "potential" space to
445  // postpone more aggressively. Should the trade-off be configurable?
446  debugs(93, 8, "postponing consumption from " << bp.status());
447  return;
448  }
449 
450  const size_t have = static_cast<size_t>(bp.buf().contentSize());
451  const uint64_t end = virginConsumed + have;
452  uint64_t offset = end;
453 
454  debugs(93, 9, "max virgin consumption offset=" << offset <<
455  " acts " << virginBodyWriting.active() << virginBodySending.active() <<
456  " consumed=" << virginConsumed <<
457  " from " << virgin.body_pipe->status());
458 
459  if (virginBodyWriting.active())
460  offset = min(virginBodyWriting.offset(), offset);
461 
462  if (virginBodySending.active())
463  offset = min(virginBodySending.offset(), offset);
464 
465  Must(virginConsumed <= offset && offset <= end);
466 
467  if (const size_t size = static_cast<size_t>(offset - virginConsumed)) {
468  debugs(93, 8, "consuming " << size << " out of " << have <<
469  " virgin body bytes");
470  bp.consume(size);
471  virginConsumed += size;
472  Must(!isRetriable); // or we should not be consuming
473  disableRepeats("consumed content");
474  disableBypass("consumed content", true);
475  }
476 }
477 
479 {
480  writeMore();
481 }
482 
483 // Called when we do not expect to call comm_write anymore.
484 // We may have a pending write though.
485 // If stopping nicely, we will just wait for that pending write, if any.
487 {
488  if (state.writing == State::writingReallyDone)
489  return;
490 
491  if (writer != NULL) {
492  if (nicely) {
493  debugs(93, 7, "will wait for the last write" << status());
494  state.writing = State::writingAlmostDone; // may already be set
495  checkConsuming();
496  return;
497  }
498  debugs(93, 3, "will NOT wait for the last write" << status());
499 
500  // Comm does not have an interface to clear the writer callback nicely,
501  // but without clearing the writer we cannot recycle the connection.
502  // We prevent connection reuse and hope that we can handle a callback
503  // call at any time, usually in the middle of the destruction sequence!
504  // Somebody should add comm_remove_write_handler() to comm API.
505  reuseConnection = false;
506  ignoreLastWrite = true;
507  }
508 
509  debugs(93, 7, "will no longer write" << status());
510  if (virginBodyWriting.active()) {
511  virginBodyWriting.disable();
512  virginConsume();
513  }
514  state.writing = State::writingReallyDone;
515  checkConsuming();
516 }
517 
519 {
520  if (!virginBodySending.active())
521  return;
522 
523  debugs(93, 7, "will no longer backup" << status());
524  virginBodySending.disable();
525  virginConsume();
526 }
527 
529 {
530  return Adaptation::Icap::Xaction::doneAll() && !state.serviceWaiting &&
531  doneSending() &&
532  doneReading() && state.doneWriting();
533 }
534 
536 {
537  Must(haveConnection());
538  Must(!reader);
539  Must(!adapted.header);
540  Must(!adapted.body_pipe);
541 
542  // we use the same buffer for headers and body and then consume headers
543  readMore();
544 }
545 
547 {
548  if (reader != NULL || doneReading()) {
549  debugs(93,3, "returning from readMore because reader or doneReading()");
550  return;
551  }
552 
553  // do not fill readBuf if we have no space to store the result
554  if (adapted.body_pipe != NULL &&
555  !adapted.body_pipe->buf().hasPotentialSpace()) {
556  debugs(93,3, "not reading because ICAP reply pipe is full");
557  return;
558  }
559 
560  if (readBuf.length() < SQUID_TCP_SO_RCVBUF)
561  scheduleRead();
562  else
563  debugs(93,3, "cannot read with a full buffer");
564 }
565 
566 // comm module read a portion of the ICAP response for us
568 {
569  Must(!state.doneParsing());
570  icap_tio_finish = current_time;
571  parseMore();
572  readMore();
573 }
574 
576 {
577  Must(state.sending == State::sendingVirgin);
578  Must(adapted.body_pipe != NULL);
579  Must(virginBodySending.active());
580 
581  const size_t sizeMax = virginContentSize(virginBodySending);
582  debugs(93,5, "will echo up to " << sizeMax << " bytes from " <<
583  virgin.body_pipe->status());
584  debugs(93,5, "will echo up to " << sizeMax << " bytes to " <<
585  adapted.body_pipe->status());
586 
587  if (sizeMax > 0) {
588  const size_t size = adapted.body_pipe->putMoreData(virginContentData(virginBodySending), sizeMax);
589  debugs(93,5, "echoed " << size << " out of " << sizeMax <<
590  " bytes");
591  virginBodySending.progress(size);
592  disableRepeats("echoed content");
593  disableBypass("echoed content", true);
594  virginConsume();
595  }
596 
597  if (virginBodyEndReached(virginBodySending)) {
598  debugs(93, 5, "echoed all" << status());
599  stopSending(true);
600  } else {
601  debugs(93, 5, "has " <<
602  virgin.body_pipe->buf().contentSize() << " bytes " <<
603  "and expects more to echo" << status());
604  // TODO: timeout if virgin or adapted pipes are broken
605  }
606 }
607 
609 {
610  return state.sending == State::sendingDone;
611 }
612 
613 // stop (or do not start) sending adapted message body
615 {
616  debugs(93, 7, "Enter stop sending ");
617  if (doneSending())
618  return;
619  debugs(93, 7, "Proceed with stop sending ");
620 
621  if (state.sending != State::sendingUndecided) {
622  debugs(93, 7, "will no longer send" << status());
623  if (adapted.body_pipe != NULL) {
624  virginBodySending.disable();
625  // we may leave debts if we were echoing and the virgin
626  // body_pipe got exhausted before we echoed all planned bytes
627  const bool leftDebts = adapted.body_pipe->needsMoreData();
628  stopProducingFor(adapted.body_pipe, nicely && !leftDebts);
629  }
630  } else {
631  debugs(93, 7, "will not start sending" << status());
632  Must(!adapted.body_pipe);
633  }
634 
635  state.sending = State::sendingDone;
636  checkConsuming();
637 }
638 
639 // should be called after certain state.writing or state.sending changes
641 {
642  // quit if we already stopped or are still using the pipe
643  if (!virgin.body_pipe || !state.doneConsumingVirgin())
644  return;
645 
646  debugs(93, 7, "will stop consuming" << status());
647  stopConsumingFrom(virgin.body_pipe);
648 }
649 
651 {
652  debugs(93, 5, "have " << readBuf.length() << " bytes to parse" << status());
653  debugs(93, 5, "\n" << readBuf);
654 
655  if (state.parsingHeaders())
656  parseHeaders();
657 
658  if (state.parsing == State::psBody)
659  parseBody();
660 
661  if (state.parsing == State::psIcapTrailer)
662  parseIcapTrailer();
663 }
664 
665 void Adaptation::Icap::ModXact::callException(const std::exception &e)
666 {
667  if (!canStartBypass || isRetriable) {
668  if (!isRetriable) {
669  if (const TextException *te = dynamic_cast<const TextException *>(&e))
670  detailError(new ExceptionErrorDetail(te->id()));
671  else
672  detailError(new ExceptionErrorDetail(Here().id()));
673  }
675  return;
676  }
677 
678  try {
679  debugs(93, 3, "bypassing " << inCall << " exception: " <<
680  e.what() << ' ' << status());
681  bypassFailure();
682  } catch (const TextException &bypassTe) {
683  detailError(new ExceptionErrorDetail(bypassTe.id()));
685  } catch (const std::exception &bypassE) {
686  detailError(new ExceptionErrorDetail(Here().id()));
688  }
689 }
690 
692 {
693  disableBypass("already started to bypass", false);
694 
695  Must(!isRetriable); // or we should not be bypassing
696  // TODO: should the same be enforced for isRepeatable? Check icap_repeat??
697 
698  prepEchoing();
699 
700  startSending();
701 
702  // end all activities associated with the ICAP server
703 
704  stopParsing(false);
705 
706  stopWriting(true); // or should we force it?
707  if (haveConnection()) {
708  reuseConnection = false; // be conservative
709  cancelRead(); // may not work; and we cannot stop connecting either
710  if (!doneWithIo())
711  debugs(93, 7, "Warning: bypass failed to stop I/O" << status());
712  }
713 
714  service().noteFailure(); // we are bypassing, but this is still a failure
715 }
716 
717 void Adaptation::Icap::ModXact::disableBypass(const char *reason, bool includingGroupBypass)
718 {
719  if (canStartBypass) {
720  debugs(93,7, "will never start bypass because " << reason);
721  canStartBypass = false;
722  }
723  if (protectGroupBypass && includingGroupBypass) {
724  debugs(93,7, "not protecting group bypass because " << reason);
725  protectGroupBypass = false;
726  }
727 }
728 
729 // note that allocation for echoing is done in handle204NoContent()
731 {
732  if (adapted.header) // already allocated
733  return;
734 
735  if (gotEncapsulated("res-hdr")) {
736  adapted.setHeader(new HttpReply);
737  setOutcome(service().cfg().method == ICAP::methodReqmod ?
739  } else if (gotEncapsulated("req-hdr")) {
740  adapted.setHeader(new HttpRequest(virginRequest().masterXaction));
741  setOutcome(xoModified);
742  } else
743  throw TexcHere("Neither res-hdr nor req-hdr in maybeAllocateHttpMsg()");
744 }
745 
747 {
748  Must(state.parsingHeaders());
749 
750  if (state.parsing == State::psIcapHeader) {
751  debugs(93, 5, "parse ICAP headers");
752  parseIcapHead();
753  }
754 
755  if (state.parsing == State::psHttpHeader) {
756  debugs(93, 5, "parse HTTP headers");
757  parseHttpHead();
758  }
759 
760  if (state.parsingHeaders()) { // need more data
761  Must(mayReadMore());
762  return;
763  }
764 
765  startSending();
766 }
767 
768 // called after parsing all headers or when bypassing an exception
770 {
771  disableRepeats("sent headers");
772  disableBypass("sent headers", true);
773  sendAnswer(Answer::Forward(adapted.header));
774 
775  if (state.sending == State::sendingVirgin)
776  echoMore();
777  else {
778  // If we are not using the virgin HTTP object update the
779  // Http::Message::sources flag.
780  // The state.sending may set to State::sendingVirgin in the case
781  // of 206 responses too, where we do not want to update Http::Message::sources
782  // flag. However even for 206 responses the state.sending is
783  // not set yet to sendingVirgin. This is done in later step
784  // after the parseBody method called.
785  updateSources();
786  }
787 }
788 
790 {
791  Must(state.sending == State::sendingUndecided);
792 
793  if (!parseHead(icapReply.getRaw()))
794  return;
795 
796  if (expectIcapTrailers()) {
797  Must(!trailerParser);
798  trailerParser = new TrailerParser;
799  }
800 
801  static SBuf close("close", 5);
802  if (httpHeaderHasConnDir(&icapReply->header, close)) {
803  debugs(93, 5, "found connection close");
804  reuseConnection = false;
805  }
806 
807  switch (icapReply->sline.status()) {
808 
809  case Http::scContinue:
810  handle100Continue();
811  break;
812 
813  case Http::scOkay:
814  case Http::scCreated: // Symantec Scan Engine 5.0 and later when modifying HTTP msg
815 
816  if (!validate200Ok()) {
817  throw TexcHere("Invalid ICAP Response");
818  } else {
819  handle200Ok();
820  }
821 
822  break;
823 
824  case Http::scNoContent:
825  handle204NoContent();
826  break;
827 
829  handle206PartialContent();
830  break;
831 
832  default:
833  debugs(93, 5, "ICAP status " << icapReply->sline.status());
834  handleUnknownScode();
835  break;
836  }
837 
838  const HttpRequest *request = dynamic_cast<HttpRequest*>(adapted.header);
839  if (!request)
840  request = &virginRequest();
841 
842  // update the cross-transactional database if needed (all status codes!)
843  if (const char *xxName = Adaptation::Config::masterx_shared_name) {
844  Adaptation::History::Pointer ah = request->adaptHistory(true);
845  if (ah != NULL) { // TODO: reorder checks to avoid creating history
846  const String val = icapReply->header.getByName(xxName);
847  if (val.size() > 0) // XXX: HttpHeader lacks empty value detection
848  ah->updateXxRecord(xxName, val);
849  }
850  }
851 
852  // update the adaptation plan if needed (all status codes!)
853  if (service().cfg().routing) {
854  String services;
855  if (icapReply->header.getList(Http::HdrType::X_NEXT_SERVICES, &services)) {
856  Adaptation::History::Pointer ah = request->adaptHistory(true);
857  if (ah != NULL)
858  ah->updateNextServices(services);
859  }
860  } // TODO: else warn (occasionally!) if we got Http::HdrType::X_NEXT_SERVICES
861 
862  // We need to store received ICAP headers for <icapLastHeader logformat option.
863  // If we already have stored headers from previous ICAP transaction related to this
864  // request, old headers will be replaced with the new one.
865 
866  Adaptation::History::Pointer ah = request->adaptLogHistory();
867  if (ah != NULL)
868  ah->recordMeta(&icapReply->header);
869 
870  // handle100Continue() manages state.writing on its own.
871  // Non-100 status means the server needs no postPreview data from us.
872  if (state.writing == State::writingPaused)
873  stopWriting(true);
874 }
875 
879 
880  if (parsePart(trailerParser, "trailer")) {
881  for (const auto &e: trailerParser->trailer.entries)
882  debugs(93, 5, "ICAP trailer: " << e->name << ": " << e->value);
883  stopParsing();
884  }
885 }
886 
888 {
889  if (service().cfg().method == ICAP::methodRespmod)
890  return gotEncapsulated("res-hdr");
891 
892  return service().cfg().method == ICAP::methodReqmod &&
893  expectHttpHeader();
894 }
895 
897 {
898  Must(state.writing == State::writingPaused);
899  // server must not respond before the end of preview: we may send ieof
900  Must(preview.enabled() && preview.done() && !preview.ieof());
901 
902  // 100 "Continue" cancels our Preview commitment,
903  // but not commitment to handle 204 or 206 outside Preview
904  if (!state.allowedPostview204 && !state.allowedPostview206)
905  stopBackup();
906 
907  state.parsing = State::psIcapHeader; // eventually
908  icapReply->reset();
909 
910  state.writing = State::writingPrime;
911 
912  writeMore();
913 }
914 
916 {
917  state.parsing = State::psHttpHeader;
918  state.sending = State::sendingAdapted;
919  stopBackup();
920  checkConsuming();
921 }
922 
924 {
925  stopParsing();
926  prepEchoing();
927 }
928 
930 {
931  if (state.writing == State::writingPaused) {
932  Must(preview.enabled());
933  Must(state.allowedPreview206);
934  debugs(93, 7, "206 inside preview");
935  } else {
936  Must(state.writing > State::writingPaused);
937  Must(state.allowedPostview206);
938  debugs(93, 7, "206 outside preview");
939  }
940  state.parsing = State::psHttpHeader;
941  state.sending = State::sendingAdapted;
942  state.readyForUob = true;
943  checkConsuming();
944 }
945 
946 // Called when we receive a 204 No Content response and
947 // when we are trying to bypass a service failure.
948 // We actually start sending (echoig or not) in startSending.
950 {
951  disableRepeats("preparing to echo content");
952  disableBypass("preparing to echo content", true);
953  setOutcome(xoEcho);
954 
955  // We want to clone the HTTP message, but we do not want
956  // to copy some non-HTTP state parts that Http::Message kids carry in them.
957  // Thus, we cannot use a smart pointer, copy constructor, or equivalent.
958  // Instead, we simply write the HTTP message and "clone" it by parsing.
959  // TODO: use Http::Message::clone()!
960 
961  Http::Message *oldHead = virgin.header;
962  debugs(93, 7, "cloning virgin message " << oldHead);
963 
964  MemBuf httpBuf;
965 
966  // write the virgin message into a memory buffer
967  httpBuf.init();
968  packHead(httpBuf, oldHead);
969 
970  // allocate the adapted message and copy metainfo
971  Must(!adapted.header);
972  {
973  Http::MessagePointer newHead;
974  if (const HttpRequest *r = dynamic_cast<const HttpRequest*>(oldHead)) {
975  newHead = new HttpRequest(r->masterXaction);
976  } else if (dynamic_cast<const HttpReply*>(oldHead)) {
977  newHead = new HttpReply;
978  }
979  Must(newHead);
980 
981  newHead->inheritProperties(oldHead);
982 
983  adapted.setHeader(newHead.getRaw());
984  }
985 
986  // parse the buffer back
988 
989  httpBuf.terminate(); // Http::Message::parse requires nil-terminated buffer
990  Must(adapted.header->parse(httpBuf.content(), httpBuf.contentSize(), true, &error));
991  Must(adapted.header->hdr_sz == httpBuf.contentSize()); // no leftovers
992 
993  httpBuf.clean();
994 
995  debugs(93, 7, "cloned virgin message " << oldHead << " to " <<
996  adapted.header);
997 
998  // setup adapted body pipe if needed
999  if (oldHead->body_pipe != NULL) {
1000  debugs(93, 7, "will echo virgin body from " <<
1001  oldHead->body_pipe);
1002  if (!virginBodySending.active())
1003  virginBodySending.plan(); // will throw if not possible
1004  state.sending = State::sendingVirgin;
1005  checkConsuming();
1006 
1007  // TODO: optimize: is it possible to just use the oldHead pipe and
1008  // remove ICAP from the loop? This echoing is probably a common case!
1009  makeAdaptedBodyPipe("echoed virgin response");
1010  if (oldHead->body_pipe->bodySizeKnown())
1011  adapted.body_pipe->setBodySize(oldHead->body_pipe->bodySize());
1012  debugs(93, 7, "will echo virgin body to " <<
1013  adapted.body_pipe);
1014  } else {
1015  debugs(93, 7, "no virgin body to echo");
1016  stopSending(true);
1017  }
1018 }
1019 
1023 {
1024  Must(virginBodySending.active());
1025  Must(virgin.header->body_pipe != NULL);
1026 
1027  setOutcome(xoPartEcho);
1028 
1029  debugs(93, 7, "will echo virgin body suffix from " <<
1030  virgin.header->body_pipe << " offset " << pos );
1031 
1032  // check that use-original-body=N does not point beyond buffered data
1033  const uint64_t virginDataEnd = virginConsumed +
1034  virgin.body_pipe->buf().contentSize();
1035  Must(pos <= virginDataEnd);
1036  virginBodySending.progress(static_cast<size_t>(pos));
1037 
1038  state.sending = State::sendingVirgin;
1039  checkConsuming();
1040 
1041  if (virgin.header->body_pipe->bodySizeKnown())
1042  adapted.body_pipe->expectProductionEndAfter(virgin.header->body_pipe->bodySize() - pos);
1043 
1044  debugs(93, 7, "will echo virgin body suffix to " <<
1045  adapted.body_pipe);
1046 
1047  // Start echoing data
1048  echoMore();
1049 }
1050 
1052 {
1053  stopParsing(false);
1054  stopBackup();
1055  // TODO: mark connection as "bad"
1056 
1057  // Terminate the transaction; we do not know how to handle this response.
1058  throw TexcHere("Unsupported ICAP status code");
1059 }
1060 
1062 {
1063  if (expectHttpHeader()) {
1064  replyHttpHeaderSize = 0;
1065  maybeAllocateHttpMsg();
1066 
1067  if (!parseHead(adapted.header))
1068  return; // need more header data
1069 
1070  if (adapted.header)
1071  replyHttpHeaderSize = adapted.header->hdr_sz;
1072 
1073  if (dynamic_cast<HttpRequest*>(adapted.header)) {
1074  const HttpRequest *oldR = dynamic_cast<const HttpRequest*>(virgin.header);
1075  Must(oldR);
1076  // TODO: the adapted request did not really originate from the
1077  // client; give proxy admin an option to prevent copying of
1078  // sensitive client information here. See the following thread:
1079  // http://www.squid-cache.org/mail-archive/squid-dev/200703/0040.html
1080  }
1081 
1082  // Maybe adapted.header==NULL if HttpReply and have Http 0.9 ....
1083  if (adapted.header)
1084  adapted.header->inheritProperties(virgin.header);
1085  }
1086 
1087  decideOnParsingBody();
1088 }
1089 
1090 template<class Part>
1091 bool Adaptation::Icap::ModXact::parsePart(Part *part, const char *description)
1092 {
1093  Must(part);
1094  debugs(93, 5, "have " << readBuf.length() << ' ' << description << " bytes to parse; state: " << state.parsing);
1096  // XXX: performance regression. c_str() data copies
1097  // XXX: Http::Message::parse requires a terminated string buffer
1098  const char *tmpBuf = readBuf.c_str();
1099  const bool parsed = part->parse(tmpBuf, readBuf.length(), commEof, &error);
1100  debugs(93, (!parsed && error) ? 2 : 5, description << " parsing result: " << parsed << " detail: " << error);
1101  Must(parsed || !error);
1102  if (parsed)
1103  readBuf.consume(part->hdr_sz);
1104  return parsed;
1105 }
1106 
1107 // parses both HTTP and ICAP headers
1108 bool
1110 {
1111  if (!parsePart(head, "head")) {
1112  head->reset();
1113  return false;
1114  }
1115  return true;
1116 }
1117 
1119 {
1120  return gotEncapsulated("res-hdr") || gotEncapsulated("req-hdr");
1121 }
1122 
1124 {
1125  return gotEncapsulated("res-body") || gotEncapsulated("req-body");
1126 }
1127 
1129 {
1130  String trailers;
1131  const bool promisesToSendTrailer = icapReply->header.getByIdIfPresent(Http::HdrType::TRAILER, &trailers);
1132  const bool supportsTrailers = icapReply->header.hasListMember(Http::HdrType::ALLOW, "trailers", ',');
1133  // ICAP Trailer specs require us to reject transactions having either Trailer
1134  // header or Allow:trailers
1135  Must((promisesToSendTrailer == supportsTrailers) || (!promisesToSendTrailer && supportsTrailers));
1136  if (promisesToSendTrailer && !trailers.size())
1137  debugs(93, DBG_IMPORTANT, "ERROR: ICAP Trailer response header field must not be empty (salvaged)");
1138  return promisesToSendTrailer;
1139 }
1140 
1142 {
1143  if (expectHttpBody()) {
1144  debugs(93, 5, "expecting a body");
1145  state.parsing = State::psBody;
1146  replyHttpBodySize = 0;
1147  bodyParser = new Http1::TeChunkedParser;
1148  bodyParser->parseExtensionValuesWith(&extensionParser);
1149  makeAdaptedBodyPipe("adapted response from the ICAP server");
1150  Must(state.sending == State::sendingAdapted);
1151  } else {
1152  debugs(93, 5, "not expecting a body");
1153  if (trailerParser)
1154  state.parsing = State::psIcapTrailer;
1155  else
1156  stopParsing();
1157  stopSending(true);
1158  }
1159 }
1160 
1162 {
1163  Must(state.parsing == State::psBody);
1164  Must(bodyParser);
1165 
1166  debugs(93, 5, "have " << readBuf.length() << " body bytes to parse");
1167 
1168  // the parser will throw on errors
1169  BodyPipeCheckout bpc(*adapted.body_pipe);
1170  bodyParser->setPayloadBuffer(&bpc.buf);
1171  const bool parsed = bodyParser->parse(readBuf);
1172  readBuf = bodyParser->remaining(); // sync buffers after parse
1173  bpc.checkIn();
1174 
1175  debugs(93, 5, "have " << readBuf.length() << " body bytes after parsed all: " << parsed);
1176  replyHttpBodySize += adapted.body_pipe->buf().contentSize();
1177 
1178  // TODO: expose BodyPipe::putSize() to make this check simpler and clearer
1179  // TODO: do we really need this if we disable when sending headers?
1180  if (adapted.body_pipe->buf().contentSize() > 0) { // parsed something sometime
1181  disableRepeats("sent adapted content");
1182  disableBypass("sent adapted content", true);
1183  }
1184 
1185  if (parsed) {
1186  if (state.readyForUob && extensionParser.sawUseOriginalBody())
1187  prepPartialBodyEchoing(extensionParser.useOriginalBody());
1188  else
1189  stopSending(true); // the parser succeeds only if all parsed data fits
1190  if (trailerParser)
1191  state.parsing = State::psIcapTrailer;
1192  else
1193  stopParsing();
1194  return;
1195  }
1196 
1197  debugs(93,3, this << " needsMoreData = " << bodyParser->needsMoreData());
1198 
1199  if (bodyParser->needsMoreData()) {
1200  debugs(93,3, this);
1201  Must(mayReadMore());
1202  readMore();
1203  }
1204 
1205  if (bodyParser->needsMoreSpace()) {
1206  Must(!doneSending()); // can hope for more space
1207  Must(adapted.body_pipe->buf().contentSize() > 0); // paranoid
1208  // TODO: there should be a timeout in case the sink is broken
1209  // or cannot consume partial content (while we need more space)
1210  }
1211 }
1212 
1213 void Adaptation::Icap::ModXact::stopParsing(const bool checkUnparsedData)
1214 {
1215  if (state.parsing == State::psDone)
1216  return;
1217 
1218  if (checkUnparsedData)
1219  Must(readBuf.isEmpty());
1220 
1221  debugs(93, 7, "will no longer parse" << status());
1222 
1223  delete bodyParser;
1224  bodyParser = nullptr;
1225 
1226  delete trailerParser;
1227  trailerParser = nullptr;
1228 
1229  state.parsing = State::psDone;
1230 }
1231 
1232 // HTTP side added virgin body data
1234 {
1235  writeMore();
1236 
1237  if (state.sending == State::sendingVirgin)
1238  echoMore();
1239 }
1240 
1241 // HTTP side sent us all virgin info
1243 {
1244  Must(virgin.body_pipe->productionEnded());
1245 
1246  // push writer and sender in case we were waiting for the last-chunk
1247  writeMore();
1248 
1249  if (state.sending == State::sendingVirgin)
1250  echoMore();
1251 }
1252 
1253 // body producer aborted, but the initiator may still want to know
1254 // the answer, even though the HTTP message has been truncated
1256 {
1257  Must(virgin.body_pipe->productionEnded());
1258 
1259  // push writer and sender in case we were waiting for the last-chunk
1260  writeMore();
1261 
1262  if (state.sending == State::sendingVirgin)
1263  echoMore();
1264 }
1265 
1266 // adapted body consumer wants more adapted data and
1267 // possibly freed some buffer space
1269 {
1270  if (state.sending == State::sendingVirgin)
1271  echoMore();
1272  else if (state.sending == State::sendingAdapted)
1273  parseMore();
1274  else
1275  Must(state.sending == State::sendingUndecided);
1276 }
1277 
1278 // adapted body consumer aborted
1280 {
1281  static const auto d = MakeNamedErrorDetail("ICAP_XACT_BODY_CONSUMER_ABORT");
1282  detailError(d);
1283  mustStop("adapted body consumer aborted");
1284 }
1285 
1287 {
1288  delete bodyParser;
1289  delete trailerParser;
1290 }
1291 
1292 // internal cleanup
1294 {
1295  debugs(93, 5, "swan sings" << status());
1296 
1297  stopWriting(false);
1298  stopSending(false);
1299 
1300  if (theInitiator.set()) { // we have not sent the answer to the initiator
1301  static const auto d = MakeNamedErrorDetail("ICAP_XACT_OTHER");
1302  detailError(d);
1303  }
1304 
1305  // update adaptation history if start was called and we reserved a slot
1306  Adaptation::History::Pointer ah = virginRequest().adaptLogHistory();
1307  if (ah != NULL && adaptHistoryId >= 0)
1308  ah->recordXactFinish(adaptHistoryId);
1309 
1311 }
1312 
1314 
1316 {
1317  HttpRequest *adapted_request_ = nullptr;
1318  HttpReply *adapted_reply_ = nullptr;
1319  HttpRequest *virgin_request_ = const_cast<HttpRequest*>(&virginRequest());
1320  if (!(adapted_request_ = dynamic_cast<HttpRequest*>(adapted.header))) {
1321  // if the request was not adapted, use virgin request to simplify
1322  // the code further below
1323  adapted_request_ = virgin_request_;
1324  adapted_reply_ = dynamic_cast<HttpReply*>(adapted.header);
1325  }
1326 
1327  Adaptation::Icap::History::Pointer h = virgin_request_->icapHistory();
1328  Must(h != NULL); // ICAPXaction::maybeLog calls only if there is a log
1329  al.icp.opcode = ICP_INVALID;
1330  al.url = h->log_uri.termedBuf();
1331  const Adaptation::Icap::ServiceRep &s = service();
1332  al.icap.reqMethod = s.cfg().method;
1333 
1334  al.cache.caddr = virgin_request_->client_addr;
1335 
1336  al.request = virgin_request_;
1337  HTTPMSGLOCK(al.request);
1338  al.adapted_request = adapted_request_;
1339  HTTPMSGLOCK(al.adapted_request);
1340 
1341  // XXX: This reply (and other ALE members!) may have been needed earlier.
1342  al.reply = adapted_reply_;
1343 
1344  if (h->rfc931.size())
1345  al.cache.rfc931 = h->rfc931.termedBuf();
1346 
1347 #if USE_OPENSSL
1348  if (h->ssluser.size())
1349  al.cache.ssluser = h->ssluser.termedBuf();
1350 #endif
1351  al.cache.code = h->logType;
1352 
1353  const Http::Message *virgin_msg = dynamic_cast<HttpReply*>(virgin.header);
1354  if (!virgin_msg)
1355  virgin_msg = virgin_request_;
1356  assert(virgin_msg != virgin.cause);
1357  al.http.clientRequestSz.header = virgin_msg->hdr_sz;
1358  if (virgin_msg->body_pipe != NULL)
1359  al.http.clientRequestSz.payloadData = virgin_msg->body_pipe->producedSize();
1360 
1361  // leave al.icap.bodyBytesRead negative if no body
1362  if (replyHttpHeaderSize >= 0 || replyHttpBodySize >= 0) {
1363  const int64_t zero = 0; // to make max() argument types the same
1364  const uint64_t headerSize = max(zero, replyHttpHeaderSize);
1365  const uint64_t bodySize = max(zero, replyHttpBodySize);
1366  al.icap.bodyBytesRead = headerSize + bodySize;
1367  al.http.clientReplySz.header = headerSize;
1368  al.http.clientReplySz.payloadData = bodySize;
1369  }
1370 
1371  if (adapted_reply_) {
1372  al.http.code = adapted_reply_->sline.status();
1373  al.http.content_type = adapted_reply_->content_type.termedBuf();
1374  if (replyHttpBodySize >= 0)
1375  al.cache.highOffset = replyHttpBodySize;
1376  //don't set al.cache.objectSize because it hasn't exist yet
1377  }
1378  prepareLogWithRequestDetails(adapted_request_, alep);
1380 }
1381 
1383 {
1384  char ntoabuf[MAX_IPSTRLEN];
1385  /*
1386  * XXX These should use HttpHdr interfaces instead of Printfs
1387  */
1388  const Adaptation::ServiceConfig &s = service().cfg();
1389  buf.appendf("%s " SQUIDSTRINGPH " ICAP/1.0\r\n", s.methodStr(), SQUIDSTRINGPRINT(s.uri));
1390  buf.appendf("Host: " SQUIDSTRINGPH ":%d\r\n", SQUIDSTRINGPRINT(s.host), s.port);
1391  buf.appendf("Date: %s\r\n", Time::FormatRfc1123(squid_curtime));
1392 
1394  buf.appendf("Connection: close\r\n");
1395 
1396  const HttpRequest *request = &virginRequest();
1397 
1398  // we must forward "Proxy-Authenticate" and "Proxy-Authorization"
1399  // as ICAP headers.
1400  if (virgin.header->header.has(Http::HdrType::PROXY_AUTHENTICATE)) {
1401  String vh=virgin.header->header.getById(Http::HdrType::PROXY_AUTHENTICATE);
1402  buf.appendf("Proxy-Authenticate: " SQUIDSTRINGPH "\r\n",SQUIDSTRINGPRINT(vh));
1403  }
1404 
1405  if (virgin.header->header.has(Http::HdrType::PROXY_AUTHORIZATION)) {
1406  String vh=virgin.header->header.getById(Http::HdrType::PROXY_AUTHORIZATION);
1407  buf.appendf("Proxy-Authorization: " SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(vh));
1408  } else if (request->extacl_user.size() > 0 && request->extacl_passwd.size() > 0) {
1409  struct base64_encode_ctx ctx;
1410  base64_encode_init(&ctx);
1411  char base64buf[base64_encode_len(MAX_LOGIN_SZ)];
1412  size_t resultLen = base64_encode_update(&ctx, base64buf, request->extacl_user.size(), reinterpret_cast<const uint8_t*>(request->extacl_user.rawBuf()));
1413  resultLen += base64_encode_update(&ctx, base64buf+resultLen, 1, reinterpret_cast<const uint8_t*>(":"));
1414  resultLen += base64_encode_update(&ctx, base64buf+resultLen, request->extacl_passwd.size(), reinterpret_cast<const uint8_t*>(request->extacl_passwd.rawBuf()));
1415  resultLen += base64_encode_final(&ctx, base64buf+resultLen);
1416  buf.appendf("Proxy-Authorization: Basic %.*s\r\n", (int)resultLen, base64buf);
1417  }
1418 
1419  // share the cross-transactional database records if needed
1421  Adaptation::History::Pointer ah = request->adaptHistory(false);
1422  if (ah != NULL) {
1423  String name, value;
1424  if (ah->getXxRecord(name, value)) {
1425  buf.appendf(SQUIDSTRINGPH ": " SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(name), SQUIDSTRINGPRINT(value));
1426  }
1427  }
1428  }
1429 
1430  buf.append("Encapsulated: ", 14);
1431 
1432  MemBuf httpBuf;
1433 
1434  httpBuf.init();
1435 
1436  // build HTTP request header, if any
1437  ICAP::Method m = s.method;
1438 
1439  // to simplify, we could assume that request is always available
1440 
1441  if (request) {
1442  if (ICAP::methodRespmod == m)
1443  encapsulateHead(buf, "req-hdr", httpBuf, request);
1444  else if (ICAP::methodReqmod == m)
1445  encapsulateHead(buf, "req-hdr", httpBuf, virgin.header);
1446  }
1447 
1448  if (ICAP::methodRespmod == m)
1449  if (const Http::Message *prime = virgin.header)
1450  encapsulateHead(buf, "res-hdr", httpBuf, prime);
1451 
1452  if (!virginBody.expected())
1453  buf.appendf("null-body=%d", (int) httpBuf.contentSize());
1454  else if (ICAP::methodReqmod == m)
1455  buf.appendf("req-body=%d", (int) httpBuf.contentSize());
1456  else
1457  buf.appendf("res-body=%d", (int) httpBuf.contentSize());
1458 
1459  buf.append(ICAP::crlf, 2); // terminate Encapsulated line
1460 
1461  if (preview.enabled()) {
1462  buf.appendf("Preview: %d\r\n", (int)preview.ad());
1463  if (!virginBody.expected()) // there is no body to preview
1464  finishNullOrEmptyBodyPreview(httpBuf);
1465  }
1466 
1467  makeAllowHeader(buf);
1468 
1469  if (TheConfig.send_client_ip && request) {
1470  Ip::Address client_addr;
1471 #if FOLLOW_X_FORWARDED_FOR
1473  client_addr = request->indirect_client_addr;
1474  } else
1475 #endif
1476  client_addr = request->client_addr;
1477  if (!client_addr.isAnyAddr() && !client_addr.isNoAddr())
1478  buf.appendf("X-Client-IP: %s\r\n", client_addr.toStr(ntoabuf,MAX_IPSTRLEN));
1479  }
1480 
1482  makeUsernameHeader(request, buf);
1483 
1484  // Adaptation::Config::metaHeaders
1485  for (auto h: Adaptation::Config::metaHeaders) {
1486  HttpRequest *r = virgin.cause ?
1487  virgin.cause : dynamic_cast<HttpRequest*>(virgin.header);
1488  Must(r);
1489 
1490  HttpReply *reply = dynamic_cast<HttpReply*>(virgin.header);
1491 
1492  SBuf matched;
1493  if (h->match(r, reply, alMaster, matched)) {
1494  buf.append(h->key().rawContent(), h->key().length());
1495  buf.append(": ", 2);
1496  buf.append(matched.rawContent(), matched.length());
1497  buf.append("\r\n", 2);
1498  Adaptation::History::Pointer ah = request->adaptHistory(false);
1499  if (ah != NULL) {
1500  if (ah->metaHeaders == NULL)
1501  ah->metaHeaders = new NotePairs;
1502  if (!ah->metaHeaders->hasPair(h->key(), matched))
1503  ah->metaHeaders->add(h->key(), matched);
1504  }
1505  }
1506  }
1507 
1508  // fprintf(stderr, "%s\n", buf.content());
1509 
1510  buf.append(ICAP::crlf, 2); // terminate ICAP header
1511 
1512  // fill icapRequest for logging
1513  Must(icapRequest->parseCharBuf(buf.content(), buf.contentSize()));
1514 
1515  // start ICAP request body with encapsulated HTTP headers
1516  buf.append(httpBuf.content(), httpBuf.contentSize());
1517 
1518  httpBuf.clean();
1519 }
1520 
1521 // decides which Allow values to write and updates the request buffer
1523 {
1524  const bool allow204in = preview.enabled(); // TODO: add shouldAllow204in()
1525  const bool allow204out = state.allowedPostview204 = shouldAllow204();
1526  const bool allow206in = state.allowedPreview206 = shouldAllow206in();
1527  const bool allow206out = state.allowedPostview206 = shouldAllow206out();
1528  const bool allowTrailers = true; // TODO: make configurable
1529 
1530  debugs(93, 9, "Allows: " << allow204in << allow204out <<
1531  allow206in << allow206out << allowTrailers);
1532 
1533  const bool allow204 = allow204in || allow204out;
1534  const bool allow206 = allow206in || allow206out;
1535 
1536  if ((allow204 || allow206) && virginBody.expected())
1537  virginBodySending.plan(); // if there is a virgin body, plan to send it
1538 
1539  // writing Preview:... means we will honor 204 inside preview
1540  // writing Allow/204 means we will honor 204 outside preview
1541  // writing Allow:206 means we will honor 206 inside preview
1542  // writing Allow:204,206 means we will honor 206 outside preview
1543  if (allow204 || allow206 || allowTrailers) {
1544  buf.appendf("Allow: ");
1545  if (allow204out)
1546  buf.appendf("204, ");
1547  if (allow206)
1548  buf.appendf("206, ");
1549  if (allowTrailers)
1550  buf.appendf("trailers");
1551  buf.appendf("\r\n");
1552  }
1553 }
1554 
1556 {
1557 #if USE_AUTH
1558  struct base64_encode_ctx ctx;
1559  base64_encode_init(&ctx);
1560 
1561  const char *value = NULL;
1562  if (request->auth_user_request != NULL) {
1563  value = request->auth_user_request->username();
1564  } else if (request->extacl_user.size() > 0) {
1565  value = request->extacl_user.termedBuf();
1566  }
1567 
1568  if (value) {
1570  char base64buf[base64_encode_len(MAX_LOGIN_SZ)];
1571  size_t resultLen = base64_encode_update(&ctx, base64buf, strlen(value), reinterpret_cast<const uint8_t*>(value));
1572  resultLen += base64_encode_final(&ctx, base64buf+resultLen);
1573  buf.appendf("%s: %.*s\r\n", TheConfig.client_username_header, (int)resultLen, base64buf);
1574  } else
1575  buf.appendf("%s: %s\r\n", TheConfig.client_username_header, value);
1576  }
1577 #else
1578  (void)request;
1579  (void)buf;
1580 #endif
1581 }
1582 
1583 void
1584 Adaptation::Icap::ModXact::encapsulateHead(MemBuf &icapBuf, const char *section, MemBuf &httpBuf, const Http::Message *head)
1585 {
1586  // update ICAP header
1587  icapBuf.appendf("%s=%d, ", section, (int) httpBuf.contentSize());
1588 
1589  // begin cloning
1590  Http::MessagePointer headClone;
1591 
1592  if (const HttpRequest* old_request = dynamic_cast<const HttpRequest*>(head)) {
1593  HttpRequest::Pointer new_request(new HttpRequest(old_request->masterXaction));
1594  // copy the request-line details
1595  new_request->method = old_request->method;
1596  new_request->url = old_request->url;
1597  new_request->http_ver = old_request->http_ver;
1598  headClone = new_request.getRaw();
1599  } else if (const HttpReply *old_reply = dynamic_cast<const HttpReply*>(head)) {
1600  HttpReply::Pointer new_reply(new HttpReply);
1601  new_reply->sline = old_reply->sline;
1602  headClone = new_reply.getRaw();
1603  }
1604  Must(headClone);
1605  headClone->inheritProperties(head);
1606 
1608  while (HttpHeaderEntry* p_head_entry = head->header.getEntry(&pos))
1609  headClone->header.addEntry(p_head_entry->clone());
1610 
1611  // end cloning
1612 
1613  // remove all hop-by-hop headers from the clone
1615  headClone->header.removeHopByHopEntries();
1616 
1617  // TODO: modify HttpHeader::removeHopByHopEntries to accept a list of
1618  // excluded hop-by-hop headers
1619  if (head->header.has(Http::HdrType::UPGRADE)) {
1620  const auto upgrade = head->header.getList(Http::HdrType::UPGRADE);
1621  headClone->header.putStr(Http::HdrType::UPGRADE, upgrade.termedBuf());
1622  }
1623 
1624  // pack polished HTTP header
1625  packHead(httpBuf, headClone.getRaw());
1626 
1627  // headClone unlocks and, hence, deletes the message we packed
1628 }
1629 
1630 void
1632 {
1633  head->packInto(&httpBuf, true);
1634 }
1635 
1636 // decides whether to offer a preview and calculates its size
1638 {
1639  if (!TheConfig.preview_enable) {
1640  debugs(93, 5, "preview disabled by squid.conf");
1641  return;
1642  }
1643 
1644  const SBuf urlPath(virginRequest().url.path());
1645  size_t wantedSize;
1646  if (!service().wantsPreview(urlPath, wantedSize)) {
1647  debugs(93, 5, "should not offer preview for " << urlPath);
1648  return;
1649  }
1650 
1651  // we decided to do preview, now compute its size
1652 
1653  // cannot preview more than we can backup
1654  size_t ad = min(wantedSize, TheBackupLimit);
1655 
1656  if (!virginBody.expected())
1657  ad = 0;
1658  else if (virginBody.knownSize())
1659  ad = min(static_cast<uint64_t>(ad), virginBody.size()); // not more than we have
1660 
1661  debugs(93, 5, "should offer " << ad << "-byte preview " <<
1662  "(service wanted " << wantedSize << ")");
1663 
1664  preview.enable(ad);
1665  Must(preview.enabled());
1666 }
1667 
1668 // decides whether to allow 204 responses
1670 {
1671  if (!service().allows204())
1672  return false;
1673 
1674  return canBackupEverything();
1675 }
1676 
1677 // decides whether to allow 206 responses in some mode
1679 {
1680  return TheConfig.allow206_enable && service().allows206() &&
1681  virginBody.expected(); // no need for 206 without a body
1682 }
1683 
1684 // decides whether to allow 206 responses in preview mode
1686 {
1687  return shouldAllow206any() && preview.enabled();
1688 }
1689 
1690 // decides whether to allow 206 responses outside of preview
1692 {
1693  return shouldAllow206any() && canBackupEverything();
1694 }
1695 
1696 // used by shouldAllow204 and decideOnRetries
1698 {
1699  if (!virginBody.expected())
1700  return true; // no body means no problems with backup
1701 
1702  // if there is a body, check whether we can backup it all
1703 
1704  if (!virginBody.knownSize())
1705  return false;
1706 
1707  // or should we have a different backup limit?
1708  // note that '<' allows for 0-termination of the "full" backup buffer
1709  return virginBody.size() < TheBackupLimit;
1710 }
1711 
1712 // Decide whether this transaction can be retried if pconn fails
1713 // Must be called after decideOnPreview and before openConnection()
1715 {
1716  if (!isRetriable)
1717  return; // no, already decided
1718 
1719  if (preview.enabled())
1720  return; // yes, because preview provides enough guarantees
1721 
1722  if (canBackupEverything())
1723  return; // yes, because we can back everything up
1724 
1725  disableRetries(); // no, because we cannot back everything up
1726 }
1727 
1728 // Normally, the body-writing code handles preview body. It can deal with
1729 // bodies of unexpected size, including those that turn out to be empty.
1730 // However, that code assumes that the body was expected and body control
1731 // structures were initialized. This is not the case when there is no body
1732 // or the body is known to be empty, because the virgin message will lack a
1733 // body_pipe. So we handle preview of null-body and zero-size bodies here.
1735 {
1736  Must(!virginBodyWriting.active()); // one reason we handle it here
1737  Must(!virgin.body_pipe); // another reason we handle it here
1738  Must(!preview.ad());
1739 
1740  // do not add last-chunk because our Encapsulated header says null-body
1741  // addLastRequestChunk(buf);
1742  preview.wrote(0, true);
1743 
1744  Must(preview.done());
1745  Must(preview.ieof());
1746 }
1747 
1749 {
1751 
1752  if (state.serviceWaiting)
1753  buf.append("U", 1);
1754 
1755  if (virgin.body_pipe != NULL)
1756  buf.append("R", 1);
1757 
1758  if (haveConnection() && !doneReading())
1759  buf.append("r", 1);
1760 
1761  if (!state.doneWriting() && state.writing != State::writingInit)
1762  buf.appendf("w(%d)", state.writing);
1763 
1764  if (preview.enabled()) {
1765  if (!preview.done())
1766  buf.appendf("P(%d)", (int) preview.debt());
1767  }
1768 
1769  if (virginBodySending.active())
1770  buf.append("B", 1);
1771 
1772  if (!state.doneParsing() && state.parsing != State::psIcapHeader)
1773  buf.appendf("p(%d)", state.parsing);
1774 
1775  if (!doneSending() && state.sending != State::sendingUndecided)
1776  buf.appendf("S(%d)", state.sending);
1777 
1778  if (state.readyForUob)
1779  buf.append("6", 1);
1780 
1781  if (canStartBypass)
1782  buf.append("Y", 1);
1783 
1784  if (protectGroupBypass)
1785  buf.append("G", 1);
1786 }
1787 
1789 {
1791 
1792  if (!virgin.body_pipe)
1793  buf.append("R", 1);
1794 
1795  if (state.doneWriting())
1796  buf.append("w", 1);
1797 
1798  if (preview.enabled()) {
1799  if (preview.done())
1800  buf.appendf("P%s", preview.ieof() ? "(ieof)" : "");
1801  }
1802 
1803  if (doneReading())
1804  buf.append("r", 1);
1805 
1806  if (state.doneParsing())
1807  buf.append("p", 1);
1808 
1809  if (doneSending())
1810  buf.append("S", 1);
1811 }
1812 
1813 bool Adaptation::Icap::ModXact::gotEncapsulated(const char *section) const
1814 {
1815  return !icapReply->header.getByNameListMember("Encapsulated",
1816  section, ',').isEmpty();
1817 }
1818 
1819 // calculate whether there is a virgin HTTP body and
1820 // whether its expected size is known
1821 // TODO: rename because we do not just estimate
1823 {
1824  // note: lack of size info may disable previews and 204s
1825 
1826  Http::Message *msg = virgin.header;
1827  Must(msg);
1828 
1829  HttpRequestMethod method;
1830 
1831  if (virgin.cause)
1832  method = virgin.cause->method;
1833  else if (HttpRequest *req = dynamic_cast<HttpRequest*>(msg))
1834  method = req->method;
1835  else
1836  method = Http::METHOD_NONE;
1837 
1838  int64_t size;
1839  // expectingBody returns true for zero-sized bodies, but we will not
1840  // get a pipe for that body, so we treat the message as bodyless
1841  if (method != Http::METHOD_NONE && msg->expectingBody(method, size) && size) {
1842  debugs(93, 6, "expects virgin body from " <<
1843  virgin.body_pipe << "; size: " << size);
1844 
1845  virginBody.expect(size);
1846  virginBodyWriting.plan();
1847 
1848  // sign up as a body consumer
1849  Must(msg->body_pipe != NULL);
1850  Must(msg->body_pipe == virgin.body_pipe);
1851  Must(virgin.body_pipe->setConsumerIfNotLate(this));
1852 
1853  // make sure TheBackupLimit is in-sync with the buffer size
1854  Must(TheBackupLimit <= static_cast<size_t>(msg->body_pipe->buf().max_capacity));
1855  } else {
1856  debugs(93, 6, "does not expect virgin body");
1857  Must(msg->body_pipe == NULL);
1858  checkConsuming();
1859  }
1860 }
1861 
1863 {
1864  Must(!adapted.body_pipe);
1865  Must(!adapted.header->body_pipe);
1866  adapted.header->body_pipe = new BodyPipe(this);
1867  adapted.body_pipe = adapted.header->body_pipe;
1868  debugs(93, 7, "will supply " << what << " via " <<
1869  adapted.body_pipe << " pipe");
1870 }
1871 
1872 // TODO: Move SizedEstimate and Preview elsewhere
1873 
1875  : theData(dtUnexpected)
1876 {}
1877 
1879 {
1880  theData = (aSize >= 0) ? aSize : (int64_t)dtUnknown;
1881 }
1882 
1884 {
1885  return theData != dtUnexpected;
1886 }
1887 
1889 {
1890  Must(expected());
1891  return theData != dtUnknown;
1892 }
1893 
1895 {
1896  Must(knownSize());
1897  return static_cast<uint64_t>(theData);
1898 }
1899 
1900 Adaptation::Icap::VirginBodyAct::VirginBodyAct(): theStart(0), theState(stUndecided)
1901 {}
1902 
1904 {
1905  Must(!disabled());
1906  Must(!theStart); // not started
1907  theState = stActive;
1908 }
1909 
1911 {
1912  theState = stDisabled;
1913 }
1914 
1916 {
1917  Must(active());
1918 #if SIZEOF_SIZE_T > 4
1919  /* always true for smaller size_t's */
1920  Must(static_cast<int64_t>(size) >= 0);
1921 #endif
1922  theStart += static_cast<int64_t>(size);
1923 }
1924 
1926 {
1927  Must(active());
1928  return static_cast<uint64_t>(theStart);
1929 }
1930 
1931 Adaptation::Icap::Preview::Preview(): theWritten(0), theAd(0), theState(stDisabled)
1932 {}
1933 
1935 {
1936  // TODO: check for anAd not exceeding preview size limit
1937  Must(!enabled());
1938  theAd = anAd;
1939  theState = stWriting;
1940 }
1941 
1943 {
1944  return theState != stDisabled;
1945 }
1946 
1948 {
1949  Must(enabled());
1950  return theAd;
1951 }
1952 
1954 {
1955  Must(enabled());
1956  return theState >= stIeof;
1957 }
1958 
1960 {
1961  Must(enabled());
1962  return theState == stIeof;
1963 }
1964 
1966 {
1967  Must(enabled());
1968  return done() ? 0 : (theAd - theWritten);
1969 }
1970 
1971 void Adaptation::Icap::Preview::wrote(size_t size, bool wroteEof)
1972 {
1973  Must(enabled());
1974 
1975  theWritten += size;
1976 
1977  Must(theWritten <= theAd);
1978 
1979  if (wroteEof)
1980  theState = stIeof; // written size is irrelevant
1981  else if (theWritten >= theAd)
1982  theState = stDone;
1983 }
1984 
1986 {
1987  if (virgin.header == NULL)
1988  return false;
1989 
1990  virgin.header->firstLineBuf(mb);
1991 
1992  return true;
1993 }
1994 
1996 {
1997  HttpRequest *request = dynamic_cast<HttpRequest*>(adapted.header);
1998  // if no adapted request, update virgin (and inherit its properties later)
1999  // TODO: make this and HttpRequest::detailError constant, like adaptHistory
2000  if (!request)
2001  request = const_cast<HttpRequest*>(&virginRequest());
2002 
2003  if (request)
2004  request->detailError(ERR_ICAP_FAILURE, errDetail);
2005 }
2006 
2008 {
2009  HttpRequest *request = dynamic_cast<HttpRequest*>(adapted.header);
2010  // if no adapted request, update virgin (and inherit its properties later)
2011  if (!request)
2012  request = const_cast<HttpRequest*>(&virginRequest());
2013 
2014  if (request)
2015  request->clearError();
2016 }
2017 
2019 {
2020  Must(adapted.header);
2021  adapted.header->sources |= (service().cfg().connectionEncryption ? Http::Message::srcIcaps : Http::Message::srcIcap);
2022 }
2023 
2024 /* Adaptation::Icap::ModXactLauncher */
2025 
2027  AsyncJob("Adaptation::Icap::ModXactLauncher"),
2028  Adaptation::Icap::Launcher("Adaptation::Icap::ModXactLauncher", aService),
2029  al(alp)
2030 {
2031  virgin.setHeader(virginHeader);
2032  virgin.setCause(virginCause);
2033  updateHistory(true);
2034 }
2035 
2037 {
2039  dynamic_cast<Adaptation::Icap::ServiceRep*>(theService.getRaw());
2040  Must(s != NULL);
2041  return new Adaptation::Icap::ModXact(virgin.header, virgin.cause, al, s);
2042 }
2043 
2045 {
2046  debugs(93, 5, "swan sings");
2047  updateHistory(false);
2049 }
2050 
2052 {
2053  HttpRequest *r = virgin.cause ?
2054  virgin.cause : dynamic_cast<HttpRequest*>(virgin.header);
2055 
2056  // r should never be NULL but we play safe; TODO: add Should()
2057  if (r) {
2059  if (h != NULL) {
2060  if (doStart)
2061  h->start("ICAPModXactLauncher");
2062  else
2063  h->stop("ICAPModXactLauncher");
2064  }
2065  }
2066 }
2067 
2068 bool Adaptation::Icap::TrailerParser::parse(const char *buf, int len, int atEnd, Http::StatusCode *error) {
2070  // RFC 7230 section 4.1.2: MUST NOT generate a trailer that contains
2071  // a field necessary for message framing (e.g., Transfer-Encoding and Content-Length)
2072  clen.applyTrailerRules();
2073  const int parsed = trailer.parse(buf, len, atEnd, hdr_sz, clen);
2074  if (parsed < 0)
2075  *error = Http::scInvalidHeader; // TODO: should we add a new Http::scInvalidTrailer?
2076  return parsed > 0;
2077 }
2078 
2079 void
2081 {
2082  if (extName == UseOriginalBodyName) {
2083  useOriginalBody_ = tok.udec64("use-original-body");
2084  assert(useOriginalBody_ >= 0);
2085  } else {
2086  Ignore(tok, extName);
2087  }
2088 }
2089 
void disableBypass(const char *reason, bool includeGroupBypass)
Definition: ModXact.cc:717
const MemBuf & buf() const
Definition: BodyPipe.h:137
int hdr_sz
Definition: Message.h:82
void stop(const char *context)
note the end of an ICAP processing interval
Definition: History.cc:32
void prepPartialBodyEchoing(uint64_t pos)
Definition: ModXact.cc:1022
String content_type
Definition: HttpReply.h:46
SourceLocationId id() const
same-location exceptions have the same ID
Definition: TextException.h:40
#define Here()
source code location of the caller
Definition: Here.h:15
AccessLogEntry::Pointer alMaster
Master transaction AccessLogEntry.
Definition: ModXact.h:373
ModXactLauncher(Http::Message *virginHeader, HttpRequest *virginCause, AccessLogEntry::Pointer &alp, Adaptation::ServicePointer s)
Definition: ModXact.cc:2026
ModXact(Http::Message *virginHeader, HttpRequest *virginCause, AccessLogEntry::Pointer &alp, ServiceRep::Pointer &s)
Definition: ModXact.cc:54
void terminate()
Definition: MemBuf.cc:241
AnyP::ProtocolVersion http_ver
Definition: Message.h:73
#define base64_encode_len(length)
Definition: base64.h:169
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
@ srvBlock
Definition: Elements.h:19
void openChunk(MemBuf &buf, size_t chunkSize, bool ieof)
Definition: ModXact.cc:376
void applyTrailerRules()
prohibits Content-Length in GET/HEAD requests
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
common parts of HttpRequest and HttpReply
Definition: Message.h:26
void progress(size_t size)
Definition: ModXact.cc:1915
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:98
virtual void fillDoneStatus(MemBuf &buf) const
Definition: ModXact.cc:1788
void enable(size_t anAd)
Definition: ModXact.cc:1934
size_t virginContentSize(const VirginBodyAct &act) const
Definition: ModXact.cc:404
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const =0
void makeRequestHeaders(MemBuf &buf)
Definition: ModXact.cc:1382
virtual void fillDoneStatus(MemBuf &buf) const
Definition: Xaction.cc:664
Config TheConfig
Definition: Config.cc:19
const XactOutcome xoPartEcho
preserved virgin msg part (ICAP 206)
Definition: Elements.cc:24
void packHead(MemBuf &httpBuf, const Http::Message *head)
Definition: ModXact.cc:1631
@ scNone
Definition: StatusCode.h:21
void removeHopByHopEntries()
Definition: HttpHeader.cc:1739
HttpHeader header
Definition: Message.h:75
virtual void callException(const std::exception &e)
called when the job throws during an async call
Definition: ModXact.cc:665
Http1::TeChunkedParser * bodyParser
Definition: ModXact.h:303
virtual void noteMoreBodyDataAvailable(BodyPipe::Pointer)
Definition: ModXact.cc:1233
bool virginBodyEndReached(const VirginBodyAct &act) const
Definition: ModXact.cc:395
Parses and stores ICAP trailer header block.
Definition: ModXact.h:111
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
static int send_client_ip
Definition: Config.h:47
@ srcIcaps
Secure ICAP service.
Definition: Message.h:35
char * client_username_header
Definition: Config.h:36
void updateXxRecord(const char *name, const String &value)
sets or resets a cross-transactional database record
Definition: History.cc:105
size_t base64_encode_final(struct base64_encode_ctx *ctx, char *dst)
Definition: base64.c:308
bool isAnyAddr() const
Definition: Address.cc:170
void updateNextServices(const String &services)
sets or resets next services for the Adaptation::Iterator to notice
Definition: History.cc:121
CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap, ModXact)
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
void error(char *format,...)
bool hasContent() const
Definition: MemBuf.h:54
#define SQUIDSTRINGPRINT(s)
Definition: SquidString.h:22
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:93
Definition: SBuf.h:87
virtual bool doneAll() const
whether positive goal has been reached
Definition: Xaction.cc:386
void setHeader(Header *h)
Definition: InOut.h:48
virtual void fillPendingStatus(MemBuf &buf) const
Definition: Xaction.cc:646
void stopParsing(const bool checkUnparsedData=true)
Definition: ModXact.cc:1213
const XactOutcome xoEcho
preserved virgin message (ICAP 204)
Definition: Elements.cc:23
static int send_username
Definition: Config.h:48
void closeChunk(MemBuf &buf)
Definition: ModXact.cc:381
static int use_indirect_client
Definition: Config.h:49
C * getRaw() const
Definition: RefCount.h:80
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:209
String log_uri
the request uri
Definition: History.h:45
TrailerParser * trailerParser
Definition: ModXact.h:321
String rfc931
the username from ident
Definition: History.h:39
Http::StatusLine sline
Definition: HttpReply.h:56
virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer)
Definition: ModXact.cc:1268
void makeAllowHeader(MemBuf &buf)
Definition: ModXact.cc:1522
bool getXxRecord(String &name, String &value) const
returns true and fills the record fields iff there is a db record
Definition: History.cc:111
virtual void noteBodyProducerAborted(BodyPipe::Pointer)
Definition: ModXact.cc:1255
bool parsePart(Part *part, const char *description)
Definition: ModXact.cc:1091
StatusCode
Definition: StatusCode.h:20
#define SQUIDSTRINGPH
Definition: SquidString.h:21
@ MaxCapacity
Definition: BodyPipe.h:100
virtual bool doneAll() const
whether positive goal has been reached
Definition: ModXact.cc:528
virtual void noteBodyConsumerAborted(BodyPipe::Pointer)
Definition: ModXact.cc:1279
int const char size_t
Definition: stub_liblog.cc:86
A const & max(A const &lhs, A const &rhs)
void addLastRequestChunk(MemBuf &buf)
Definition: ModXact.cc:369
char * toStr(char *buf, const unsigned int blen, int force=AF_UNSPEC) const
Definition: Address.cc:792
void encapsulateHead(MemBuf &icapBuf, const char *section, MemBuf &httpBuf, const Http::Message *head)
Definition: ModXact.cc:1584
LogTags logType
the squid request status (TCP_MISS etc)
Definition: History.h:43
#define TexcHere(msg)
legacy convenience macro; it is not difficult to type Here() now
Definition: TextException.h:59
virtual bool inheritProperties(const Http::Message *)=0
void consume(size_t size)
Definition: BodyPipe.cc:309
void stopSending(bool nicely)
Definition: ModXact.cc:614
mb_size_t max_capacity
Definition: MemBuf.h:142
#define MAX_IPSTRLEN
Length of buffer that needs to be allocated to old a null-terminated IP-string.
Definition: forward.h:25
void recordXactFinish(int hid)
record the end of a xact identified by its history ID
Definition: History.cc:61
bool bodySizeKnown() const
Definition: BodyPipe.h:109
MemBuf & buf
Definition: BodyPipe.h:74
const char * FormatRfc1123(time_t)
Definition: rfc1123.cc:202
void wrote(size_t size, bool wroteEof)
Definition: ModXact.cc:1971
bool doneSending() const
Definition: ModXact.cc:608
@ ERR_ICAP_FAILURE
Definition: forward.h:64
void makeAdaptedBodyPipe(const char *what)
Definition: ModXact.cc:1862
@ ICP_INVALID
Definition: icp_opcode.h:15
bool httpHeaderHasConnDir(const HttpHeader *hdr, const SBuf &directive)
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
uint64_t bodySize() const
Definition: BodyPipe.cc:161
virtual void start()
called by AsyncStart; do not call directly
Definition: ModXact.cc:88
int size
Definition: ModDevPoll.cc:75
struct timeval current_time
the current UNIX time in timeval {seconds, microseconds} format
Definition: gadgets.cc:17
void parseExtensionValuesWith(ChunkExtensionValueParser *parser)
#define NULL
Definition: types.h:166
const char * rawContent() const
Definition: SBuf.cc:509
virtual void swanSong()
Definition: Launcher.cc:105
void stopWriting(bool nicely)
Definition: ModXact.cc:486
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
virtual void parse(Tokenizer &tok, const SBuf &extName) override
Definition: ModXact.cc:2080
#define MAX_LOGIN_SZ
Definition: defines.h:79
uint64_t producedSize() const
Definition: BodyPipe.h:112
#define true
Definition: GnuRegex.c:234
bool expectHttpHeader() const
whether ICAP response header indicates HTTP header presence
Definition: ModXact.cc:1118
Definition: MemBuf.h:24
bool expectHttpBody() const
whether ICAP response header indicates HTTP body presence
Definition: ModXact.cc:1123
void clean()
Definition: MemBuf.cc:110
@ scPartialContent
Definition: StatusCode.h:32
int delById(Http::HdrType id)
Definition: HttpHeader.cc:695
@ srcIcap
traditional ICAP service without encryption
Definition: Message.h:41
Adaptation::Icap::History::Pointer icapHistory() const
Returns possibly nil history, creating it if icap logging is enabled.
Definition: HttpRequest.cc:390
@ scCreated
Definition: StatusCode.h:27
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:765
void updateSources()
Update the Http::Message sources.
Definition: ModXact.cc:2018
#define assert(EX)
Definition: assert.h:19
char const * termedBuf() const
Definition: SquidString.h:92
void writeSomeBody(const char *label, size_t size)
Definition: ModXact.cc:319
virtual Xaction * createXaction()
Definition: ModXact.cc:2036
@ scContinue
Definition: StatusCode.h:22
virtual void fillPendingStatus(MemBuf &buf) const
Definition: ModXact.cc:1748
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:161
virtual void finalizeLogInfo()
Definition: ModXact.cc:1315
NotePairs::Pointer metaHeaders
Definition: History.h:66
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
void prepareLogWithRequestDetails(HttpRequest *, const AccessLogEntryPointer &)
Definition: client_side.cc:323
void const char int sz
Definition: stub_cbdata.cc:16
const char * virginContentData(const VirginBodyAct &act) const
Definition: ModXact.cc:416
void base64_encode_init(struct base64_encode_ctx *ctx)
Definition: base64.c:232
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:408
virtual bool fillVirginHttpHeader(MemBuf &) const
Definition: ModXact.cc:1985
time_t squid_curtime
Definition: stub_libtime.cc:20
virtual void callException(const std::exception &e)
called when the job throws during an async call
Definition: Xaction.cc:370
static Notes metaHeaders
The list of configured meta headers.
Definition: Config.h:58
bool isNoAddr() const
Definition: Address.cc:284
void setCause(HttpRequest *r)
Definition: InOut.h:38
virtual void handleCommRead(size_t size)
Definition: ModXact.cc:567
bool canBackupEverything() const
Definition: ModXact.cc:1697
bool expectIcapTrailers() const
whether ICAP response header indicates ICAP trailers presence
Definition: ModXact.cc:1128
static Answer Forward(Http::Message *aMsg)
create an akForward answer
Definition: Answer.cc:26
virtual void finalizeLogInfo()
Definition: Xaction.cc:609
const HttpRequest & virginRequest() const
locates the request, either as a cause or as a virgin message itself
Definition: ModXact.cc:386
HttpRequestMethod method
Definition: HttpRequest.h:114
void start(const char *context)
record the start of an ICAP processing interval
Definition: History.cc:23
@ methodRespmod
Definition: Elements.h:17
String protoPrefix
Definition: HttpReply.h:60
const char * status() const
Definition: BodyPipe.cc:446
static const size_t TheBackupLimit
Definition: ModXact.cc:45
void decideWritingAfterPreview(const char *previewKind)
determine state.writing after we wrote the entire preview
Definition: ModXact.cc:292
Definition: parse.c:160
bool hasPair(const SBuf &key, const SBuf &value) const
Definition: Notes.cc:365
squidaio_request_t * head
Definition: aiops.cc:126
virtual void handleCommWrote(size_t size)
Definition: ModXact.cc:208
@ scNoContent
Definition: StatusCode.h:30
@ PROXY_AUTHENTICATE
virtual void startShoveling()
starts sending/receiving ICAP messages
Definition: ModXact.cc:189
an std::runtime_error with thrower location info
Definition: TextException.h:21
char * content()
start of the added data
Definition: MemBuf.h:41
mb_size_t spaceSize() const
Definition: MemBuf.cc:155
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1052
size_type size() const
Definition: SquidString.h:73
@ METHOD_NONE
Definition: MethodType.h:22
#define Must(condition)
Definition: TextException.h:71
const char * methodStr() const
#define DBG_IMPORTANT
Definition: Stream.h:41
size_t base64_encode_update(struct base64_encode_ctx *ctx, char *dst, size_t length, const uint8_t *src)
Definition: base64.c:265
bool gotEncapsulated(const char *section) const
Definition: ModXact.cc:1813
void add(const SBuf &key, const SBuf &value)
Definition: Notes.cc:312
virtual void swanSong()
Definition: Xaction.cc:570
@ methodReqmod
Definition: Elements.h:17
static char * masterx_shared_name
Definition: Config.h:45
@ scInvalidHeader
Definition: StatusCode.h:86
String ssluser
the username from SSL
Definition: History.h:41
virtual const char * status() const
internal cleanup; do not call directly
Definition: Xaction.cc:632
const XactOutcome xoSatisfied
request satisfaction
Definition: Elements.cc:26
virtual void swanSong()
Definition: ModXact.cc:1293
void finishNullOrEmptyBodyPreview(MemBuf &buf)
Definition: ModXact.cc:1734
virtual void detailError(const ErrorDetail::Pointer &errDetail)
record error detail in the virgin request if possible
Definition: ModXact.cc:1995
virtual void clearError()
clear stored error details, if any; used for retries/repeats
Definition: ModXact.cc:2007
void expect(int64_t aSize)
Definition: ModXact.cc:1878
@ scOkay
Definition: StatusCode.h:26
const ServiceConfig & cfg() const
Definition: Service.h:51
HttpReply::Pointer icapReply
received ICAP reply, if any
Definition: Xaction.h:64
void recordMeta(const HttpHeader *lm)
store the last meta header fields received from the adaptation service
Definition: History.cc:140
int recordXactStart(const String &serviceId, const timeval &when, bool retrying)
record the start of a xact, return xact history ID
Definition: History.cc:51
bool parse(const char *buf, int len, int atEnd, Http::StatusCode *error)
Definition: ModXact.cc:2068
#define false
Definition: GnuRegex.c:233
bool parseHead(Http::Message *head)
Definition: ModXact.cc:1109
A const & min(A const &lhs, A const &rhs)
const XactOutcome xoModified
replaced virgin msg with adapted
Definition: Elements.cc:25
int adaptHistoryId
adaptation history slot reservation
Definition: ModXact.h:319
Ip::Address client_addr
Definition: HttpRequest.h:149
void makeUsernameHeader(const HttpRequest *request, MemBuf &buf)
Definition: ModXact.cc:1555
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
virtual void start()
called by AsyncStart; do not call directly
Definition: Xaction.cc:140
size_t debt() const
Definition: ModXact.cc:1965
void updateHistory(bool start)
starts or stops transaction accounting in ICAP history
Definition: ModXact.cc:2051
const CharacterSet crlf("crlf","\r\n")
virtual void noteBodyProductionEnded(BodyPipe::Pointer)
Definition: ModXact.cc:1242
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
@ PROXY_AUTHORIZATION

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors