Include.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 86 ESI processing */
10 
11 #include "squid.h"
12 
13 #if USE_SQUID_ESI
14 
15 #include "client_side.h"
16 #include "client_side_request.h"
17 #include "esi/Include.h"
18 #include "esi/VarState.h"
19 #include "fatal.h"
20 #include "http/Stream.h"
21 #include "HttpReply.h"
22 #include "log/access_log.h"
23 
25 
26 /* other */
29 /* esiStreamContext */
31 
32 /* ESI TO CONSIDER:
33  * 1. retry failed upstream requests
34  */
35 
36 /* Detach from a buffering stream
37  */
38 void
40 {
41  /* Detach ourselves */
42  clientStreamDetach (node, http);
43 }
44 
62 void
64 {
65  /* Test preconditions */
66  assert (node != NULL);
67  /* ESI TODO: handle thisNode rather than asserting
68  * - it should only ever happen if we cause an
69  * abort and the callback chain loops back to
70  * here, so we can simply return. However, that
71  * itself shouldn't happen, so it stays as an
72  * assert for now. */
74  assert (node->node.next == NULL);
75  assert (http->getConn() == NULL);
76 
77  ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
78  assert (esiStream.getRaw() != NULL);
79  /* If segments become more flexible, ignore thisNode */
80  assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
81  assert (!esiStream->finished);
82 
83  debugs (86,5, "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
84  assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
85 
86  /* trivial case */
87 
88  if (http->out.offset != 0) {
89  assert(rep == NULL);
90  } else {
91  if (rep) {
92  if (rep->sline.status() != Http::scOkay) {
93  rep = NULL;
94  esiStream->include->includeFail (esiStream);
95  esiStream->finished = 1;
96  httpRequestFree (http);
97  return;
98  }
99 
100 #if HEADERS_LOG
101  /* should be done in the store rather than every recipient? */
102  headersLog(0, 0, http->request->method, rep);
103 
104 #endif
105  rep = NULL;
106  }
107  }
108 
109  if (receivedData.data && receivedData.length) {
110  http->out.offset += receivedData.length;
111 
112  if (receivedData.data >= esiStream->localbuffer->buf &&
113  receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
114  /* original static buffer */
115 
116  if (receivedData.data != esiStream->localbuffer->buf) {
117  /* But not the start of it */
118  memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
119  }
120 
121  esiStream->localbuffer->len = receivedData.length;
122  } else {
123  assert (esiStream->buffer.getRaw() != NULL);
124  esiStream->buffer->len = receivedData.length;
125  }
126  }
127 
128  /* EOF / Read error / aborted entry */
129  if (rep == NULL && receivedData.data == NULL && receivedData.length == 0) {
130  /* TODO: get stream status to test the entry for aborts */
131  debugs(86, 5, "Finished reading upstream data in subrequest");
132  esiStream->include->subRequestDone (esiStream, true);
133  esiStream->finished = 1;
134  httpRequestFree (http);
135  return;
136  }
137 
138  switch (clientStreamStatus (node, http)) {
139 
141  case STREAM_COMPLETE: /* ok */
142  debugs(86, 3, "ESI subrequest finished OK");
143  esiStream->include->subRequestDone (esiStream, true);
144  esiStream->finished = 1;
145  httpRequestFree (http);
146  return;
147 
148  case STREAM_FAILED:
149  debugs(86, DBG_IMPORTANT, "ERROR: ESI subrequest failed transfer");
150  esiStream->include->includeFail (esiStream);
151  esiStream->finished = 1;
152  httpRequestFree (http);
153  return;
154 
155  case STREAM_NONE: {
156  StoreIOBuffer tempBuffer;
157 
158  if (!esiStream->buffer.getRaw()) {
159  esiStream->buffer = esiStream->localbuffer;
160  }
161 
162  esiStream->buffer = esiStream->buffer->tail();
163 
164  if (esiStream->buffer->len) {
165  esiStream->buffer->next = new ESISegment;
166  esiStream->buffer = esiStream->buffer->next;
167  }
168 
169  tempBuffer.offset = http->out.offset;
170  tempBuffer.length = sizeof (esiStream->buffer->buf);
171  tempBuffer.data = esiStream->buffer->buf;
172  /* now just read into 'buffer' */
173  clientStreamRead (node, http, tempBuffer);
174  debugs(86, 5, "Requested more data for ESI subrequest");
175  }
176 
177  break;
178 
179  default:
180  fatal ("Hit unreachable code in esiBufferRecipient\n");
181  }
182 
183 }
184 
185 /* esiStream functions */
187 {
188  freeResources();
189 }
190 
191 void
193 {
194  debugs(86, 5, "Freeing stream context resources.");
195  buffer = NULL;
196  localbuffer = NULL;
197  include = NULL;
198 }
199 
202 {
204  rv->include = include;
205  return rv;
206 }
207 
208 /* ESIInclude */
210 {
211  debugs(86, 5, "ESIInclude::Free " << this);
215  safe_free (srcurl);
216  safe_free (alturl);
217 }
218 
219 void
221 {
222  parent = NULL;
223 }
224 
227 {
228  return new ESIInclude (*this);
229 }
230 
233 {
234  ESIInclude *resultI = new ESIInclude (*this);
235  ESIElement::Pointer result = resultI;
236  resultI->parent = newParent;
237  resultI->varState = cbdataReference (&newVarState);
238 
239  if (resultI->srcurl)
240  resultI->src = ESIStreamContextNew (resultI);
241 
242  if (resultI->alturl)
243  resultI->alt = ESIStreamContextNew (resultI);
244 
245  return result;
246 }
247 
249  varState(NULL),
250  srcurl(NULL),
251  alturl(NULL),
252  parent(NULL),
253  started(false),
254  sent(false)
255 {
256  memset(&flags, 0, sizeof(flags));
257  flags.onerrorcontinue = old.flags.onerrorcontinue;
258 
259  if (old.srcurl)
260  srcurl = xstrdup(old.srcurl);
261 
262  if (old.alturl)
263  alturl = xstrdup(old.alturl);
264 }
265 
266 void
268 {
269  tempheaders.update(&vars->header());
270  tempheaders.removeHopByHopEntries();
271 }
272 
273 void
275 {
276  if (!stream.getRaw())
277  return;
278 
279  HttpHeader tempheaders(hoRequest);
280 
281  prepareRequestHeaders(tempheaders, vars);
282 
283  /* Ensure variable state is clean */
284  vars->feedData(url, strlen (url));
285 
286  /* tempUrl is eaten by the request */
287  char const *tempUrl = vars->extractChar ();
288 
289  debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
290  const auto mx = MasterXaction::MakePortless<XactionInitiator::initEsi>();
291  if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ, mx)) {
292  debugs(86, DBG_CRITICAL, "ERROR: starting new ESI subrequest failed");
293  }
294 
295  tempheaders.clean();
296 }
297 
298 ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
299  varState(NULL),
300  srcurl(NULL),
301  alturl(NULL),
302  parent(aParent),
303  started(false),
304  sent(false)
305 {
306  assert (aContext);
307  memset(&flags, 0, sizeof(flags));
308 
309  for (int i = 0; i < attrcount && attr[i]; i += 2) {
310  if (!strcmp(attr[i],"src")) {
311  /* Start a request for thisNode url */
312  debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
313 
314  /* TODO: don't assert on thisNode, ignore the duplicate */
315  assert (src.getRaw() == NULL);
316  src = ESIStreamContextNew (this);
317  assert (src.getRaw() != NULL);
318  srcurl = xstrdup(attr[i+1]);
319  } else if (!strcmp(attr[i],"alt")) {
320  /* Start a secondary request for thisNode url */
321  /* TODO: make a config parameter to wait on requesting alt's
322  * for the src to fail
323  */
324  debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
325 
326  assert (alt.getRaw() == NULL); /* TODO: fix? */
327  alt = ESIStreamContextNew (this);
328  assert (alt.getRaw() != NULL);
329  alturl = xstrdup(attr[i+1]);
330  } else if (!strcmp(attr[i],"onerror")) {
331  if (!strcmp(attr[i+1], "continue")) {
332  flags.onerrorcontinue = 1;
333  } else {
334  /* ignore mistyped attributes */
335  debugs(86, DBG_IMPORTANT, "ERROR: invalid value for onerror='" << attr[i+1] << "'");
336  }
337  } else {
338  /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
339  */
340  }
341  }
342 
343  varState = cbdataReference(aContext->varState);
344 }
345 
346 void
348 {
349  /* prevent freeing ourselves */
350  ESIIncludePtr foo(this);
351 
352  if (started)
353  return;
354 
355  started = true;
356 
357  if (src.getRaw()) {
358  Start (src, srcurl, varState);
359  Start (alt, alturl, varState);
360  } else {
361  alt = NULL;
362 
363  debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
364 
365  flags.failed = 1;
366  }
367 }
368 
369 void
371 {
372  if (sent)
373  return;
374 
375  ESISegment::Pointer myout;
376 
377  debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
378 
379  assert (flags.finished || (flags.failed && flags.onerrorcontinue));
380 
381  if (flags.failed && flags.onerrorcontinue) {
382  return;
383  }
384 
385  /* Render the content */
386  if (srccontent.getRaw()) {
387  myout = srccontent;
388  srccontent = NULL;
389  } else if (altcontent.getRaw()) {
390  myout = altcontent;
391  altcontent = NULL;
392  } else
393  fatal ("ESIIncludeRender called with no content, and no failure!\n");
394 
395  assert (output->next == NULL);
396 
397  output->next = myout;
398 
399  sent = true;
400 }
401 
404 {
405  /* Prevent refcount race leading to free */
406  Pointer me (this);
407  start();
408  debugs(86, 5, "ESIIncludeRender: Processing include " << this);
409 
410  if (flags.failed) {
411  if (flags.onerrorcontinue)
412  return ESI_PROCESS_COMPLETE;
413  else
414  return ESI_PROCESS_FAILED;
415  }
416 
417  if (!flags.finished) {
418  if (flags.onerrorcontinue)
420  else
422  }
423 
424  return ESI_PROCESS_COMPLETE;
425 }
426 
427 void
429 {
430  subRequestDone (stream, false);
431 }
432 
433 bool
435 {
436  return !(flags.finished || flags.failed);
437 }
438 
439 void
441 {
442  if (!dataNeeded())
443  return;
444 
445  if (stream == src) {
446  debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
447 
448  if (success) {
449  /* copy the lead segment */
450  debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
451  assert (!srccontent.getRaw());
453  /* we're done! */
454  flags.finished = 1;
455  } else {
456  /* Fail if there is no alt being retrieved */
457  debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
458 
459  if (!(alt.getRaw() || altcontent.getRaw())) {
460  debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
461  flags.failed = 1;
462  } else if (altcontent.getRaw()) {
463  debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
464  /* ALT was already retrieved, we are done */
465  flags.finished = 1;
466  }
467  }
468 
469  src = NULL;
470  } else if (stream == alt) {
471  debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
472 
473  if (success) {
474  debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
475  /* copy the lead segment */
476  assert (!altcontent.getRaw());
478  /* we're done! */
479 
480  if (!(src.getRaw() || srccontent.getRaw())) {
481  /* src already failed, kick ESI processor */
482  debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
483  flags.finished = 1;
484  }
485  } else {
486  if (!(src.getRaw() || srccontent.getRaw())) {
487  debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
488  /* src already failed */
489  flags.failed = 1;
490  }
491  }
492 
493  alt = NULL;
494  } else {
495  fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
496  }
497 
498  if (flags.finished || flags.failed) {
499  /* Kick ESI Processor */
500  debugs (86, 5, "ESIInclude " << this <<
501  " SubRequest " << stream.getRaw() <<
502  " completed, kicking processor , status " <<
503  (flags.finished ? "OK" : "FAILED"));
504  /* There is a race condition - and we have no reproducible test case -
505  * during a subrequest the parent will get set to NULL, which is not
506  * meant to be possible. Rather than killing squid, we let it leak
507  * memory but complain in the log.
508  *
509  * Someone wanting to debug this could well start by running squid with
510  * a hardware breakpoint set to this location.
511  * Its probably due to parent being set to null - by a call to
512  * 'this.finish' while the subrequest is still not completed.
513  */
514  if (parent.getRaw() == NULL) {
515  debugs(86, DBG_CRITICAL, "ERROR: Squid Bug #951: ESIInclude::subRequestDone: Sub request completed "
516  "after finish() called and parent unlinked. Unable to "
517  "continue handling the request, and may be memory leaking. "
518  "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
519  "are looking for a reproducible test case. This will require "
520  "an ESI template with includes, probably with alt-options, "
521  "and we're likely to need traffic dumps to allow us to "
522  "reconstruct the exact tcp handling sequences to trigger this "
523  "rather elusive bug.");
524  return;
525  }
526  assert (parent.getRaw());
527 
528  if (!flags.failed) {
529  sent = true;
531 
532  if (srccontent.getRaw())
533  srccontent = NULL;
534  else
535  altcontent = NULL;
536  } else if (flags.onerrorcontinue) {
537  /* render nothing but inform of completion */
538 
539  if (!sent) {
540  sent = true;
541  parent->provideData (new ESISegment, this);
542  } else
543  assert (0);
544  } else
545  parent->fail(this, "esi:include could not be completed.");
546  }
547 }
548 
549 #endif /* USE_SQUID_ESI */
550 
ESIStreamContext::Pointer src
Definition: Include.h:54
void fatal(const char *message)
Definition: fatal.cc:28
Definition: parse.c:104
@ ESI_PROCESS_PENDING_WONTFAIL
Definition: Element.h:20
clientStream_status_t clientStreamStatus(clientStreamNode *thisObject, ClientHttpRequest *http)
#define DBG_CRITICAL
Definition: Stream.h:40
static CSD esiBufferDetach
Definition: Include.cc:28
bool dataNeeded() const
Definition: Include.cc:434
void Start(ESIStreamContext::Pointer, char const *, ESIVarState *)
Definition: Include.cc:274
void removeHopByHopEntries()
Definition: HttpHeader.cc:1739
size_t len
Definition: Segment.h:41
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
void ESISegmentFreeList(ESISegment::Pointer &head)
Definition: Segment.cc:19
ESISegment::Pointer buffer
Definition: Include.h:33
struct node * next
Definition: parse.c:105
ConnStateData * getConn() const
@ STREAM_NONE
Definition: enums.h:126
int64_t offset
Definition: StoreIOBuffer.h:55
void httpRequestFree(void *data)
Definition: client_side.cc:488
#define xstrdup
Pointer next
Definition: Segment.h:42
esiTreeParentPtr parent
Definition: Include.h:65
struct ClientHttpRequest::Out out
C * getRaw() const
Definition: RefCount.h:80
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:398
Http::StatusLine sline
Definition: HttpReply.h:56
ESIVarState * varState
Definition: Include.h:58
@ STREAM_COMPLETE
Definition: enums.h:127
char * srcurl
Definition: Include.h:59
static void ListTransfer(Pointer &from, Pointer &to)
Definition: Segment.cc:53
#define cbdataReference(var)
Definition: cbdata.h:341
CBDATA_CLASS_INIT(ESIStreamContext)
@ STREAM_FAILED
Definition: enums.h:137
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
struct ESIInclude::@62 flags
void subRequestDone(ESIStreamContext::Pointer, bool)
Definition: Include.cc:440
void includeFail(ESIStreamContext::Pointer)
Definition: Include.cc:428
esiProcessResult_t process(int dovars)
Definition: Include.cc:403
ESISegment::Pointer localbuffer
Definition: Include.h:32
char * alturl
Definition: Include.h:59
Pointer makeCacheable() const
Definition: Include.cc:226
#define NULL
Definition: types.h:166
static CSCB esiBufferRecipient
Definition: Include.cc:27
bool started
Definition: Include.h:67
void CSD(clientStreamNode *, ClientHttpRequest *)
client stream detach
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
ESISegment::Pointer altcontent
Definition: Include.h:57
static ESIStreamContext * ESIStreamContextNew(ESIIncludePtr)
Definition: Include.cc:201
@ STREAM_UNPLANNED_COMPLETE
Definition: enums.h:132
void render(ESISegment::Pointer)
Definition: Include.cc:370
void freeResources()
Definition: Include.cc:192
#define safe_free(x)
Definition: xalloc.h:73
@ ESI_PROCESS_FAILED
Definition: Element.h:22
#define assert(EX)
Definition: assert.h:19
ESISegment::Pointer srccontent
Definition: Include.h:56
#define cbdataReferenceDone(var)
Definition: cbdata.h:350
ESIVarState * varState
Definition: Context.h:133
ESIIncludePtr include
Definition: Include.h:31
char buf[HTTP_REQBUF_SZ]
Definition: Segment.h:40
void start()
Definition: Include.cc:347
~ESIInclude()
Definition: Include.cc:209
int onerrorcontinue
Definition: Include.h:50
esiProcessResult_t
Definition: Element.h:18
bool sent
Definition: Include.h:68
HttpRequestMethod method
Definition: HttpRequest.h:114
@ ESI_PROCESS_PENDING_MAYFAIL
Definition: Element.h:21
ESIInclude(esiTreeParentPtr, int attributes, const char **attr, ESIContext *)
Definition: Include.cc:298
HttpHeader & header()
Definition: VarState.cc:70
@ ESI_PROCESS_COMPLETE
Definition: Element.h:19
int clientBeginRequest(const HttpRequestMethod &method, char const *url, CSCB *streamcallback, CSD *streamdetach, ClientStreamData streamdata, HttpHeader const *header, char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
char * extractChar()
Definition: VarState.cc:117
#define DBG_IMPORTANT
Definition: Stream.h:41
#define HTTP_REQBUF_SZ
Definition: forward.h:14
virtual void fail(ESIElement *, char const *=nullptr)
Definition: Element.h:33
void update(const HttpHeader *fresh)
Definition: HttpHeader.cc:294
void prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
Definition: Include.cc:267
void finish()
Definition: Include.cc:220
Pointer makeUsable(esiTreeParentPtr, ESIVarState &) const
Definition: Include.cc:232
@ scOkay
Definition: StatusCode.h:26
ESIStreamContext::Pointer alt
Definition: Include.h:55
@ METHOD_GET
Definition: MethodType.h:25
#define false
Definition: GnuRegex.c:233
void CSCB(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
client stream read callback
ESISegment const * tail() const
Definition: Segment.cc:153
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
void feedData(const char *buf, size_t len)
Definition: VarState.cc:99
@ hoRequest
Definition: HttpHeader.h:36
virtual void provideData(ESISegment::Pointer, ESIElement *)
Definition: Element.h:28
void clean()
Definition: HttpHeader.cc:190
HttpRequest *const request

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors