Include.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 86 ESI processing */
10 
11 #include "squid.h"
12 
13 #if USE_SQUID_ESI
14 
15 #include "client_side.h"
16 #include "client_side_request.h"
17 #include "esi/Include.h"
18 #include "esi/VarState.h"
19 #include "fatal.h"
20 #include "http/Stream.h"
21 #include "HttpReply.h"
22 #include "log/access_log.h"
23 
25 
26 /* other */
29 /* esiStreamContext */
31 
32 /* ESI TO CONSIDER:
33  * 1. retry failed upstream requests
34  */
35 
36 /* Detach from a buffering stream
37  */
38 void
40 {
41  /* Detach ourselves */
42  clientStreamDetach (node, http);
43 }
44 
62 void
64 {
65  /* Test preconditions */
66  assert (node != NULL);
67  /* ESI TODO: handle thisNode rather than asserting
68  * - it should only ever happen if we cause an
69  * abort and the callback chain loops back to
70  * here, so we can simply return. However, that
71  * itself shouldn't happen, so it stays as an
72  * assert for now. */
74  assert (node->node.next == NULL);
75  assert (http->getConn() == NULL);
76 
77  ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
78  assert (esiStream.getRaw() != NULL);
79  /* If segments become more flexible, ignore thisNode */
80  assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
81  assert (!esiStream->finished);
82 
83  debugs (86,5, HERE << "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
84  assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
85 
86  /* trivial case */
87 
88  if (http->out.offset != 0) {
89  assert(rep == NULL);
90  } else {
91  if (rep) {
92  if (rep->sline.status() != Http::scOkay) {
93  rep = NULL;
94  esiStream->include->includeFail (esiStream);
95  esiStream->finished = 1;
96  httpRequestFree (http);
97  return;
98  }
99 
100 #if HEADERS_LOG
101  /* should be done in the store rather than every recipient? */
102  headersLog(0, 0, http->request->method, rep);
103 
104 #endif
105  rep = NULL;
106  }
107  }
108 
109  if (receivedData.data && receivedData.length) {
110  http->out.offset += receivedData.length;
111 
112  if (receivedData.data >= esiStream->localbuffer->buf &&
113  receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
114  /* original static buffer */
115 
116  if (receivedData.data != esiStream->localbuffer->buf) {
117  /* But not the start of it */
118  memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
119  }
120 
121  esiStream->localbuffer->len = receivedData.length;
122  } else {
123  assert (esiStream->buffer.getRaw() != NULL);
124  esiStream->buffer->len = receivedData.length;
125  }
126  }
127 
128  /* EOF / Read error / aborted entry */
129  if (rep == NULL && receivedData.data == NULL && receivedData.length == 0) {
130  /* TODO: get stream status to test the entry for aborts */
131  debugs(86, 5, HERE << "Finished reading upstream data in subrequest");
132  esiStream->include->subRequestDone (esiStream, true);
133  esiStream->finished = 1;
134  httpRequestFree (http);
135  return;
136  }
137 
138  /* after the write to the user occurs, (ie here, or in a callback)
139  * we call */
140  if (clientHttpRequestStatus(-1, http)) {
141  /* TODO: Does thisNode if block leak htto ? */
142  /* XXX when reviewing ESI this is the first place to look */
143  node->data = NULL;
144  esiStream->finished = 1;
145  esiStream->include->includeFail (esiStream);
146  return;
147  };
148 
149  switch (clientStreamStatus (node, http)) {
150 
151  case STREAM_UNPLANNED_COMPLETE: /* fallthru ok */
152 
153  case STREAM_COMPLETE: /* ok */
154  debugs(86, 3, "ESI subrequest finished OK");
155  esiStream->include->subRequestDone (esiStream, true);
156  esiStream->finished = 1;
157  httpRequestFree (http);
158  return;
159 
160  case STREAM_FAILED:
161  debugs(86, DBG_IMPORTANT, "ESI subrequest failed transfer");
162  esiStream->include->includeFail (esiStream);
163  esiStream->finished = 1;
164  httpRequestFree (http);
165  return;
166 
167  case STREAM_NONE: {
168  StoreIOBuffer tempBuffer;
169 
170  if (!esiStream->buffer.getRaw()) {
171  esiStream->buffer = esiStream->localbuffer;
172  }
173 
174  esiStream->buffer = esiStream->buffer->tail();
175 
176  if (esiStream->buffer->len) {
177  esiStream->buffer->next = new ESISegment;
178  esiStream->buffer = esiStream->buffer->next;
179  }
180 
181  tempBuffer.offset = http->out.offset;
182  tempBuffer.length = sizeof (esiStream->buffer->buf);
183  tempBuffer.data = esiStream->buffer->buf;
184  /* now just read into 'buffer' */
185  clientStreamRead (node, http, tempBuffer);
186  debugs(86, 5, HERE << "Requested more data for ESI subrequest");
187  }
188 
189  break;
190 
191  default:
192  fatal ("Hit unreachable code in esiBufferRecipient\n");
193  }
194 
195 }
196 
197 /* esiStream functions */
199 {
200  freeResources();
201 }
202 
203 void
205 {
206  debugs(86, 5, "Freeing stream context resources.");
207  buffer = NULL;
208  localbuffer = NULL;
209  include = NULL;
210 }
211 
214 {
216  rv->include = include;
217  return rv;
218 }
219 
220 /* ESIInclude */
222 {
223  debugs(86, 5, "ESIInclude::Free " << this);
227  safe_free (srcurl);
228  safe_free (alturl);
229 }
230 
231 void
233 {
234  parent = NULL;
235 }
236 
239 {
240  return new ESIInclude (*this);
241 }
242 
245 {
246  ESIInclude *resultI = new ESIInclude (*this);
247  ESIElement::Pointer result = resultI;
248  resultI->parent = newParent;
249  resultI->varState = cbdataReference (&newVarState);
250 
251  if (resultI->srcurl)
252  resultI->src = ESIStreamContextNew (resultI);
253 
254  if (resultI->alturl)
255  resultI->alt = ESIStreamContextNew (resultI);
256 
257  return result;
258 }
259 
261  varState(NULL),
262  srcurl(NULL),
263  alturl(NULL),
264  parent(NULL),
265  started(false),
266  sent(false)
267 {
268  memset(&flags, 0, sizeof(flags));
269  flags.onerrorcontinue = old.flags.onerrorcontinue;
270 
271  if (old.srcurl)
272  srcurl = xstrdup(old.srcurl);
273 
274  if (old.alturl)
275  alturl = xstrdup(old.alturl);
276 }
277 
278 void
280 {
281  tempheaders.update(&vars->header());
282  tempheaders.removeHopByHopEntries();
283 }
284 
285 void
287 {
288  if (!stream.getRaw())
289  return;
290 
291  HttpHeader tempheaders(hoRequest);
292 
293  prepareRequestHeaders(tempheaders, vars);
294 
295  /* Ensure variable state is clean */
296  vars->feedData(url, strlen (url));
297 
298  /* tempUrl is eaten by the request */
299  char const *tempUrl = vars->extractChar ();
300 
301  debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
303  if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ, mx)) {
304  debugs(86, DBG_CRITICAL, "starting new ESI subrequest failed");
305  }
306 
307  tempheaders.clean();
308 }
309 
310 ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
311  varState(NULL),
312  srcurl(NULL),
313  alturl(NULL),
314  parent(aParent),
315  started(false),
316  sent(false)
317 {
318  assert (aContext);
319  memset(&flags, 0, sizeof(flags));
320 
321  for (int i = 0; i < attrcount && attr[i]; i += 2) {
322  if (!strcmp(attr[i],"src")) {
323  /* Start a request for thisNode url */
324  debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
325 
326  /* TODO: don't assert on thisNode, ignore the duplicate */
327  assert (src.getRaw() == NULL);
328  src = ESIStreamContextNew (this);
329  assert (src.getRaw() != NULL);
330  srcurl = xstrdup(attr[i+1]);
331  } else if (!strcmp(attr[i],"alt")) {
332  /* Start a secondary request for thisNode url */
333  /* TODO: make a config parameter to wait on requesting alt's
334  * for the src to fail
335  */
336  debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
337 
338  assert (alt.getRaw() == NULL); /* TODO: FIXME */
339  alt = ESIStreamContextNew (this);
340  assert (alt.getRaw() != NULL);
341  alturl = xstrdup(attr[i+1]);
342  } else if (!strcmp(attr[i],"onerror")) {
343  if (!strcmp(attr[i+1], "continue")) {
344  flags.onerrorcontinue = 1;
345  } else {
346  /* ignore mistyped attributes */
347  debugs(86, DBG_IMPORTANT, "invalid value for onerror='" << attr[i+1] << "'");
348  }
349  } else {
350  /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
351  */
352  }
353  }
354 
355  varState = cbdataReference(aContext->varState);
356 }
357 
358 void
360 {
361  /* prevent freeing ourselves */
362  ESIIncludePtr foo(this);
363 
364  if (started)
365  return;
366 
367  started = true;
368 
369  if (src.getRaw()) {
370  Start (src, srcurl, varState);
371  Start (alt, alturl, varState);
372  } else {
373  alt = NULL;
374 
375  debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
376 
377  flags.failed = 1;
378  }
379 }
380 
381 void
383 {
384  if (sent)
385  return;
386 
387  ESISegment::Pointer myout;
388 
389  debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
390 
391  assert (flags.finished || (flags.failed && flags.onerrorcontinue));
392 
393  if (flags.failed && flags.onerrorcontinue) {
394  return;
395  }
396 
397  /* Render the content */
398  if (srccontent.getRaw()) {
399  myout = srccontent;
400  srccontent = NULL;
401  } else if (altcontent.getRaw()) {
402  myout = altcontent;
403  altcontent = NULL;
404  } else
405  fatal ("ESIIncludeRender called with no content, and no failure!\n");
406 
407  assert (output->next == NULL);
408 
409  output->next = myout;
410 
411  sent = true;
412 }
413 
416 {
417  /* Prevent refcount race leading to free */
418  Pointer me (this);
419  start();
420  debugs(86, 5, "ESIIncludeRender: Processing include " << this);
421 
422  if (flags.failed) {
423  if (flags.onerrorcontinue)
424  return ESI_PROCESS_COMPLETE;
425  else
426  return ESI_PROCESS_FAILED;
427  }
428 
429  if (!flags.finished) {
430  if (flags.onerrorcontinue)
432  else
434  }
435 
436  return ESI_PROCESS_COMPLETE;
437 }
438 
439 void
441 {
442  subRequestDone (stream, false);
443 }
444 
445 bool
447 {
448  return !(flags.finished || flags.failed);
449 }
450 
451 void
453 {
454  if (!dataNeeded())
455  return;
456 
457  if (stream == src) {
458  debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
459 
460  if (success) {
461  /* copy the lead segment */
462  debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
463  assert (!srccontent.getRaw());
465  /* we're done! */
466  flags.finished = 1;
467  } else {
468  /* Fail if there is no alt being retrieved */
469  debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
470 
471  if (!(alt.getRaw() || altcontent.getRaw())) {
472  debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
473  flags.failed = 1;
474  } else if (altcontent.getRaw()) {
475  debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
476  /* ALT was already retrieved, we are done */
477  flags.finished = 1;
478  }
479  }
480 
481  src = NULL;
482  } else if (stream == alt) {
483  debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
484 
485  if (success) {
486  debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
487  /* copy the lead segment */
488  assert (!altcontent.getRaw());
490  /* we're done! */
491 
492  if (!(src.getRaw() || srccontent.getRaw())) {
493  /* src already failed, kick ESI processor */
494  debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
495  flags.finished = 1;
496  }
497  } else {
498  if (!(src.getRaw() || srccontent.getRaw())) {
499  debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
500  /* src already failed */
501  flags.failed = 1;
502  }
503  }
504 
505  alt = NULL;
506  } else {
507  fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
508  }
509 
510  if (flags.finished || flags.failed) {
511  /* Kick ESI Processor */
512  debugs (86, 5, "ESIInclude " << this <<
513  " SubRequest " << stream.getRaw() <<
514  " completed, kicking processor , status " <<
515  (flags.finished ? "OK" : "FAILED"));
516  /* There is a race condition - and we have no reproducible test case -
517  * during a subrequest the parent will get set to NULL, which is not
518  * meant to be possible. Rather than killing squid, we let it leak
519  * memory but complain in the log.
520  *
521  * Someone wanting to debug this could well start by running squid with
522  * a hardware breakpoint set to this location.
523  * Its probably due to parent being set to null - by a call to
524  * 'this.finish' while the subrequest is still not completed.
525  */
526  if (parent.getRaw() == NULL) {
527  debugs (86, 0, "ESIInclude::subRequestDone: Sub request completed "
528  "after finish() called and parent unlinked. Unable to "
529  "continue handling the request, and may be memory leaking. "
530  "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
531  "are looking for a reproducible test case. This will require "
532  "an ESI template with includes, probably with alt-options, "
533  "and we're likely to need traffic dumps to allow us to "
534  "reconstruct the exact tcp handling sequences to trigger this "
535  "rather elusive bug.");
536  return;
537  }
538  assert (parent.getRaw());
539 
540  if (!flags.failed) {
541  sent = true;
543 
544  if (srccontent.getRaw())
545  srccontent = NULL;
546  else
547  altcontent = NULL;
548  } else if (flags.onerrorcontinue) {
549  /* render nothing but inform of completion */
550 
551  if (!sent) {
552  sent = true;
553  parent->provideData (new ESISegment, this);
554  } else
555  assert (0);
556  } else
557  parent->fail(this, "esi:include could not be completed.");
558  }
559 }
560 
561 #endif /* USE_SQUID_ESI */
562 
#define HTTP_REQBUF_SZ
Definition: defines.h:216
#define assert(EX)
Definition: assert.h:17
ESISegment::Pointer srccontent
Definition: Include.h:56
ConnStateData * getConn() const
#define cbdataReferenceDone(var)
Definition: cbdata.h:350
int clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
static CSD esiBufferDetach
Definition: Include.cc:28
HttpRequestMethod method
Definition: HttpRequest.h:102
void Start(ESIStreamContext::Pointer, char const *, ESIVarState *)
Definition: Include.cc:286
int i
Definition: membanger.c:49
#define xstrdup
void removeHopByHopEntries()
Definition: HttpHeader.cc:1713
#define safe_free(x)
Definition: xalloc.h:73
ESIVarState * varState
Definition: Include.h:58
int clientBeginRequest(const HttpRequestMethod &method, char const *url, CSCB *streamcallback, CSD *streamdetach, ClientStreamData streamdata, HttpHeader const *header, char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
esiProcessResult_t
Definition: Element.h:16
HttpHeader & header()
Definition: VarState.cc:70
void finish()
Definition: Include.cc:232
bool dataNeeded() const
Definition: Include.cc:446
char * extractChar()
Definition: VarState.cc:117
ESIInclude(esiTreeParentPtr, int attributes, const char **attr, ESIContext *)
Definition: Include.cc:310
#define DBG_CRITICAL
Definition: Debug.h:44
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
CBDATA_CLASS_INIT(ESIStreamContext)
Pointer next
Definition: Segment.h:42
virtual void fail(ESIElement *source, char const *reason=NULL)
Definition: Element.h:31
bool started
Definition: Include.h:67
esiTreeParentPtr parent
Definition: Include.h:65
static CSCB esiBufferRecipient
Definition: Include.cc:27
clientStream_status_t clientStreamStatus(clientStreamNode *thisObject, ClientHttpRequest *http)
bool sent
Definition: Include.h:68
Pointer makeUsable(esiTreeParentPtr, ESIVarState &) const
Definition: Include.cc:244
int64_t offset
Definition: StoreIOBuffer.h:55
Definition: parse.c:104
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
#define cbdataReference(var)
Definition: cbdata.h:341
#define DBG_IMPORTANT
Definition: Debug.h:45
static ESIStreamContext * ESIStreamContextNew(ESIIncludePtr)
Definition: Include.cc:213
dlink_node node
Definition: clientStream.h:88
void CSCB(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
client stream read callback
struct ESIInclude::@64 flags
void feedData(const char *buf, size_t len)
Definition: VarState.cc:99
void prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
Definition: Include.cc:279
ESISegment::Pointer buffer
Definition: Include.h:33
void fatal(const char *message)
Definition: fatal.cc:39
Pointer makeCacheable() const
Definition: Include.cc:238
ClientStreamData data
Definition: clientStream.h:94
ESIStreamContext::Pointer src
Definition: Include.h:54
Http::StatusLine sline
Definition: HttpReply.h:60
char buf[HTTP_REQBUF_SZ]
Definition: Segment.h:40
bool update(HttpHeader const *fresh)
Definition: HttpHeader.cc:280
void start()
Definition: Include.cc:359
ESISegment::Pointer altcontent
Definition: Include.h:57
~ESIInclude()
Definition: Include.cc:221
void ESISegmentFreeList(ESISegment::Pointer &head)
Definition: Segment.cc:19
struct ClientHttpRequest::Out out
void subRequestDone(ESIStreamContext::Pointer, bool)
Definition: Include.cc:452
void render(ESISegment::Pointer)
Definition: Include.cc:382
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:147
void clean()
Definition: HttpHeader.cc:184
ESISegment::Pointer localbuffer
Definition: Include.h:32
StoreIOBuffer readBuffer
Definition: clientStream.h:95
static void ListTransfer(Pointer &from, Pointer &to)
Definition: Segment.cc:53
ESI processing code.
ESIStreamContext::Pointer alt
Definition: Include.h:55
int onerrorcontinue
Definition: Include.h:50
virtual void provideData(ESISegment::Pointer data, ESIElement *source)
Definition: Element.h:26
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:412
esiProcessResult_t process(int dovars)
Definition: Include.cc:415
char * srcurl
Definition: Include.h:59
void includeFail(ESIStreamContext::Pointer)
Definition: Include.cc:440
ESIIncludePtr include
Definition: Include.h:31
void freeResources()
Definition: Include.cc:204
void CSD(clientStreamNode *, ClientHttpRequest *)
client stream detach
ESIVarState * varState
Definition: Context.h:132
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
char * alturl
Definition: Include.h:59
void httpRequestFree(void *data)
Definition: client_side.cc:488
C * getRaw() const
Definition: RefCount.h:74
#define NULL
Definition: types.h:166
#define false
Definition: GnuRegex.c:233
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors