Include.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 86 ESI processing */
10
11#include "squid.h"
12
13#if USE_SQUID_ESI
14
15#include "client_side.h"
16#include "client_side_request.h"
17#include "esi/Include.h"
18#include "esi/VarState.h"
19#include "fatal.h"
20#include "http/Stream.h"
21#include "HttpReply.h"
22#include "log/access_log.h"
23
25
26/* other */
29/* esiStreamContext */
31
32/* ESI TO CONSIDER:
33 * 1. retry failed upstream requests
34 */
35
36/* Detach from a buffering stream
37 */
38void
40{
41 /* Detach ourselves */
43}
44
62void
64{
65 /* Test preconditions */
66 assert (node != nullptr);
67 /* ESI TODO: handle thisNode rather than asserting
68 * - it should only ever happen if we cause an
69 * abort and the callback chain loops back to
70 * here, so we can simply return. However, that
71 * itself shouldn't happen, so it stays as an
72 * assert for now. */
74 assert (node->node.next == nullptr);
75 assert (http->getConn() == nullptr);
76
77 ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
78 assert (esiStream.getRaw() != nullptr);
79 /* If segments become more flexible, ignore thisNode */
80 assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
81 assert (!esiStream->finished);
82
83 debugs (86,5, "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
84 assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
85
86 /* trivial case */
87
88 if (http->out.offset != 0) {
89 assert(rep == nullptr);
90 } else {
91 if (rep) {
92 if (rep->sline.status() != Http::scOkay) {
93 rep = nullptr;
94 esiStream->include->includeFail (esiStream);
95 esiStream->finished = 1;
96 httpRequestFree (http);
97 return;
98 }
99
100#if HEADERS_LOG
101 /* should be done in the store rather than every recipient? */
102 headersLog(0, 0, http->request->method, rep);
103
104#endif
105 rep = nullptr;
106 }
107 }
108
109 if (receivedData.data && receivedData.length) {
110 http->out.offset += receivedData.length;
111
112 if (receivedData.data >= esiStream->localbuffer->buf &&
113 receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
114 /* original static buffer */
115
116 if (receivedData.data != esiStream->localbuffer->buf) {
117 /* But not the start of it */
118 memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
119 }
120
121 esiStream->localbuffer->len = receivedData.length;
122 } else {
123 assert (esiStream->buffer.getRaw() != nullptr);
124 esiStream->buffer->len = receivedData.length;
125 }
126 }
127
128 /* EOF / Read error / aborted entry */
129 if (rep == nullptr && receivedData.data == nullptr && receivedData.length == 0) {
130 /* TODO: get stream status to test the entry for aborts */
131 debugs(86, 5, "Finished reading upstream data in subrequest");
132 esiStream->include->subRequestDone (esiStream, true);
133 esiStream->finished = 1;
134 httpRequestFree (http);
135 return;
136 }
137
138 switch (clientStreamStatus (node, http)) {
139
141 case STREAM_COMPLETE: /* ok */
142 debugs(86, 3, "ESI subrequest finished OK");
143 esiStream->include->subRequestDone (esiStream, true);
144 esiStream->finished = 1;
145 httpRequestFree (http);
146 return;
147
148 case STREAM_FAILED:
149 debugs(86, DBG_IMPORTANT, "ERROR: ESI subrequest failed transfer");
150 esiStream->include->includeFail (esiStream);
151 esiStream->finished = 1;
152 httpRequestFree (http);
153 return;
154
155 case STREAM_NONE: {
156 StoreIOBuffer tempBuffer;
157
158 if (!esiStream->buffer.getRaw()) {
159 esiStream->buffer = esiStream->localbuffer;
160 }
161
162 esiStream->buffer = esiStream->buffer->tail();
163
164 if (esiStream->buffer->len) {
165 esiStream->buffer->next = new ESISegment;
166 esiStream->buffer = esiStream->buffer->next;
167 }
168
169 tempBuffer.offset = http->out.offset;
170 tempBuffer.length = sizeof (esiStream->buffer->buf);
171 tempBuffer.data = esiStream->buffer->buf;
172 /* now just read into 'buffer' */
173 clientStreamRead (node, http, tempBuffer);
174 debugs(86, 5, "Requested more data for ESI subrequest");
175 }
176
177 break;
178
179 default:
180 fatal ("Hit unreachable code in esiBufferRecipient\n");
181 }
182
183}
184
185/* esiStream functions */
187{
189}
190
191void
193{
194 debugs(86, 5, "Freeing stream context resources.");
195 buffer = nullptr;
196 localbuffer = nullptr;
197 include = nullptr;
198}
199
202{
204 rv->include = include;
205 return rv;
206}
207
208/* ESIInclude */
210{
211 debugs(86, 5, "ESIInclude::Free " << this);
217}
218
219void
221{
222 parent = nullptr;
223}
224
227{
228 return new ESIInclude (*this);
229}
230
233{
234 ESIInclude *resultI = new ESIInclude (*this);
235 ESIElement::Pointer result = resultI;
236 resultI->parent = newParent;
237 resultI->varState = cbdataReference (&newVarState);
238
239 if (resultI->srcurl)
240 resultI->src = ESIStreamContextNew (resultI);
241
242 if (resultI->alturl)
243 resultI->alt = ESIStreamContextNew (resultI);
244
245 return result;
246}
247
249 varState(nullptr),
250 srcurl(nullptr),
251 alturl(nullptr),
252 parent(nullptr),
253 started(false),
254 sent(false)
255{
256 memset(&flags, 0, sizeof(flags));
257 flags.onerrorcontinue = old.flags.onerrorcontinue;
258
259 if (old.srcurl)
260 srcurl = xstrdup(old.srcurl);
261
262 if (old.alturl)
263 alturl = xstrdup(old.alturl);
264}
265
266void
268{
269 tempheaders.update(&vars->header());
270 tempheaders.removeHopByHopEntries();
271}
272
273void
275{
276 if (!stream.getRaw())
277 return;
278
279 HttpHeader tempheaders(hoRequest);
280
281 prepareRequestHeaders(tempheaders, vars);
282
283 /* Ensure variable state is clean */
284 vars->feedData(url, strlen (url));
285
286 /* tempUrl is eaten by the request */
287 char const *tempUrl = vars->extractChar ();
288
289 debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
290 const auto mx = MasterXaction::MakePortless<XactionInitiator::initEsi>();
291 if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ, mx)) {
292 debugs(86, DBG_CRITICAL, "ERROR: starting new ESI subrequest failed");
293 }
294
295 tempheaders.clean();
296}
297
298ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
299 varState(nullptr),
300 srcurl(nullptr),
301 alturl(nullptr),
302 parent(aParent),
303 started(false),
304 sent(false)
305{
306 assert (aContext);
307 memset(&flags, 0, sizeof(flags));
308
309 for (int i = 0; i < attrcount && attr[i]; i += 2) {
310 if (!strcmp(attr[i],"src")) {
311 /* Start a request for thisNode url */
312 debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
313
314 /* TODO: don't assert on thisNode, ignore the duplicate */
315 assert (src.getRaw() == nullptr);
316 src = ESIStreamContextNew (this);
317 assert (src.getRaw() != nullptr);
318 srcurl = xstrdup(attr[i+1]);
319 } else if (!strcmp(attr[i],"alt")) {
320 /* Start a secondary request for thisNode url */
321 /* TODO: make a config parameter to wait on requesting alt's
322 * for the src to fail
323 */
324 debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
325
326 assert (alt.getRaw() == nullptr); /* TODO: fix? */
327 alt = ESIStreamContextNew (this);
328 assert (alt.getRaw() != nullptr);
329 alturl = xstrdup(attr[i+1]);
330 } else if (!strcmp(attr[i],"onerror")) {
331 if (!strcmp(attr[i+1], "continue")) {
332 flags.onerrorcontinue = 1;
333 } else {
334 /* ignore mistyped attributes */
335 debugs(86, DBG_IMPORTANT, "ERROR: invalid value for onerror='" << attr[i+1] << "'");
336 }
337 } else {
338 /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
339 */
340 }
341 }
342
343 varState = cbdataReference(aContext->varState);
344}
345
346void
348{
349 /* prevent freeing ourselves */
350 ESIIncludePtr foo(this);
351
352 if (started)
353 return;
354
355 started = true;
356
357 if (src.getRaw()) {
360 } else {
361 alt = nullptr;
362
363 debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
364
365 flags.failed = 1;
366 }
367}
368
369void
371{
372 if (sent)
373 return;
374
376
377 debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
378
379 assert (flags.finished || (flags.failed && flags.onerrorcontinue));
380
381 if (flags.failed && flags.onerrorcontinue) {
382 return;
383 }
384
385 /* Render the content */
386 if (srccontent.getRaw()) {
387 myout = srccontent;
388 srccontent = nullptr;
389 } else if (altcontent.getRaw()) {
390 myout = altcontent;
391 altcontent = nullptr;
392 } else
393 fatal ("ESIIncludeRender called with no content, and no failure!\n");
394
395 assert (output->next == nullptr);
396
397 output->next = myout;
398
399 sent = true;
400}
401
404{
405 /* Prevent refcount race leading to free */
406 Pointer me (this);
407 start();
408 debugs(86, 5, "ESIIncludeRender: Processing include " << this);
409
410 if (flags.failed) {
411 if (flags.onerrorcontinue)
413 else
414 return ESI_PROCESS_FAILED;
415 }
416
417 if (!flags.finished) {
418 if (flags.onerrorcontinue)
420 else
422 }
423
425}
426
427void
429{
430 subRequestDone (stream, false);
431}
432
433bool
435{
436 return !(flags.finished || flags.failed);
437}
438
439void
441{
442 if (!dataNeeded())
443 return;
444
445 if (stream == src) {
446 debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
447
448 if (success) {
449 /* copy the lead segment */
450 debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
453 /* we're done! */
454 flags.finished = 1;
455 } else {
456 /* Fail if there is no alt being retrieved */
457 debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
458
459 if (!(alt.getRaw() || altcontent.getRaw())) {
460 debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
461 flags.failed = 1;
462 } else if (altcontent.getRaw()) {
463 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
464 /* ALT was already retrieved, we are done */
465 flags.finished = 1;
466 }
467 }
468
469 src = nullptr;
470 } else if (stream == alt) {
471 debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
472
473 if (success) {
474 debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
475 /* copy the lead segment */
478 /* we're done! */
479
480 if (!(src.getRaw() || srccontent.getRaw())) {
481 /* src already failed, kick ESI processor */
482 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
483 flags.finished = 1;
484 }
485 } else {
486 if (!(src.getRaw() || srccontent.getRaw())) {
487 debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
488 /* src already failed */
489 flags.failed = 1;
490 }
491 }
492
493 alt = nullptr;
494 } else {
495 fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
496 }
497
498 if (flags.finished || flags.failed) {
499 /* Kick ESI Processor */
500 debugs (86, 5, "ESIInclude " << this <<
501 " SubRequest " << stream.getRaw() <<
502 " completed, kicking processor , status " <<
503 (flags.finished ? "OK" : "FAILED"));
504 /* There is a race condition - and we have no reproducible test case -
505 * during a subrequest the parent will get set to NULL, which is not
506 * meant to be possible. Rather than killing squid, we let it leak
507 * memory but complain in the log.
508 *
509 * Someone wanting to debug this could well start by running squid with
510 * a hardware breakpoint set to this location.
511 * Its probably due to parent being set to null - by a call to
512 * 'this.finish' while the subrequest is still not completed.
513 */
514 if (parent.getRaw() == nullptr) {
515 debugs(86, DBG_CRITICAL, "ERROR: Squid Bug #951: ESIInclude::subRequestDone: Sub request completed "
516 "after finish() called and parent unlinked. Unable to "
517 "continue handling the request, and may be memory leaking. "
518 "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
519 "are looking for a reproducible test case. This will require "
520 "an ESI template with includes, probably with alt-options, "
521 "and we're likely to need traffic dumps to allow us to "
522 "reconstruct the exact tcp handling sequences to trigger this "
523 "rather elusive bug.");
524 return;
525 }
526 assert (parent.getRaw());
527
528 if (!flags.failed) {
529 sent = true;
531
532 if (srccontent.getRaw())
533 srccontent = nullptr;
534 else
535 altcontent = nullptr;
536 } else if (flags.onerrorcontinue) {
537 /* render nothing but inform of completion */
538
539 if (!sent) {
540 sent = true;
541 parent->provideData (new ESISegment, this);
542 } else
543 assert (0);
544 } else
545 parent->fail(this, "esi:include could not be completed.");
546 }
547}
548
549#endif /* USE_SQUID_ESI */
550
esiProcessResult_t
Definition: Element.h:18
@ ESI_PROCESS_FAILED
Definition: Element.h:22
@ ESI_PROCESS_PENDING_WONTFAIL
Definition: Element.h:20
@ ESI_PROCESS_COMPLETE
Definition: Element.h:19
@ ESI_PROCESS_PENDING_MAYFAIL
Definition: Element.h:21
#define false
Definition: GnuRegex.c:240
@ hoRequest
Definition: HttpHeader.h:36
static CSCB esiBufferRecipient
Definition: Include.cc:27
static ESIStreamContext * ESIStreamContextNew(ESIIncludePtr)
Definition: Include.cc:201
CBDATA_CLASS_INIT(ESIStreamContext)
static CSD esiBufferDetach
Definition: Include.cc:28
#define assert(EX)
Definition: assert.h:19
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:398
#define cbdataReferenceDone(var)
Definition: cbdata.h:350
#define cbdataReference(var)
Definition: cbdata.h:341
struct ClientHttpRequest::Out out
HttpRequest *const request
ConnStateData * getConn() const
ESIVarState * varState
Definition: Context.h:133
void prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
Definition: Include.cc:267
void includeFail(ESIStreamContext::Pointer)
Definition: Include.cc:428
esiProcessResult_t process(int dovars)
Definition: Include.cc:403
void render(ESISegment::Pointer)
Definition: Include.cc:370
ESISegment::Pointer srccontent
Definition: Include.h:56
ESIVarState * varState
Definition: Include.h:58
ESISegment::Pointer altcontent
Definition: Include.h:57
int onerrorcontinue
Definition: Include.h:50
esiTreeParentPtr parent
Definition: Include.h:65
Pointer makeCacheable() const
Definition: Include.cc:226
void finish()
Definition: Include.cc:220
ESIStreamContext::Pointer alt
Definition: Include.h:55
bool started
Definition: Include.h:67
void subRequestDone(ESIStreamContext::Pointer, bool)
Definition: Include.cc:440
Pointer makeUsable(esiTreeParentPtr, ESIVarState &) const
Definition: Include.cc:232
char * alturl
Definition: Include.h:59
bool dataNeeded() const
Definition: Include.cc:434
ESIStreamContext::Pointer src
Definition: Include.h:54
void Start(ESIStreamContext::Pointer, char const *, ESIVarState *)
Definition: Include.cc:274
bool sent
Definition: Include.h:68
~ESIInclude()
Definition: Include.cc:209
ESIInclude(esiTreeParentPtr, int attributes, const char **attr, ESIContext *)
Definition: Include.cc:298
char * srcurl
Definition: Include.h:59
void start()
Definition: Include.cc:347
struct ESIInclude::@65 flags
static void ListTransfer(Pointer &from, Pointer &to)
Definition: Segment.cc:53
size_t len
Definition: Segment.h:41
ESISegment const * tail() const
Definition: Segment.cc:153
Pointer next
Definition: Segment.h:42
char buf[HTTP_REQBUF_SZ]
Definition: Segment.h:40
ESISegment::Pointer buffer
Definition: Include.h:33
ESISegment::Pointer localbuffer
Definition: Include.h:32
ESIIncludePtr include
Definition: Include.h:31
void freeResources()
Definition: Include.cc:192
HttpHeader & header()
Definition: VarState.cc:70
char * extractChar()
Definition: VarState.cc:117
void feedData(const char *buf, size_t len)
Definition: VarState.cc:99
void removeHopByHopEntries()
Definition: HttpHeader.cc:1707
void update(const HttpHeader *fresh)
Definition: HttpHeader.cc:272
void clean()
Definition: HttpHeader.cc:190
Http::StatusLine sline
Definition: HttpReply.h:56
HttpRequestMethod method
Definition: HttpRequest.h:114
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
C * getRaw() const
Definition: RefCount.h:80
int64_t offset
Definition: StoreIOBuffer.h:55
void CSD(clientStreamNode *, ClientHttpRequest *)
client stream detach
void CSCB(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
client stream read callback
void httpRequestFree(void *data)
Definition: client_side.cc:488
int clientBeginRequest(const HttpRequestMethod &method, char const *url, CSCB *streamcallback, CSD *streamdetach, ClientStreamData streamdata, HttpHeader const *header, char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
#define DBG_IMPORTANT
Definition: Stream.h:41
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
#define DBG_CRITICAL
Definition: Stream.h:40
@ STREAM_COMPLETE
Definition: enums.h:127
@ STREAM_UNPLANNED_COMPLETE
Definition: enums.h:132
@ STREAM_NONE
Definition: enums.h:126
@ STREAM_FAILED
Definition: enums.h:137
void ESISegmentFreeList(ESISegment::Pointer &head)
Definition: Segment.cc:19
void fatal(const char *message)
Definition: fatal.cc:28
clientStream_status_t clientStreamStatus(clientStreamNode *thisObject, ClientHttpRequest *http)
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
#define HTTP_REQBUF_SZ
Definition: forward.h:14
@ scOkay
Definition: StatusCode.h:26
@ METHOD_GET
Definition: MethodType.h:25
#define xstrdup
virtual void fail(ESIElement *, char const *=nullptr)
Definition: Element.h:33
virtual void provideData(ESISegment::Pointer, ESIElement *)
Definition: Element.h:28
Definition: parse.c:104
struct node * next
Definition: parse.c:105
#define safe_free(x)
Definition: xalloc.h:73

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors