Include.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 86 ESI processing */
10
11#include "squid.h"
12
13#if USE_SQUID_ESI
14
15#include "client_side.h"
16#include "client_side_request.h"
17#include "esi/Include.h"
18#include "esi/VarState.h"
19#include "fatal.h"
20#include "http/Stream.h"
21#include "HttpReply.h"
22#include "log/access_log.h"
23
25
26/* other */
29/* esiStreamContext */
31
32/* ESI TO CONSIDER:
33 * 1. retry failed upstream requests
34 */
35
36/* Detach from a buffering stream
37 */
38void
40{
41 /* Detach ourselves */
43}
44
62void
64{
65 /* Test preconditions */
66 assert (node != nullptr);
67 /* ESI TODO: handle thisNode rather than asserting
68 * - it should only ever happen if we cause an
69 * abort and the callback chain loops back to
70 * here, so we can simply return. However, that
71 * itself shouldn't happen, so it stays as an
72 * assert for now. */
74 assert (node->node.next == nullptr);
75 assert (http->getConn() == nullptr);
76
77 ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
78 assert (esiStream.getRaw() != nullptr);
79 /* If segments become more flexible, ignore thisNode */
80 assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
81 assert (!esiStream->finished);
82
83 debugs (86,5, "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
84 assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
85
86 /* trivial case */
87
88 if (http->out.offset != 0) {
89 assert(rep == nullptr);
90 } else {
91 if (rep) {
92 if (rep->sline.status() != Http::scOkay) {
93 rep = nullptr;
94 esiStream->include->includeFail (esiStream);
95 esiStream->finished = 1;
96 httpRequestFree (http);
97 return;
98 }
99
100 rep = nullptr;
101 }
102 }
103
104 if (receivedData.data && receivedData.length) {
105 http->out.offset += receivedData.length;
106
107 if (receivedData.data >= esiStream->localbuffer->buf &&
108 receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
109 /* original static buffer */
110
111 if (receivedData.data != esiStream->localbuffer->buf) {
112 /* But not the start of it */
113 memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
114 }
115
116 esiStream->localbuffer->len = receivedData.length;
117 } else {
118 assert (esiStream->buffer.getRaw() != nullptr);
119 esiStream->buffer->len = receivedData.length;
120 }
121 }
122
123 /* EOF / Read error / aborted entry */
124 if (rep == nullptr && receivedData.data == nullptr && receivedData.length == 0) {
125 /* TODO: get stream status to test the entry for aborts */
126 debugs(86, 5, "Finished reading upstream data in subrequest");
127 esiStream->include->subRequestDone (esiStream, true);
128 esiStream->finished = 1;
129 httpRequestFree (http);
130 return;
131 }
132
133 switch (clientStreamStatus (node, http)) {
134
136 case STREAM_COMPLETE: /* ok */
137 debugs(86, 3, "ESI subrequest finished OK");
138 esiStream->include->subRequestDone (esiStream, true);
139 esiStream->finished = 1;
140 httpRequestFree (http);
141 return;
142
143 case STREAM_FAILED:
144 debugs(86, DBG_IMPORTANT, "ERROR: ESI subrequest failed transfer");
145 esiStream->include->includeFail (esiStream);
146 esiStream->finished = 1;
147 httpRequestFree (http);
148 return;
149
150 case STREAM_NONE: {
151 StoreIOBuffer tempBuffer;
152
153 if (!esiStream->buffer.getRaw()) {
154 esiStream->buffer = esiStream->localbuffer;
155 }
156
157 esiStream->buffer = esiStream->buffer->tail();
158
159 if (esiStream->buffer->len) {
160 esiStream->buffer->next = new ESISegment;
161 esiStream->buffer = esiStream->buffer->next;
162 }
163
164 tempBuffer.offset = http->out.offset;
165 tempBuffer.length = sizeof (esiStream->buffer->buf);
166 tempBuffer.data = esiStream->buffer->buf;
167 /* now just read into 'buffer' */
168 clientStreamRead (node, http, tempBuffer);
169 debugs(86, 5, "Requested more data for ESI subrequest");
170 }
171
172 break;
173
174 default:
175 fatal ("Hit unreachable code in esiBufferRecipient\n");
176 }
177
178}
179
180/* esiStream functions */
182{
184}
185
186void
188{
189 debugs(86, 5, "Freeing stream context resources.");
190 buffer = nullptr;
191 localbuffer = nullptr;
192 include = nullptr;
193}
194
197{
199 rv->include = include;
200 return rv;
201}
202
203/* ESIInclude */
205{
206 debugs(86, 5, "ESIInclude::Free " << this);
212}
213
214void
216{
217 parent = nullptr;
218}
219
222{
223 return new ESIInclude (*this);
224}
225
228{
229 ESIInclude *resultI = new ESIInclude (*this);
230 ESIElement::Pointer result = resultI;
231 resultI->parent = newParent;
232 resultI->varState = cbdataReference (&newVarState);
233
234 if (resultI->srcurl)
235 resultI->src = ESIStreamContextNew (resultI);
236
237 if (resultI->alturl)
238 resultI->alt = ESIStreamContextNew (resultI);
239
240 return result;
241}
242
244 varState(nullptr),
245 srcurl(nullptr),
246 alturl(nullptr),
247 parent(nullptr),
248 started(false),
249 sent(false)
250{
251 memset(&flags, 0, sizeof(flags));
252 flags.onerrorcontinue = old.flags.onerrorcontinue;
253
254 if (old.srcurl)
255 srcurl = xstrdup(old.srcurl);
256
257 if (old.alturl)
258 alturl = xstrdup(old.alturl);
259}
260
261void
263{
264 tempheaders.update(&vars->header());
265 tempheaders.removeHopByHopEntries();
266}
267
268void
270{
271 if (!stream.getRaw())
272 return;
273
274 HttpHeader tempheaders(hoRequest);
275
276 prepareRequestHeaders(tempheaders, vars);
277
278 /* Ensure variable state is clean */
279 vars->feedData(url, strlen (url));
280
281 /* tempUrl is eaten by the request */
282 char const *tempUrl = vars->extractChar ();
283
284 debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
285 const auto mx = MasterXaction::MakePortless<XactionInitiator::initEsi>();
286 if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ, mx)) {
287 debugs(86, DBG_CRITICAL, "ERROR: starting new ESI subrequest failed");
288 }
289
290 tempheaders.clean();
291}
292
293ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
294 varState(nullptr),
295 srcurl(nullptr),
296 alturl(nullptr),
297 parent(aParent),
298 started(false),
299 sent(false)
300{
301 assert (aContext);
302 memset(&flags, 0, sizeof(flags));
303
304 for (int i = 0; i < attrcount && attr[i]; i += 2) {
305 if (!strcmp(attr[i],"src")) {
306 /* Start a request for thisNode url */
307 debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
308
309 /* TODO: don't assert on thisNode, ignore the duplicate */
310 assert (src.getRaw() == nullptr);
311 src = ESIStreamContextNew (this);
312 assert (src.getRaw() != nullptr);
313 srcurl = xstrdup(attr[i+1]);
314 } else if (!strcmp(attr[i],"alt")) {
315 /* Start a secondary request for thisNode url */
316 /* TODO: make a config parameter to wait on requesting alt's
317 * for the src to fail
318 */
319 debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
320
321 assert (alt.getRaw() == nullptr); /* TODO: fix? */
322 alt = ESIStreamContextNew (this);
323 assert (alt.getRaw() != nullptr);
324 alturl = xstrdup(attr[i+1]);
325 } else if (!strcmp(attr[i],"onerror")) {
326 if (!strcmp(attr[i+1], "continue")) {
327 flags.onerrorcontinue = 1;
328 } else {
329 /* ignore mistyped attributes */
330 debugs(86, DBG_IMPORTANT, "ERROR: invalid value for onerror='" << attr[i+1] << "'");
331 }
332 } else {
333 /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
334 */
335 }
336 }
337
338 varState = cbdataReference(aContext->varState);
339}
340
341void
343{
344 /* prevent freeing ourselves */
345 ESIIncludePtr foo(this);
346
347 if (started)
348 return;
349
350 started = true;
351
352 if (src.getRaw()) {
355 } else {
356 alt = nullptr;
357
358 debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
359
360 flags.failed = 1;
361 }
362}
363
364void
366{
367 if (sent)
368 return;
369
371
372 debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
373
374 assert (flags.finished || (flags.failed && flags.onerrorcontinue));
375
376 if (flags.failed && flags.onerrorcontinue) {
377 return;
378 }
379
380 /* Render the content */
381 if (srccontent.getRaw()) {
382 myout = srccontent;
383 srccontent = nullptr;
384 } else if (altcontent.getRaw()) {
385 myout = altcontent;
386 altcontent = nullptr;
387 } else
388 fatal ("ESIIncludeRender called with no content, and no failure!\n");
389
390 assert (output->next == nullptr);
391
392 output->next = myout;
393
394 sent = true;
395}
396
399{
400 /* Prevent refcount race leading to free */
401 Pointer me (this);
402 start();
403 debugs(86, 5, "ESIIncludeRender: Processing include " << this);
404
405 if (flags.failed) {
406 if (flags.onerrorcontinue)
408 else
409 return ESI_PROCESS_FAILED;
410 }
411
412 if (!flags.finished) {
413 if (flags.onerrorcontinue)
415 else
417 }
418
420}
421
422void
424{
425 subRequestDone (stream, false);
426}
427
428bool
430{
431 return !(flags.finished || flags.failed);
432}
433
434void
436{
437 if (!dataNeeded())
438 return;
439
440 if (stream == src) {
441 debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
442
443 if (success) {
444 /* copy the lead segment */
445 debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
448 /* we're done! */
449 flags.finished = 1;
450 } else {
451 /* Fail if there is no alt being retrieved */
452 debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
453
454 if (!(alt.getRaw() || altcontent.getRaw())) {
455 debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
456 flags.failed = 1;
457 } else if (altcontent.getRaw()) {
458 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
459 /* ALT was already retrieved, we are done */
460 flags.finished = 1;
461 }
462 }
463
464 src = nullptr;
465 } else if (stream == alt) {
466 debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
467
468 if (success) {
469 debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
470 /* copy the lead segment */
473 /* we're done! */
474
475 if (!(src.getRaw() || srccontent.getRaw())) {
476 /* src already failed, kick ESI processor */
477 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
478 flags.finished = 1;
479 }
480 } else {
481 if (!(src.getRaw() || srccontent.getRaw())) {
482 debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
483 /* src already failed */
484 flags.failed = 1;
485 }
486 }
487
488 alt = nullptr;
489 } else {
490 fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
491 }
492
493 if (flags.finished || flags.failed) {
494 /* Kick ESI Processor */
495 debugs (86, 5, "ESIInclude " << this <<
496 " SubRequest " << stream.getRaw() <<
497 " completed, kicking processor , status " <<
498 (flags.finished ? "OK" : "FAILED"));
499 /* There is a race condition - and we have no reproducible test case -
500 * during a subrequest the parent will get set to NULL, which is not
501 * meant to be possible. Rather than killing squid, we let it leak
502 * memory but complain in the log.
503 *
504 * Someone wanting to debug this could well start by running squid with
505 * a hardware breakpoint set to this location.
506 * Its probably due to parent being set to null - by a call to
507 * 'this.finish' while the subrequest is still not completed.
508 */
509 if (parent.getRaw() == nullptr) {
510 debugs(86, DBG_CRITICAL, "ERROR: Squid Bug #951: ESIInclude::subRequestDone: Sub request completed "
511 "after finish() called and parent unlinked. Unable to "
512 "continue handling the request, and may be memory leaking. "
513 "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
514 "are looking for a reproducible test case. This will require "
515 "an ESI template with includes, probably with alt-options, "
516 "and we're likely to need traffic dumps to allow us to "
517 "reconstruct the exact tcp handling sequences to trigger this "
518 "rather elusive bug.");
519 return;
520 }
521 assert (parent.getRaw());
522
523 if (!flags.failed) {
524 sent = true;
526
527 if (srccontent.getRaw())
528 srccontent = nullptr;
529 else
530 altcontent = nullptr;
531 } else if (flags.onerrorcontinue) {
532 /* render nothing but inform of completion */
533
534 if (!sent) {
535 sent = true;
536 parent->provideData (new ESISegment, this);
537 } else
538 assert (0);
539 } else
540 parent->fail(this, "esi:include could not be completed.");
541 }
542}
543
544#endif /* USE_SQUID_ESI */
545
esiProcessResult_t
Definition: Element.h:18
@ ESI_PROCESS_FAILED
Definition: Element.h:22
@ ESI_PROCESS_PENDING_WONTFAIL
Definition: Element.h:20
@ ESI_PROCESS_COMPLETE
Definition: Element.h:19
@ ESI_PROCESS_PENDING_MAYFAIL
Definition: Element.h:21
@ hoRequest
Definition: HttpHeader.h:36
static CSCB esiBufferRecipient
Definition: Include.cc:27
static ESIStreamContext * ESIStreamContextNew(ESIIncludePtr)
Definition: Include.cc:196
CBDATA_CLASS_INIT(ESIStreamContext)
static CSD esiBufferDetach
Definition: Include.cc:28
#define assert(EX)
Definition: assert.h:17
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:265
#define cbdataReferenceDone(var)
Definition: cbdata.h:352
#define cbdataReference(var)
Definition: cbdata.h:343
struct ClientHttpRequest::Out out
ConnStateData * getConn() const
ESIVarState * varState
Definition: Context.h:134
void prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
Definition: Include.cc:262
void includeFail(ESIStreamContext::Pointer)
Definition: Include.cc:423
struct ESIInclude::@59 flags
~ESIInclude() override
Definition: Include.cc:204
void render(ESISegment::Pointer) override
Definition: Include.cc:365
ESISegment::Pointer srccontent
Definition: Include.h:57
ESIVarState * varState
Definition: Include.h:59
ESISegment::Pointer altcontent
Definition: Include.h:58
Pointer makeCacheable() const override
Definition: Include.cc:221
esiTreeParentPtr parent
Definition: Include.h:66
void finish() override
Definition: Include.cc:215
ESIStreamContext::Pointer alt
Definition: Include.h:56
bool started
Definition: Include.h:68
void subRequestDone(ESIStreamContext::Pointer, bool)
Definition: Include.cc:435
char * alturl
Definition: Include.h:60
bool dataNeeded() const
Definition: Include.cc:429
ESIStreamContext::Pointer src
Definition: Include.h:55
void Start(ESIStreamContext::Pointer, char const *, ESIVarState *)
Definition: Include.cc:269
bool sent
Definition: Include.h:69
esiProcessResult_t process(int dovars) override
Definition: Include.cc:398
unsigned int onerrorcontinue
Definition: Include.h:51
ESIInclude(esiTreeParentPtr, int attributes, const char **attr, ESIContext *)
Definition: Include.cc:293
char * srcurl
Definition: Include.h:60
Pointer makeUsable(esiTreeParentPtr, ESIVarState &) const override
Definition: Include.cc:227
void start()
Definition: Include.cc:342
static void ListTransfer(Pointer &from, Pointer &to)
Definition: Segment.cc:53
size_t len
Definition: Segment.h:41
ESISegment const * tail() const
Definition: Segment.cc:153
Pointer next
Definition: Segment.h:42
char buf[HTTP_REQBUF_SZ]
Definition: Segment.h:40
ESISegment::Pointer buffer
Definition: Include.h:34
ESISegment::Pointer localbuffer
Definition: Include.h:33
ESIIncludePtr include
Definition: Include.h:32
void freeResources()
Definition: Include.cc:187
~ESIStreamContext() override
Definition: Include.cc:181
HttpHeader & header()
Definition: VarState.cc:70
char * extractChar()
Definition: VarState.cc:117
void feedData(const char *buf, size_t len)
Definition: VarState.cc:99
void removeHopByHopEntries()
Definition: HttpHeader.cc:1711
void update(const HttpHeader *fresh)
Definition: HttpHeader.cc:268
void clean()
Definition: HttpHeader.cc:186
Http::StatusLine sline
Definition: HttpReply.h:56
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
C * getRaw() const
Definition: RefCount.h:89
int64_t offset
Definition: StoreIOBuffer.h:58
void CSD(clientStreamNode *, ClientHttpRequest *)
client stream detach
void CSCB(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
client stream read callback
void httpRequestFree(void *data)
Definition: client_side.cc:499
int clientBeginRequest(const HttpRequestMethod &method, char const *url, CSCB *streamcallback, CSD *streamdetach, ClientStreamData streamdata, HttpHeader const *header, char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
#define DBG_IMPORTANT
Definition: Stream.h:38
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:194
#define DBG_CRITICAL
Definition: Stream.h:37
@ STREAM_COMPLETE
Definition: enums.h:127
@ STREAM_UNPLANNED_COMPLETE
Definition: enums.h:132
@ STREAM_NONE
Definition: enums.h:126
@ STREAM_FAILED
Definition: enums.h:137
void ESISegmentFreeList(ESISegment::Pointer &head)
Definition: Segment.cc:19
void fatal(const char *message)
Definition: fatal.cc:28
clientStream_status_t clientStreamStatus(clientStreamNode *thisObject, ClientHttpRequest *http)
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
#define HTTP_REQBUF_SZ
Definition: forward.h:14
@ scOkay
Definition: StatusCode.h:26
@ METHOD_GET
Definition: MethodType.h:25
#define xstrdup
virtual void fail(ESIElement *, char const *=nullptr)
Definition: Element.h:33
virtual void provideData(ESISegment::Pointer, ESIElement *)
Definition: Element.h:28
Definition: parse.c:104
struct node * next
Definition: parse.c:105
#define safe_free(x)
Definition: xalloc.h:73

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors