redirect.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 61 Redirector */
10 
11 #include "squid.h"
12 #include "acl/Checklist.h"
13 #include "cache_cf.h"
14 #include "client_side.h"
15 #include "client_side_reply.h"
16 #include "client_side_request.h"
17 #include "comm/Connection.h"
18 #include "fde.h"
19 #include "format/Format.h"
20 #include "globals.h"
21 #include "helper.h"
22 #include "helper/Reply.h"
23 #include "http/Stream.h"
24 #include "HttpRequest.h"
25 #include "mgr/Registration.h"
26 #include "redirect.h"
27 #include "rfc1738.h"
28 #include "sbuf/SBuf.h"
29 #include "SquidConfig.h"
30 #include "Store.h"
31 #if USE_AUTH
32 #include "auth/UserRequest.h"
33 #endif
34 #if USE_OPENSSL
35 #include "ssl/support.h"
36 #endif
37 
39 #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024)
40 
42 {
44 
45 public:
46  explicit RedirectStateData(const char *url);
48 
49  void *data;
51 
53 };
54 
57 static helper *redirectors = NULL;
58 static helper *storeIds = NULL;
61 static int redirectorBypassed = 0;
62 static int storeIdBypassed = 0;
63 static Format::Format *redirectorExtrasFmt = NULL;
64 static Format::Format *storeIdExtrasFmt = NULL;
65 
67 
69  data(NULL),
70  orig_url(url),
71  handler(NULL)
72 {
73 }
74 
76 {
77 }
78 
79 static void
81 {
82  RedirectStateData *r = static_cast<RedirectStateData *>(data);
83  debugs(61, 5, HERE << "reply=" << reply);
84 
85  // XXX: This function is now kept only to check for and display the garbage use-case
86  // and to map the old helper response format(s) into new format result code and key=value pairs
87  // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
88 
89  if (reply.result == Helper::Unknown) {
90  // BACKWARD COMPATIBILITY 2012-06-15:
91  // Some nasty old helpers send back the entire input line including extra format keys.
92  // This is especially bad for simple perl search-replace filter scripts.
93  //
94  // * trim all but the first word off the response.
95  // * warn once every 50 responses that this will stop being fixed-up soon.
96  //
97  if (reply.other().hasContent()) {
98  const char * res = reply.other().content();
99  size_t replySize = 0;
100  if (const char *t = strchr(res, ' ')) {
101  static int warn = 0;
102  debugs(61, (!(warn++%50)? DBG_CRITICAL:2), "UPGRADE WARNING: URL rewriter reponded with garbage '" << t <<
103  "'. Future Squid will treat this as part of the URL.");
104  replySize = t - res;
105  } else
106  replySize = reply.other().contentSize();
107 
108  // if we still have anything in other() after all that
109  // parse it into status=, url= and rewrite-url= keys
110  if (replySize) {
111  MemBuf replyBuffer;
112  replyBuffer.init(replySize, replySize);
113  replyBuffer.append(reply.other().content(), reply.other().contentSize());
114  char * result = replyBuffer.content();
115 
116  Helper::Reply newReply;
117  // BACKWARD COMPATIBILITY 2012-06-15:
118  // We got Helper::Unknown reply result but new
119  // RedirectStateData handlers require Helper::Okay,
120  // else will drop the helper reply
121  newReply.result = Helper::Okay;
122  newReply.notes.append(&reply.notes);
123 
124  // check and parse for obsoleted Squid-2 urlgroup feature
125  if (*result == '!') {
126  static int urlgroupWarning = 0;
127  if (!urlgroupWarning++)
128  debugs(85, DBG_IMPORTANT, "UPGRADE WARNING: URL rewriter using obsolete Squid-2 urlgroup feature needs updating.");
129  if (char *t = strchr(result+1, '!')) {
130  *t = '\0';
131  newReply.notes.add("urlgroup", result+1);
132  result = t + 1;
133  }
134  }
135 
136  const Http::StatusCode status = static_cast<Http::StatusCode>(atoi(result));
137 
138  if (status == Http::scMovedPermanently
139  || status == Http::scFound
140  || status == Http::scSeeOther
141  || status == Http::scPermanentRedirect
142  || status == Http::scTemporaryRedirect) {
143 
144  if (const char *t = strchr(result, ':')) {
145  char statusBuf[4];
146  snprintf(statusBuf, sizeof(statusBuf),"%3u",status);
147  newReply.notes.add("status", statusBuf);
148  ++t;
149  // TODO: validate the URL produced here is RFC 2616 compliant URI
150  newReply.notes.add("url", t);
151  } else {
152  debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << result);
153  }
154  } else {
155  // status code is not a redirect code (or does not exist)
156  // treat as a re-write URL request
157  // TODO: validate the URL produced here is RFC 2616 compliant URI
158  if (*result)
159  newReply.notes.add("rewrite-url", result);
160  }
161 
162  void *cbdata;
163  if (cbdataReferenceValidDone(r->data, &cbdata))
164  r->handler(cbdata, newReply);
165 
166  delete r;
167  return;
168  }
169  }
170  }
171 
172  void *cbdata;
173  if (cbdataReferenceValidDone(r->data, &cbdata))
174  r->handler(cbdata, reply);
175 
176  delete r;
177 }
178 
179 static void
181 {
182  RedirectStateData *r = static_cast<RedirectStateData *>(data);
183  debugs(61, 5,"StoreId helper: reply=" << reply);
184 
185  // XXX: This function is now kept only to check for and display the garbage use-case
186  // and to map the old helper response format(s) into new format result code and key=value pairs
187  // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format
188  void *cbdata;
189  if (cbdataReferenceValidDone(r->data, &cbdata))
190  r->handler(cbdata, reply);
191 
192  delete r;
193 }
194 
195 static void
197 {
198  if (redirectors == NULL) {
199  storeAppendPrintf(sentry, "No redirectors defined\n");
200  return;
201  }
202 
203  redirectors->packStatsInto(sentry, "Redirector Statistics");
204 
206  storeAppendPrintf(sentry, "\nNumber of requests bypassed "
207  "because all redirectors were busy: %d\n", redirectorBypassed);
208 }
209 
210 static void
212 {
213  if (storeIds == NULL) {
214  storeAppendPrintf(sentry, "No StoreId helpers defined\n");
215  return;
216  }
217 
218  storeIds->packStatsInto(sentry, "StoreId helper Statistics");
219 
221  storeAppendPrintf(sentry, "\nNumber of requests bypassed "
222  "because all StoreId helpers were busy: %d\n", storeIdBypassed);
223 }
224 
225 static void
226 constructHelperQuery(const char *name, helper *hlp, HLPCB *replyHandler, ClientHttpRequest * http, HLPCB *handler, void *data, Format::Format *requestExtrasFmt)
227 {
229  int sz;
230  Http::StatusCode status;
231 
235  RedirectStateData *r = new RedirectStateData(http->uri);
236  r->handler = handler;
237  r->data = cbdataReference(data);
238 
239  static MemBuf requestExtras;
240  requestExtras.reset();
241  if (requestExtrasFmt)
242  requestExtrasFmt->assemble(requestExtras, http->al, 0);
243 
244  sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s%s%s\n",
245  r->orig_url.c_str(),
246  requestExtras.hasContent() ? " " : "",
247  requestExtras.hasContent() ? requestExtras.content() : "");
248 
249  if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) {
250  if (sz<=0) {
252  debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to " << name << ". Request ABORTED.");
253  } else {
254  status = Http::scUriTooLong;
255  debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to " << name << " exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED.");
256  }
257 
259  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
260  assert (repContext);
261  Ip::Address tmpnoaddr;
262  tmpnoaddr.setNoAddr();
263  repContext->setReplyToError(ERR_GATEWAY_FAILURE, status,
264  http->request->method, NULL,
265  http->getConn() != NULL && http->getConn()->clientConnection != NULL ?
266  http->getConn()->clientConnection->remote : tmpnoaddr,
267  http->request,
268  NULL,
269 #if USE_AUTH
270  http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
271  http->getConn()->getAuth() : http->request->auth_user_request);
272 #else
273  NULL);
274 #endif
275 
276  node = (clientStreamNode *)http->client_stream.tail->data;
277  clientStreamRead(node, http, node->readBuffer);
278  return;
279  }
280 
281  debugs(61,6, HERE << "sending '" << buf << "' to the " << name << " helper");
282  helperSubmit(hlp, buf, replyHandler, r);
283 }
284 
285 /**** PUBLIC FUNCTIONS ****/
286 
287 void
289 {
290  assert(http);
291  assert(handler);
292  debugs(61, 5, "redirectStart: '" << http->uri << "'");
293 
294  // TODO: Deprecate Config.onoff.redirector_bypass in favor of either
295  // onPersistentOverload or a new onOverload option that applies to all helpers.
296  if (Config.onoff.redirector_bypass && redirectors->willOverload()) {
297  /* Skip redirector if the queue is full */
299  Helper::Reply bypassReply;
300  bypassReply.result = Helper::Okay;
301  bypassReply.notes.add("message","URL rewrite/redirect queue too long. Bypassed.");
302  handler(data, bypassReply);
303  return;
304  }
305 
306  constructHelperQuery("redirector", redirectors, redirectHandleReply, http, handler, data, redirectorExtrasFmt);
307 }
308 
313 void
315 {
316  assert(http);
317  assert(handler);
318  debugs(61, 5, "storeIdStart: '" << http->uri << "'");
319 
320  if (Config.onoff.store_id_bypass && storeIds->willOverload()) {
321  /* Skip StoreID Helper if the queue is full */
322  ++storeIdBypassed;
323  Helper::Reply bypassReply;
324 
325  bypassReply.result = Helper::Okay;
326 
327  bypassReply.notes.add("message","StoreId helper queue too long. Bypassed.");
328  handler(data, bypassReply);
329  return;
330  }
331 
332  constructHelperQuery("storeId helper", storeIds, storeIdHandleReply, http, handler, data, storeIdExtrasFmt);
333 }
334 
335 void
337 {
338  static bool init = false;
339 
340  if (!init) {
341  Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1);
342  Mgr::RegisterAction("store_id", "StoreId helper Stats", storeIdStats, 0, 1);
343  }
344 
345  if (Config.Program.redirect) {
346 
347  if (redirectors == NULL)
348  redirectors = new helper("redirector");
349 
350  redirectors->cmdline = Config.Program.redirect;
351 
352  // BACKWARD COMPATIBILITY:
353  // if redirectot_bypass is set then use queue_size=0 as default size
356 
358 
359  redirectors->ipc_type = IPC_STREAM;
360 
361  redirectors->timeout = Config.Timeout.urlRewrite;
362 
364  redirectors->retryBrokenHelper = true; // XXX: make this configurable ?
365  redirectors->onTimedOutResponse.clear();
368 
369  helperOpenServers(redirectors);
370  }
371 
372  if (Config.Program.store_id) {
373 
374  if (storeIds == NULL)
375  storeIds = new helper("store_id");
376 
377  storeIds->cmdline = Config.Program.store_id;
378 
379  // BACKWARD COMPATIBILITY:
380  // if store_id_bypass is set then use queue_size=0 as default size
383 
385 
386  storeIds->ipc_type = IPC_STREAM;
387 
388  storeIds->retryBrokenHelper = true; // XXX: make this configurable ?
389 
390  helperOpenServers(storeIds);
391  }
392 
394  delete redirectorExtrasFmt;
395  redirectorExtrasFmt = new ::Format::Format("url_rewrite_extras");
396  (void)redirectorExtrasFmt->parse(Config.redirector_extras);
397  }
398 
399  if (Config.storeId_extras) {
400  delete storeIdExtrasFmt;
401  storeIdExtrasFmt = new ::Format::Format("store_id_extras");
402  (void)storeIdExtrasFmt->parse(Config.storeId_extras);
403  }
404 
405  init = true;
406 }
407 
408 void
410 {
415  if (redirectors)
416  helperShutdown(redirectors);
417 
418  if (storeIds)
419  helperShutdown(storeIds);
420 
421  if (!shutting_down)
422  return;
423 
424  delete redirectors;
425  redirectors = NULL;
426 
427  delete storeIds;
428  storeIds = NULL;
429 
430  delete redirectorExtrasFmt;
431  redirectorExtrasFmt = NULL;
432 
433  delete storeIdExtrasFmt;
434  storeIdExtrasFmt = NULL;
435 }
436 
void redirectStart(ClientHttpRequest *http, HLPCB *handler, void *data)
Definition: redirect.cc:288
#define cbdataReferenceValidDone(var, ptr)
Definition: cbdata.h:256
static int redirectorBypassed
Definition: redirect.cc:61
#define assert(EX)
Definition: assert.h:17
static OBJH redirectStats
Definition: redirect.cc:59
SBuf & assign(const SBuf &S)
Definition: SBuf.cc:94
int redirector_bypass
Definition: SquidConfig.h:299
ConnStateData * getConn() const
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:225
static OBJH storeIdStats
Definition: redirect.cc:60
wordlist * store_id
Definition: SquidConfig.h:191
void packStatsInto(Packable *p, const char *label=NULL) const
Dump some stats about the helper state to a Packable object.
Definition: helper.cc:553
Definition: cbdata.cc:60
Definition: SBuf.h:87
HttpRequestMethod method
Definition: HttpRequest.h:102
void add(const SBuf &key, const SBuf &value)
Definition: Notes.cc:278
Helper::ChildConfig redirectChildren
Definition: SquidConfig.h:205
int store_id_bypass
Definition: SquidConfig.h:300
Definition: helper.h:60
Helper::ChildConfig childs
Configuration settings for number running.
Definition: helper.h:102
static helper * storeIds
Definition: redirect.cc:58
static helper * redirectors
Definition: redirect.cc:57
wordlist * cmdline
Definition: helper.h:98
void redirectInit(void)
Definition: redirect.cc:336
void helperShutdown(helper *hlp)
Definition: helper.cc:616
void redirectShutdown(void)
Definition: redirect.cc:409
void clear()
Definition: SBuf.cc:190
int ipc_type
Definition: helper.h:103
HLPCB * handler
Definition: redirect.cc:52
#define DBG_CRITICAL
Definition: Debug.h:44
void OBJH(StoreEntry *)
Definition: forward.h:44
wordlist * redirect
Definition: SquidConfig.h:190
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
Helper::ResultCode result
The helper response 'result' field.
Definition: Reply.h:58
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:105
time_t urlRewrite
Definition: SquidConfig.h:122
void helperSubmit(helper *hlp, const char *buf, HLPCB *callback, void *data)
Definition: helper.cc:397
void setNoAddr()
Definition: Address.cc:285
int shutting_down
Definition: testAddress.cc:36
void HLPCB(void *, const Helper::Reply &)
Definition: forward.h:27
static void constructHelperQuery(const char *name, helper *hlp, HLPCB *replyHandler, ClientHttpRequest *http, HLPCB *handler, void *data, Format::Format *requestExtrasFmt)
Definition: redirect.cc:226
void const char HLPCB void * data
Definition: stub_helper.cc:16
NotePairs notes
Definition: Reply.h:61
Definition: parse.c:104
StatusCode
Definition: StatusCode.h:20
bool hasContent() const
Definition: MemBuf.h:54
bool retryTimedOut
Whether the timed-out requests must retried.
Definition: helper.h:110
struct SquidConfig::UrlHelperTimeout onUrlRewriteTimeout
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
#define cbdataReference(var)
Definition: cbdata.h:341
static Format::Format * redirectorExtrasFmt
Definition: redirect.cc:63
#define DBG_IMPORTANT
Definition: Debug.h:45
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
ChildConfig & updateLimits(const ChildConfig &rhs)
Definition: ChildConfig.cc:44
void reset()
Definition: MemBuf.cc:141
#define IPC_STREAM
Definition: defines.h:161
#define CBDATA_CLASS(type)
Definition: cbdata.h:302
static Format::Format * storeIdExtrasFmt
Definition: redirect.cc:64
const char * c_str()
Definition: SBuf.cc:546
void warn(char *format,...)
ClientStreamData data
Definition: clientStream.h:94
AccessLogEntry::Pointer al
access.log entry
void helperOpenServers(helper *hlp)
Definition: helper.cc:125
static void handler(int signo)
Definition: purge.cc:860
char * content()
start of the added data
Definition: MemBuf.h:41
SBuf onTimedOutResponse
The response to use when helper response timedout.
Definition: helper.h:112
void const char * buf
Definition: stub_helper.cc:16
bool parse(const char *def)
Definition: Format.cc:65
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:147
struct SquidConfig::@112 onoff
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
bool retryBrokenHelper
Whether the requests must retried on BH replies.
Definition: helper.h:111
char * redirector_extras
Definition: SquidConfig.h:530
struct SquidConfig::@104 Program
static HLPCB storeIdHandleReply
Definition: redirect.cc:56
const MemBuf & other() const
Definition: Reply.h:41
Ip::Address remote
Definition: Connection.h:138
StoreIOBuffer readBuffer
Definition: clientStream.h:95
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
void assemble(MemBuf &mb, const AccessLogEntryPointer &al, int logSequenceNumber) const
assemble the state information into a formatted line.
Definition: Format.cc:367
Helper::ChildConfig storeIdChildren
Definition: SquidConfig.h:206
time_t timeout
Requests timeout.
Definition: helper.h:109
Definition: MemBuf.h:23
void storeIdStart(ClientHttpRequest *http, HLPCB *handler, void *data)
Definition: redirect.cc:314
Comm::ConnectionPointer clientConnection
Definition: Server.h:97
Auth::UserRequest::Pointer auth_user_request
Definition: HttpRequest.h:115
char * storeId_extras
Definition: SquidConfig.h:537
struct SquidConfig::@98 Timeout
static int storeIdBypassed
Definition: redirect.cc:62
#define MAX_REDIRECTOR_REQUEST_STRLEN
url maximum lengh + extra informations passed to redirector
Definition: redirect.cc:39
static HLPCB redirectHandleReply
Definition: redirect.cc:55
unsigned int queue_size
Definition: ChildConfig.h:91
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:904
RedirectStateData(const char *url)
Definition: redirect.cc:68
C * getRaw() const
Definition: RefCount.h:74
class SquidConfig Config
Definition: SquidConfig.cc:12
void append(const NotePairs *src)
Append the entries of the src NotePairs list to our list.
Definition: Notes.cc:340
#define NULL
Definition: types.h:166
bool willOverload() const
Definition: helper.cc:611
const Auth::UserRequest::Pointer & getAuth() const
Definition: client_side.h:114

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors