urn.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 52 URN Parsing */
10 
11 #include "squid.h"
12 #include "AccessLogEntry.h"
13 #include "acl/FilledChecklist.h"
14 #include "base/TextException.h"
15 #include "cbdata.h"
16 #include "errorpage.h"
17 #include "FwdState.h"
18 #include "globals.h"
19 #include "HttpReply.h"
20 #include "HttpRequest.h"
21 #include "icmp/net_db.h"
22 #include "MemBuf.h"
23 #include "mime_header.h"
24 #include "RequestFlags.h"
25 #include "SquidTime.h"
26 #include "Store.h"
27 #include "StoreClient.h"
28 #include "tools.h"
29 #include "urn.h"
30 
31 #define URN_REQBUF_SZ 4096
32 
33 class UrnState : public StoreClient
34 {
36 
37 public:
38  explicit UrnState(const AccessLogEntry::Pointer &anAle): ale(anAle) {}
39 
40  void start (HttpRequest *, StoreEntry *);
42 
43  virtual ~UrnState();
44 
45  StoreEntry *entry = nullptr;
46  store_client *sc = nullptr;
47  StoreEntry *urlres_e = nullptr;
51 
52  char reqbuf[URN_REQBUF_SZ] = { '\0' };
53  int reqofs = 0;
54 
55 private:
56  /* StoreClient API */
57  virtual LogTags *loggingTags() const { return ale ? &ale->cache.code : nullptr; }
58  virtual void fillChecklist(ACLFilledChecklist &) const;
59 
60  char *urlres = nullptr;
61 };
62 
63 typedef struct {
64  char *url;
65  char *host;
66  int rtt;
67 
68  struct {
69  int cached;
70  } flags;
71 } url_entry;
72 
74 static url_entry *urnParseReply(const char *inbuf, const HttpRequestMethod&);
75 static const char *const crlf = "\r\n";
76 
78 
80 {
82  if (urlres_e) {
83  if (sc)
84  storeUnregister(sc, urlres_e, this);
85  urlres_e->unlock("~UrnState+res");
86  }
87 
88  if (entry)
89  entry->unlock("~UrnState+prime");
90 
92  });
93 }
94 
95 static url_entry *
96 urnFindMinRtt(url_entry * urls, const HttpRequestMethod &, int *rtt_ret)
97 {
98  int min_rtt = 0;
99  url_entry *u = NULL;
100  url_entry *min_u = NULL;
101  int i;
102  int urlcnt = 0;
103  debugs(52, 3, "urnFindMinRtt");
104  assert(urls != NULL);
105 
106  for (i = 0; NULL != urls[i].url; ++i)
107  ++urlcnt;
108 
109  debugs(53, 3, "urnFindMinRtt: Counted " << i << " URLs");
110 
111  if (1 == urlcnt) {
112  debugs(52, 3, "urnFindMinRtt: Only one URL - return it!");
113  return urls;
114  }
115 
116  for (i = 0; i < urlcnt; ++i) {
117  u = &urls[i];
118  debugs(52, 3, "urnFindMinRtt: " << u->host << " rtt=" << u->rtt);
119 
120  if (u->rtt == 0)
121  continue;
122 
123  if (u->rtt > min_rtt && min_rtt != 0)
124  continue;
125 
126  min_rtt = u->rtt;
127 
128  min_u = u;
129  }
130 
131  if (rtt_ret)
132  *rtt_ret = min_rtt;
133 
134  debugs(52, DBG_IMPORTANT, "urnFindMinRtt: Returning '" <<
135  (min_u ? min_u->url : "NONE") << "' RTT " <<
136  min_rtt );
137 
138  return min_u;
139 }
140 
141 void
143 {
144  const auto &query = r->url.absolute();
145  const auto host = r->url.host();
146  // TODO: use class AnyP::Uri instead of generating a string and re-parsing
147  LOCAL_ARRAY(char, local_urlres, 4096);
148  snprintf(local_urlres, 4096, "http://%s/uri-res/N2L?" SQUIDSBUFPH, host, SQUIDSBUFPRINT(query));
149  safe_free(urlres);
150  urlres_r = HttpRequest::FromUrlXXX(local_urlres, r->masterXaction);
151 
152  if (!urlres_r) {
153  debugs(52, 3, "Bad uri-res URL " << local_urlres);
154  const auto err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, r, ale);
155  err->url = xstrdup(local_urlres);
156  errorAppendEntry(entry, err);
157  return;
158  }
159 
160  urlres = xstrdup(local_urlres);
162 }
163 
164 void
166 {
167  debugs(52, 3, "urnStart: '" << e->url() << "'" );
168  entry = e;
169  request = r;
170 
171  entry->lock("UrnState::start");
173 
174  if (urlres_r == NULL)
175  return;
176 
177  auto urlEntry = storeGetPublic(urlres, Http::METHOD_GET);
178 
179  if (!urlEntry || (urlEntry->hittingRequiresCollapsing() && !startCollapsingOn(*urlEntry, false))) {
181  sc = storeClientListAdd(urlres_e, this);
183  if (urlEntry) {
184  urlEntry->abandon(__FUNCTION__);
185  urlEntry = nullptr;
186  }
187  } else {
188  urlres_e = urlEntry;
189  urlres_e->lock("UrnState::start");
190  sc = storeClientListAdd(urlres_e, this);
191  }
192 
193  reqofs = 0;
194  StoreIOBuffer tempBuffer;
195  tempBuffer.offset = reqofs;
196  tempBuffer.length = URN_REQBUF_SZ;
197  tempBuffer.data = reqbuf;
199  tempBuffer,
201  this);
202 }
203 
204 void
206 {
207  checklist.setRequest(request.getRaw());
208  checklist.al = ale;
209 }
210 
211 void
213 {
214  const auto anUrn = new UrnState(ale);
215  anUrn->start (r, e);
216 }
217 
218 static int
219 url_entry_sort(const void *A, const void *B)
220 {
221  const url_entry *u1 = (const url_entry *)A;
222  const url_entry *u2 = (const url_entry *)B;
223 
224  if (u2->rtt == u1->rtt)
225  return 0;
226  else if (0 == u1->rtt)
227  return 1;
228  else if (0 == u2->rtt)
229  return -1;
230  else
231  return u1->rtt - u2->rtt;
232 }
233 
234 /* TODO: use the clientStream support for this */
235 static void
237 {
238  UrnState *urnState = static_cast<UrnState *>(data);
239  StoreEntry *e = urnState->entry;
240  StoreEntry *urlres_e = urnState->urlres_e;
241  char *s = NULL;
242  size_t k;
243  HttpReply *rep;
244  url_entry *urls;
245  url_entry *u;
246  url_entry *min_u;
247  ErrorState *err;
248  int i;
249  int urlcnt = 0;
250  char *buf = urnState->reqbuf;
251  StoreIOBuffer tempBuffer;
252 
253  debugs(52, 3, "urnHandleReply: Called with size=" << result.length << ".");
254 
255  if (EBIT_TEST(urlres_e->flags, ENTRY_ABORTED) || result.flags.error) {
256  delete urnState;
257  return;
258  }
259 
260  /* Update reqofs to point to where in the buffer we'd be */
261  urnState->reqofs += result.length;
262 
263  /* Handle reqofs being bigger than normal */
264  if (urnState->reqofs >= URN_REQBUF_SZ) {
265  delete urnState;
266  return;
267  }
268 
269  /* If we haven't received the entire object (urn), copy more */
270  if (urlres_e->store_status == STORE_PENDING) {
271  Must(result.length > 0); // zero length ought to imply STORE_OK
272  tempBuffer.offset = urnState->reqofs;
273  tempBuffer.length = URN_REQBUF_SZ - urnState->reqofs;
274  tempBuffer.data = urnState->reqbuf + urnState->reqofs;
275  storeClientCopy(urnState->sc, urlres_e,
276  tempBuffer,
278  urnState);
279  return;
280  }
281 
282  /* we know its STORE_OK */
283  k = headersEnd(buf, urnState->reqofs);
284 
285  if (0 == k) {
286  debugs(52, DBG_IMPORTANT, "urnHandleReply: didn't find end-of-headers for " << e->url() );
287  delete urnState;
288  return;
289  }
290 
291  s = buf + k;
292  // TODO: Check whether we should parse urlres_e reply, as before 528b2c61.
293  rep = new HttpReply;
294  rep->parseCharBuf(buf, k);
295  debugs(52, 3, "reply exists, code=" << rep->sline.status() << ".");
296 
297  if (rep->sline.status() != Http::scOkay) {
298  debugs(52, 3, "urnHandleReply: failed.");
299  err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw(), urnState->ale);
300  err->url = xstrdup(e->url());
301  errorAppendEntry(e, err);
302  delete rep;
303  delete urnState;
304  return;
305  }
306 
307  delete rep;
308 
309  while (xisspace(*s))
310  ++s;
311 
312  urls = urnParseReply(s, urnState->request->method);
313 
314  if (!urls) { /* unknown URN error */
315  debugs(52, 3, "urnTranslateDone: unknown URN " << e->url());
316  err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw(), urnState->ale);
317  err->url = xstrdup(e->url());
318  errorAppendEntry(e, err);
319  delete urnState;
320  return;
321  }
322 
323  for (i = 0; urls[i].url; ++i)
324  ++urlcnt;
325 
326  debugs(53, 3, "urnFindMinRtt: Counted " << i << " URLs");
327 
328  min_u = urnFindMinRtt(urls, urnState->request->method, NULL);
329  qsort(urls, urlcnt, sizeof(*urls), url_entry_sort);
330  e->buffer();
331  SBuf body;
332  SBuf *mb = &body; // diff reduction hack; TODO: Remove
333  mb->appendf( "<TITLE>Select URL for %s</TITLE>\n"
334  "<STYLE type=\"text/css\"><!--BODY{background-color:#ffffff;font-family:verdana,sans-serif}--></STYLE>\n"
335  "<H2>Select URL for %s</H2>\n"
336  "<TABLE BORDER=\"0\" WIDTH=\"100%%\">\n", e->url(), e->url());
337 
338  for (i = 0; i < urlcnt; ++i) {
339  u = &urls[i];
340  debugs(52, 3, "URL {" << u->url << "}");
341  mb->appendf(
342  "<TR><TD><A HREF=\"%s\">%s</A></TD>", u->url, u->url);
343 
344  if (urls[i].rtt > 0)
345  mb->appendf(
346  "<TD align=\"right\">%4d <it>ms</it></TD>", u->rtt);
347  else
348  mb->appendf("<TD align=\"right\">Unknown</TD>");
349 
350  mb->appendf("<TD>%s</TD></TR>\n", u->flags.cached ? " [cached]" : " ");
351  }
352 
353  mb->appendf(
354  "</TABLE>"
355  "<HR noshade size=\"1px\">\n"
356  "<ADDRESS>\n"
357  "Generated by %s@%s\n"
358  "</ADDRESS>\n",
360  rep = new HttpReply;
361  rep->setHeaders(Http::scFound, NULL, "text/html", mb->length(), 0, squid_curtime);
362 
363  if (min_u) {
365  }
366 
367  rep->body.set(body);
368  e->replaceHttpReply(rep);
369  e->complete();
370 
371  for (i = 0; i < urlcnt; ++i) {
372  safe_free(urls[i].url);
373  safe_free(urls[i].host);
374  }
375 
376  safe_free(urls);
377 
378  delete urnState;
379 }
380 
381 static url_entry *
382 urnParseReply(const char *inbuf, const HttpRequestMethod& m)
383 {
384  char *buf = xstrdup(inbuf);
385  char *token;
386  url_entry *list;
387  url_entry *old;
388  int n = 32;
389  int i = 0;
390  debugs(52, 3, "urnParseReply");
391  list = (url_entry *)xcalloc(n + 1, sizeof(*list));
392 
393  for (token = strtok(buf, crlf); token; token = strtok(NULL, crlf)) {
394  debugs(52, 3, "urnParseReply: got '" << token << "'");
395 
396  if (i == n) {
397  old = list;
398  n <<= 2;
399  list = (url_entry *)xcalloc(n + 1, sizeof(*list));
400  memcpy(list, old, i * sizeof(*list));
401  safe_free(old);
402  }
403 
404  AnyP::Uri uri;
405  if (!uri.parse(m, SBuf(token)) || !*uri.host())
406  continue;
407 
408 #if USE_ICMP
409  list[i].rtt = netdbHostRtt(uri.host());
410 
411  if (0 == list[i].rtt) {
412  debugs(52, 3, "Pinging " << uri.host());
413  netdbPingSite(uri.host());
414  }
415 #else
416  list[i].rtt = 0;
417 #endif
418 
419  list[i].url = xstrdup(uri.absolute().c_str());
420  list[i].host = xstrdup(uri.host());
421  // TODO: Use storeHas() or lock/unlock entry to avoid creating unlocked
422  // ones.
423  list[i].flags.cached = storeGetPublic(list[i].url, m) ? 1 : 0;
424  ++i;
425  }
426 
427  debugs(52, 3, "urnParseReply: Found " << i << " URLs");
428  return list;
429 }
430 
int reqofs
Definition: urn.cc:53
#define URN_REQBUF_SZ
Definition: urn.cc:31
virtual void buffer()
Definition: store.cc:1636
void * xcalloc(size_t n, size_t sz)
Definition: xalloc.cc:72
char reqbuf[URN_REQBUF_SZ]
Definition: urn.cc:52
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
Definition: Uri.h:30
void start(HttpRequest *, StoreEntry *)
Definition: urn.cc:165
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:75
HttpHeader header
Definition: Message.h:75
virtual LogTags * loggingTags() const
Definition: urn.cc:57
void errorAppendEntry(StoreEntry *entry, ErrorState *err)
Definition: errorpage.cc:711
const char * url() const
Definition: store.cc:1601
void lock(const char *context)
Definition: store.cc:459
#define CBDATA_CLASS(type)
Definition: cbdata.h:302
int64_t offset
Definition: StoreIOBuffer.h:55
bool parse(const HttpRequestMethod &, const SBuf &url)
Definition: Uri.cc:253
static url_entry * urnFindMinRtt(url_entry *urls, const HttpRequestMethod &, int *rtt_ret)
Definition: urn.cc:96
char * urlres
Definition: urn.cc:60
Definition: SBuf.h:86
#define xstrdup
int cached
Definition: urn.cc:69
C * getRaw() const
Definition: RefCount.h:80
double const StatHist & B
uint16_t flags
Definition: Store.h:224
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
Http::StatusLine sline
Definition: HttpReply.h:60
int netdbHostRtt(const char *host)
Definition: net_db.cc:1051
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1761
AccessLogEntry::Pointer ale
details of the requesting transaction
Definition: urn.cc:50
static void Start(const Comm::ConnectionPointer &client, StoreEntry *, HttpRequest *, const AccessLogEntryPointer &alp)
Initiates request forwarding to a peer or origin server.
Definition: FwdState.cc:343
#define DBG_IMPORTANT
Definition: Debug.h:46
bool parseCharBuf(const char *buf, ssize_t end)
Definition: Message.cc:148
HttpRequest::Pointer request
Definition: urn.cc:48
HttpRequest::Pointer urlres_r
Definition: urn.cc:49
HttpBody body
Definition: HttpReply.h:62
a storeGetPublic*() caller
Definition: StoreClient.h:24
void set(const SBuf &newContent)
Definition: HttpBody.h:26
#define NULL
Definition: types.h:166
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
static int url_entry_sort(const void *A, const void *B)
Definition: urn.cc:219
int rtt
Definition: urn.cc:66
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
Definition: urn.cc:33
static uint32 A
Definition: md4.c:43
store_client * sc
Definition: urn.cc:46
static url_entry * urnParseReply(const char *inbuf, const HttpRequestMethod &)
Definition: urn.cc:382
unsigned error
Definition: StoreIOBuffer.h:52
struct StoreIOBuffer::@148 flags
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
void const char HLPCB void * data
Definition: stub_helper.cc:16
@ ERR_URN_RESOLVE
Definition: err_type.h:34
int unlock(const char *context)
Definition: store.cc:483
#define safe_free(x)
Definition: xalloc.h:73
virtual void fillChecklist(ACLFilledChecklist &) const
configure the ACL checklist with the current transaction state
Definition: urn.cc:205
store_status_t store_status
Definition: Store.h:236
MasterXaction::Pointer masterXaction
the master transaction this request belongs to. Never nil.
Definition: HttpRequest.h:239
#define assert(EX)
Definition: assert.h:19
class AccessLogEntry::CacheDetails cache
UrnState(const AccessLogEntry::Pointer &anAle)
Definition: urn.cc:38
char * url
Definition: urn.cc:64
const char * c_str()
Definition: SBuf.cc:526
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:404
void setUriResFromRequest(HttpRequest *)
Definition: urn.cc:142
time_t squid_curtime
Definition: stub_time.cc:17
StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod &method)
Definition: store.cc:782
StoreEntry * entry
Definition: urn.cc:45
bool startCollapsingOn(const StoreEntry &, const bool doingRevalidation) const
Definition: store_client.cc:66
static const char *const crlf
Definition: urn.cc:75
void complete()
Definition: store.cc:1057
void urnStart(HttpRequest *r, StoreEntry *e, const AccessLogEntryPointer &ale)
Definition: urn.cc:212
struct url_entry::@154 flags
@ scFound
Definition: StatusCode.h:38
virtual ~UrnState()
Definition: urn.cc:79
SBuf & absolute() const
Definition: Uri.cc:592
HttpRequestMethod method
Definition: HttpRequest.h:114
@ scNotFound
Definition: StatusCode.h:48
@ ENTRY_ABORTED
Definition: enums.h:115
char * url
Definition: errorpage.h:179
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1068
void setRequest(HttpRequest *)
configure client request-related fields for the first time
const char * getMyHostname(void)
Definition: tools.cc:412
static HttpRequest * FromUrlXXX(const char *url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:546
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
Definition: urn.cc:63
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:521
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:239
char * host
Definition: urn.cc:65
void setHeaders(Http::StatusCode status, const char *reason, const char *ctype, int64_t clen, time_t lmt, time_t expires)
Definition: HttpReply.cc:168
#define xisspace(x)
Definition: xis.h:17
#define SWALLOW_EXCEPTIONS(code)
Definition: TextException.h:73
@ scOkay
Definition: StatusCode.h:26
@ METHOD_GET
Definition: MethodType.h:25
StoreEntry * urlres_e
Definition: urn.cc:47
#define APP_FULLNAME
Definition: version.h:25
int storeUnregister(store_client *sc, StoreEntry *e, void *data)
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:16
void storeClientCopy(store_client *sc, StoreEntry *e, StoreIOBuffer copyInto, STCB *callback, void *data)
void host(const char *src)
Definition: Uri.cc:98
void netdbPingSite(const char *hostname)
Definition: net_db.cc:908
void const char * buf
Definition: stub_helper.cc:16
void STCB(void *, StoreIOBuffer)
Definition: StoreClient.h:17
static STCB urnHandleReply
Definition: urn.cc:73
#define SQUIDSBUFPH
Definition: SBuf.h:31
store_client * storeClientListAdd(StoreEntry *e, void *data)
@ STORE_PENDING
Definition: enums.h:51

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors