urn.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 52 URN Parsing */
10 
11 #include "squid.h"
12 #include "AccessLogEntry.h"
13 #include "acl/FilledChecklist.h"
14 #include "base/TextException.h"
15 #include "cbdata.h"
16 #include "errorpage.h"
17 #include "FwdState.h"
18 #include "globals.h"
19 #include "HttpReply.h"
20 #include "HttpRequest.h"
21 #include "icmp/net_db.h"
22 #include "MemBuf.h"
23 #include "mime_header.h"
24 #include "RequestFlags.h"
25 #include "SquidTime.h"
26 #include "Store.h"
27 #include "StoreClient.h"
28 #include "tools.h"
29 #include "urn.h"
30 
31 #define URN_REQBUF_SZ 4096
32 
33 class UrnState : public StoreClient
34 {
36 
37 public:
38  explicit UrnState(const AccessLogEntry::Pointer &anAle): ale(anAle) {}
39 
40  void start (HttpRequest *, StoreEntry *);
42 
43  virtual ~UrnState();
44 
45  StoreEntry *entry = nullptr;
46  store_client *sc = nullptr;
47  StoreEntry *urlres_e = nullptr;
51 
52  char reqbuf[URN_REQBUF_SZ] = { '\0' };
53  int reqofs = 0;
54 
55 private:
56  /* StoreClient API */
57  virtual LogTags *loggingTags() const { return ale ? &ale->cache.code : nullptr; }
58  virtual void fillChecklist(ACLFilledChecklist &) const;
59 
60  char *urlres = nullptr;
61 };
62 
63 typedef struct {
64  char *url;
65  char *host;
66  int rtt;
67 
68  struct {
69  int cached;
70  } flags;
71 } url_entry;
72 
74 static url_entry *urnParseReply(const char *inbuf, const HttpRequestMethod&);
75 static const char *const crlf = "\r\n";
76 
78 
80 {
82  if (urlres_e) {
83  if (sc)
84  storeUnregister(sc, urlres_e, this);
85  urlres_e->unlock("~UrnState+res");
86  }
87 
88  if (entry)
89  entry->unlock("~UrnState+prime");
90 
92  });
93 }
94 
95 static url_entry *
96 urnFindMinRtt(url_entry * urls, const HttpRequestMethod &, int *rtt_ret)
97 {
98  int min_rtt = 0;
99  url_entry *u = NULL;
100  url_entry *min_u = NULL;
101  int i;
102  int urlcnt = 0;
103  debugs(52, 3, "urnFindMinRtt");
104  assert(urls != NULL);
105 
106  for (i = 0; NULL != urls[i].url; ++i)
107  ++urlcnt;
108 
109  debugs(53, 3, "urnFindMinRtt: Counted " << i << " URLs");
110 
111  if (1 == urlcnt) {
112  debugs(52, 3, "urnFindMinRtt: Only one URL - return it!");
113  return urls;
114  }
115 
116  for (i = 0; i < urlcnt; ++i) {
117  u = &urls[i];
118  debugs(52, 3, "urnFindMinRtt: " << u->host << " rtt=" << u->rtt);
119 
120  if (u->rtt == 0)
121  continue;
122 
123  if (u->rtt > min_rtt && min_rtt != 0)
124  continue;
125 
126  min_rtt = u->rtt;
127 
128  min_u = u;
129  }
130 
131  if (rtt_ret)
132  *rtt_ret = min_rtt;
133 
134  debugs(52, DBG_IMPORTANT, "urnFindMinRtt: Returning '" <<
135  (min_u ? min_u->url : "NONE") << "' RTT " <<
136  min_rtt );
137 
138  return min_u;
139 }
140 
141 void
143 {
144  const auto &query = r->url.absolute();
145  const auto host = r->url.host();
146  // TODO: use class AnyP::Uri instead of generating a string and re-parsing
147  LOCAL_ARRAY(char, local_urlres, 4096);
148  snprintf(local_urlres, 4096, "http://%s/uri-res/N2L?" SQUIDSBUFPH, host, SQUIDSBUFPRINT(query));
149  safe_free(urlres);
150  urlres_r = HttpRequest::FromUrlXXX(local_urlres, r->masterXaction);
151 
152  if (!urlres_r) {
153  debugs(52, 3, "Bad uri-res URL " << local_urlres);
154  const auto err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, r, ale);
155  err->url = xstrdup(local_urlres);
156  errorAppendEntry(entry, err);
157  return;
158  }
159 
160  urlres = xstrdup(local_urlres);
162 }
163 
164 void
166 {
167  debugs(52, 3, "urnStart: '" << e->url() << "'" );
168  entry = e;
169  request = r;
170 
171  entry->lock("UrnState::start");
173 
174  if (urlres_r == NULL)
175  return;
176 
177  auto urlEntry = storeGetPublic(urlres, Http::METHOD_GET);
178 
179  if (!urlEntry || (urlEntry->hittingRequiresCollapsing() && !startCollapsingOn(*urlEntry, false))) {
181  sc = storeClientListAdd(urlres_e, this);
183  if (urlEntry) {
184  urlEntry->abandon(__FUNCTION__);
185  urlEntry = nullptr;
186  }
187  } else {
188  urlres_e = urlEntry;
189  urlres_e->lock("UrnState::start");
190  sc = storeClientListAdd(urlres_e, this);
191  }
192 
193  reqofs = 0;
194  StoreIOBuffer tempBuffer;
195  tempBuffer.offset = reqofs;
196  tempBuffer.length = URN_REQBUF_SZ;
197  tempBuffer.data = reqbuf;
199  tempBuffer,
201  this);
202 }
203 
204 void
206 {
207  checklist.setRequest(request.getRaw());
208  checklist.al = ale;
209 }
210 
211 void
213 {
214  const auto anUrn = new UrnState(ale);
215  anUrn->start (r, e);
216 }
217 
218 static int
219 url_entry_sort(const void *A, const void *B)
220 {
221  const url_entry *u1 = (const url_entry *)A;
222  const url_entry *u2 = (const url_entry *)B;
223 
224  if (u2->rtt == u1->rtt)
225  return 0;
226  else if (0 == u1->rtt)
227  return 1;
228  else if (0 == u2->rtt)
229  return -1;
230  else
231  return u1->rtt - u2->rtt;
232 }
233 
234 /* TODO: use the clientStream support for this */
235 static void
236 urnHandleReply(void *data, StoreIOBuffer result)
237 {
238  UrnState *urnState = static_cast<UrnState *>(data);
239  StoreEntry *e = urnState->entry;
240  StoreEntry *urlres_e = urnState->urlres_e;
241  char *s = NULL;
242  size_t k;
243  HttpReply *rep;
244  url_entry *urls;
245  url_entry *u;
246  url_entry *min_u;
247  ErrorState *err;
248  int i;
249  int urlcnt = 0;
250  char *buf = urnState->reqbuf;
251  StoreIOBuffer tempBuffer;
252 
253  debugs(52, 3, "urnHandleReply: Called with size=" << result.length << ".");
254 
255  if (EBIT_TEST(urlres_e->flags, ENTRY_ABORTED) || result.flags.error) {
256  delete urnState;
257  return;
258  }
259 
260  if (!e->isAccepting()) {
261  debugs(52, 3, "terminating due to bad " << *e);
262  delete urnState;
263  return;
264  }
265 
266  /* Update reqofs to point to where in the buffer we'd be */
267  urnState->reqofs += result.length;
268 
269  /* Handle reqofs being bigger than normal */
270  if (urnState->reqofs >= URN_REQBUF_SZ) {
271  delete urnState;
272  return;
273  }
274 
275  /* If we haven't received the entire object (urn), copy more */
276  if (urlres_e->store_status == STORE_PENDING) {
277  Must(result.length > 0); // zero length ought to imply STORE_OK
278  tempBuffer.offset = urnState->reqofs;
279  tempBuffer.length = URN_REQBUF_SZ - urnState->reqofs;
280  tempBuffer.data = urnState->reqbuf + urnState->reqofs;
281  storeClientCopy(urnState->sc, urlres_e,
282  tempBuffer,
284  urnState);
285  return;
286  }
287 
288  /* we know its STORE_OK */
289  k = headersEnd(buf, urnState->reqofs);
290 
291  if (0 == k) {
292  debugs(52, DBG_IMPORTANT, "urnHandleReply: didn't find end-of-headers for " << e->url() );
293  delete urnState;
294  return;
295  }
296 
297  s = buf + k;
298  // TODO: Check whether we should parse urlres_e reply, as before 528b2c61.
299  rep = new HttpReply;
300  rep->parseCharBuf(buf, k);
301  debugs(52, 3, "reply exists, code=" << rep->sline.status() << ".");
302 
303  if (rep->sline.status() != Http::scOkay) {
304  debugs(52, 3, "urnHandleReply: failed.");
305  err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw(), urnState->ale);
306  err->url = xstrdup(e->url());
307  errorAppendEntry(e, err);
308  delete rep;
309  delete urnState;
310  return;
311  }
312 
313  delete rep;
314 
315  while (xisspace(*s))
316  ++s;
317 
318  urls = urnParseReply(s, urnState->request->method);
319 
320  if (!urls) { /* unknown URN error */
321  debugs(52, 3, "urnTranslateDone: unknown URN " << e->url());
322  err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw(), urnState->ale);
323  err->url = xstrdup(e->url());
324  errorAppendEntry(e, err);
325  delete urnState;
326  return;
327  }
328 
329  for (i = 0; urls[i].url; ++i)
330  ++urlcnt;
331 
332  debugs(53, 3, "urnFindMinRtt: Counted " << i << " URLs");
333 
334  min_u = urnFindMinRtt(urls, urnState->request->method, NULL);
335  qsort(urls, urlcnt, sizeof(*urls), url_entry_sort);
336  e->buffer();
337  SBuf body;
338  SBuf *mb = &body; // diff reduction hack; TODO: Remove
339  mb->appendf( "<TITLE>Select URL for %s</TITLE>\n"
340  "<STYLE type=\"text/css\"><!--BODY{background-color:#ffffff;font-family:verdana,sans-serif}--></STYLE>\n"
341  "<H2>Select URL for %s</H2>\n"
342  "<TABLE BORDER=\"0\" WIDTH=\"100%%\">\n", e->url(), e->url());
343 
344  for (i = 0; i < urlcnt; ++i) {
345  u = &urls[i];
346  debugs(52, 3, "URL {" << u->url << "}");
347  mb->appendf(
348  "<TR><TD><A HREF=\"%s\">%s</A></TD>", u->url, u->url);
349 
350  if (urls[i].rtt > 0)
351  mb->appendf(
352  "<TD align=\"right\">%4d <it>ms</it></TD>", u->rtt);
353  else
354  mb->appendf("<TD align=\"right\">Unknown</TD>");
355 
356  mb->appendf("<TD>%s</TD></TR>\n", u->flags.cached ? " [cached]" : " ");
357  }
358 
359  mb->appendf(
360  "</TABLE>"
361  "<HR noshade size=\"1px\">\n"
362  "<ADDRESS>\n"
363  "Generated by %s@%s\n"
364  "</ADDRESS>\n",
366  rep = new HttpReply;
367  rep->setHeaders(Http::scFound, NULL, "text/html", mb->length(), 0, squid_curtime);
368 
369  if (min_u) {
371  }
372 
373  rep->body.set(body);
374  e->replaceHttpReply(rep);
375  e->complete();
376 
377  for (i = 0; i < urlcnt; ++i) {
378  safe_free(urls[i].url);
379  safe_free(urls[i].host);
380  }
381 
382  safe_free(urls);
383 
384  delete urnState;
385 }
386 
387 static url_entry *
388 urnParseReply(const char *inbuf, const HttpRequestMethod& m)
389 {
390  char *buf = xstrdup(inbuf);
391  char *token;
392  url_entry *list;
393  url_entry *old;
394  int n = 32;
395  int i = 0;
396  debugs(52, 3, "urnParseReply");
397  list = (url_entry *)xcalloc(n + 1, sizeof(*list));
398 
399  for (token = strtok(buf, crlf); token; token = strtok(NULL, crlf)) {
400  debugs(52, 3, "urnParseReply: got '" << token << "'");
401 
402  if (i == n) {
403  old = list;
404  n <<= 2;
405  list = (url_entry *)xcalloc(n + 1, sizeof(*list));
406  memcpy(list, old, i * sizeof(*list));
407  safe_free(old);
408  }
409 
410  AnyP::Uri uri;
411  if (!uri.parse(m, SBuf(token)) || !*uri.host())
412  continue;
413 
414 #if USE_ICMP
415  list[i].rtt = netdbHostRtt(uri.host());
416 
417  if (0 == list[i].rtt) {
418  debugs(52, 3, "Pinging " << uri.host());
419  netdbPingSite(uri.host());
420  }
421 #else
422  list[i].rtt = 0;
423 #endif
424 
425  list[i].url = xstrdup(uri.absolute().c_str());
426  list[i].host = xstrdup(uri.host());
427  // TODO: Use storeHas() or lock/unlock entry to avoid creating unlocked
428  // ones.
429  list[i].flags.cached = storeGetPublic(list[i].url, m) ? 1 : 0;
430  ++i;
431  }
432 
433  debugs(52, 3, "urnParseReply: Found " << i << " URLs");
434  xfree(buf);
435  return list;
436 }
437 
int reqofs
Definition: urn.cc:53
#define URN_REQBUF_SZ
Definition: urn.cc:31
virtual void buffer()
Definition: store.cc:1618
void * xcalloc(size_t n, size_t sz)
Definition: xalloc.cc:71
char reqbuf[URN_REQBUF_SZ]
Definition: urn.cc:52
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
Definition: Uri.h:31
void start(HttpRequest *, StoreEntry *)
Definition: urn.cc:165
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:75
HttpHeader header
Definition: Message.h:75
virtual LogTags * loggingTags() const
Definition: urn.cc:57
void errorAppendEntry(StoreEntry *entry, ErrorState *err)
Definition: errorpage.cc:720
const char * url() const
Definition: store.cc:1583
void lock(const char *context)
Definition: store.cc:459
#define CBDATA_CLASS(type)
Definition: cbdata.h:302
int64_t offset
Definition: StoreIOBuffer.h:55
bool parse(const HttpRequestMethod &, const SBuf &url)
Definition: Uri.cc:253
static url_entry * urnFindMinRtt(url_entry *urls, const HttpRequestMethod &, int *rtt_ret)
Definition: urn.cc:96
char * urlres
Definition: urn.cc:60
Definition: SBuf.h:87
#define xstrdup
int cached
Definition: urn.cc:69
C * getRaw() const
Definition: RefCount.h:80
uint16_t flags
Definition: Store.h:233
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
bool isAccepting() const
Definition: store.cc:1990
Http::StatusLine sline
Definition: HttpReply.h:56
struct StoreIOBuffer::@147 flags
int netdbHostRtt(const char *host)
Definition: net_db.cc:1056
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1722
AccessLogEntry::Pointer ale
details of the requesting transaction
Definition: urn.cc:50
static void Start(const Comm::ConnectionPointer &client, StoreEntry *, HttpRequest *, const AccessLogEntryPointer &alp)
Initiates request forwarding to a peer or origin server.
Definition: FwdState.cc:362
#define DBG_IMPORTANT
Definition: Debug.h:41
bool parseCharBuf(const char *buf, ssize_t end)
Definition: Message.cc:140
HttpRequest::Pointer request
Definition: urn.cc:48
HttpRequest::Pointer urlres_r
Definition: urn.cc:49
@ ENTRY_ABORTED
Definition: enums.h:115
HttpBody body
Definition: HttpReply.h:58
a storeGetPublic*() caller
Definition: StoreClient.h:26
void set(const SBuf &newContent)
Definition: HttpBody.h:26
#define NULL
Definition: types.h:166
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
static int url_entry_sort(const void *A, const void *B)
Definition: urn.cc:219
int rtt
Definition: urn.cc:66
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
Definition: urn.cc:34
static uint32 A
Definition: md4.c:43
store_client * sc
Definition: urn.cc:46
static url_entry * urnParseReply(const char *inbuf, const HttpRequestMethod &)
Definition: urn.cc:388
unsigned error
Definition: StoreIOBuffer.h:52
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
int unlock(const char *context)
Definition: store.cc:483
#define safe_free(x)
Definition: xalloc.h:73
virtual void fillChecklist(ACLFilledChecklist &) const
configure the given checklist (to reflect the current transaction state)
Definition: urn.cc:205
store_status_t store_status
Definition: Store.h:245
MasterXaction::Pointer masterXaction
the master transaction this request belongs to. Never nil.
Definition: HttpRequest.h:238
#define assert(EX)
Definition: assert.h:19
class AccessLogEntry::CacheDetails cache
UrnState(const AccessLogEntry::Pointer &anAle)
Definition: urn.cc:38
char * url
Definition: urn.cc:64
const char * c_str()
Definition: SBuf.cc:516
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:408
void setUriResFromRequest(HttpRequest *)
Definition: urn.cc:142
time_t squid_curtime
Definition: stub_time.cc:17
StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod &method)
Definition: store.cc:773
#define xfree
StoreEntry * entry
Definition: urn.cc:45
bool startCollapsingOn(const StoreEntry &, const bool doingRevalidation) const
Definition: store_client.cc:65
static const char *const crlf
Definition: urn.cc:75
void complete()
Definition: store.cc:1045
void urnStart(HttpRequest *r, StoreEntry *e, const AccessLogEntryPointer &ale)
Definition: urn.cc:212
@ scFound
Definition: StatusCode.h:38
virtual ~UrnState()
Definition: urn.cc:79
SBuf & absolute() const
Definition: Uri.cc:583
HttpRequestMethod method
Definition: HttpRequest.h:114
@ scNotFound
Definition: StatusCode.h:48
char * url
Definition: errorpage.h:176
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1051
void setRequest(HttpRequest *)
configure client request-related fields for the first time
const char * getMyHostname(void)
Definition: tools.cc:414
static HttpRequest * FromUrlXXX(const char *url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:529
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:73
Definition: urn.cc:63
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:518
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:229
char * host
Definition: urn.cc:65
void setHeaders(Http::StatusCode status, const char *reason, const char *ctype, int64_t clen, time_t lmt, time_t expires)
Definition: HttpReply.cc:168
#define xisspace(x)
Definition: xis.h:17
#define SWALLOW_EXCEPTIONS(code)
Definition: TextException.h:77
static uint32 B
Definition: md4.c:43
@ scOkay
Definition: StatusCode.h:26
@ ERR_URN_RESOLVE
Definition: forward.h:36
struct url_entry::@152 flags
@ METHOD_GET
Definition: MethodType.h:25
StoreEntry * urlres_e
Definition: urn.cc:47
static StatHist s
#define APP_FULLNAME
Definition: version.h:25
int storeUnregister(store_client *sc, StoreEntry *e, void *data)
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:17
void storeClientCopy(store_client *sc, StoreEntry *e, StoreIOBuffer copyInto, STCB *callback, void *data)
void host(const char *src)
Definition: Uri.cc:98
void netdbPingSite(const char *hostname)
Definition: net_db.cc:908
void STCB(void *, StoreIOBuffer)
Definition: StoreClient.h:18
static STCB urnHandleReply
Definition: urn.cc:73
#define SQUIDSBUFPH
Definition: SBuf.h:31
store_client * storeClientListAdd(StoreEntry *e, void *data)
@ STORE_PENDING
Definition: enums.h:51

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors