urn.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 52 URN Parsing */
10 
11 #include "squid.h"
12 #include "AccessLogEntry.h"
13 #include "acl/FilledChecklist.h"
14 #include "cbdata.h"
15 #include "errorpage.h"
16 #include "FwdState.h"
17 #include "globals.h"
18 #include "HttpReply.h"
19 #include "HttpRequest.h"
20 #include "icmp/net_db.h"
21 #include "MemBuf.h"
22 #include "mime_header.h"
23 #include "RequestFlags.h"
24 #include "SquidTime.h"
25 #include "Store.h"
26 #include "StoreClient.h"
27 #include "tools.h"
28 #include "urn.h"
29 
30 #define URN_REQBUF_SZ 4096
31 
32 class UrnState : public StoreClient
33 {
35 
36 public:
37  explicit UrnState(const AccessLogEntry::Pointer &anAle): ale(anAle) {}
38 
39  void created (StoreEntry *newEntry);
40  void start (HttpRequest *, StoreEntry *);
41  char *getHost(const SBuf &urlpath);
43 
44  virtual ~UrnState();
45 
46  StoreEntry *entry = nullptr;
47  store_client *sc = nullptr;
48  StoreEntry *urlres_e = nullptr;
51 
52  struct {
53  bool force_menu = false;
54  } flags;
55  char reqbuf[URN_REQBUF_SZ] = { '\0' };
56  int reqofs = 0;
57 
58 private:
59  /* StoreClient API */
60  virtual LogTags *loggingTags() { return ale ? &ale->cache.code : nullptr; }
61  virtual void fillChecklist(ACLFilledChecklist &) const;
62 
63  char *urlres = nullptr;
65 };
66 
67 typedef struct {
68  char *url;
69  char *host;
70  int rtt;
71 
72  struct {
73  int cached;
74  } flags;
75 } url_entry;
76 
78 static url_entry *urnParseReply(const char *inbuf, const HttpRequestMethod&);
79 static const char *const crlf = "\r\n";
80 
82 
84 {
86 }
87 
88 static url_entry *
89 urnFindMinRtt(url_entry * urls, const HttpRequestMethod &, int *rtt_ret)
90 {
91  int min_rtt = 0;
92  url_entry *u = NULL;
93  url_entry *min_u = NULL;
94  int i;
95  int urlcnt = 0;
96  debugs(52, 3, "urnFindMinRtt");
97  assert(urls != NULL);
98 
99  for (i = 0; NULL != urls[i].url; ++i)
100  ++urlcnt;
101 
102  debugs(53, 3, "urnFindMinRtt: Counted " << i << " URLs");
103 
104  if (1 == urlcnt) {
105  debugs(52, 3, "urnFindMinRtt: Only one URL - return it!");
106  return urls;
107  }
108 
109  for (i = 0; i < urlcnt; ++i) {
110  u = &urls[i];
111  debugs(52, 3, "urnFindMinRtt: " << u->host << " rtt=" << u->rtt);
112 
113  if (u->rtt == 0)
114  continue;
115 
116  if (u->rtt > min_rtt && min_rtt != 0)
117  continue;
118 
119  min_rtt = u->rtt;
120 
121  min_u = u;
122  }
123 
124  if (rtt_ret)
125  *rtt_ret = min_rtt;
126 
127  debugs(52, DBG_IMPORTANT, "urnFindMinRtt: Returning '" <<
128  (min_u ? min_u->url : "NONE") << "' RTT " <<
129  min_rtt );
130 
131  return min_u;
132 }
133 
134 char *
135 UrnState::getHost(const SBuf &urlpath)
136 {
138  /* a proper encapsulated URI/URL type needs to clear this up. */
139  size_t p;
140  if ((p = urlpath.find(':')) != SBuf::npos)
141  return SBufToCstring(urlpath.substr(0, p-1));
142 
143  return SBufToCstring(urlpath);
144 }
145 
146 void
148 {
149  static const SBuf menu(".menu");
150  if (r->url.path().startsWith(menu)) {
151  r->url.path(r->url.path().substr(5)); // strip prefix "menu."
152  flags.force_menu = true;
153  }
154 
155  SBuf uri = r->url.path();
156  // TODO: use class AnyP::Uri instead of generating a string and re-parsing
157  LOCAL_ARRAY(char, local_urlres, 4096);
158  char *host = getHost(uri);
159  snprintf(local_urlres, 4096, "http://%s/uri-res/N2L?urn:" SQUIDSBUFPH, host, SQUIDSBUFPRINT(uri));
160  safe_free(host);
161  safe_free(urlres);
162  urlres_r = HttpRequest::FromUrl(local_urlres, r->masterXaction);
163 
164  if (!urlres_r) {
165  debugs(52, 3, "Bad uri-res URL " << local_urlres);
167  err->url = xstrdup(local_urlres);
168  errorAppendEntry(entry, err);
169  return;
170  }
171 
172  urlres = xstrdup(local_urlres);
174 }
175 
176 void
178 {
179  debugs(52, 3, "urnStart: '" << e->url() << "'" );
180  entry = e;
181  request = r;
182 
183  entry->lock("UrnState::start");
185 
186  if (urlres_r == NULL)
187  return;
188 
190 }
191 
192 void
194 {
195  checklist.setRequest(request.getRaw());
196  checklist.al = ale;
197 }
198 
199 void
201 {
202  if (!e || (e->hittingRequiresCollapsing() && !startCollapsingOn(*e, false))) {
204  sc = storeClientListAdd(urlres_e, this);
206  // TODO: StoreClients must either store/lock or abandon found entries.
207  //if (e)
208  // e->abandon();
209  } else {
210  urlres_e = e;
211  urlres_e->lock("UrnState::created");
212  sc = storeClientListAdd(urlres_e, this);
213  }
214 
215  reqofs = 0;
216  StoreIOBuffer tempBuffer;
217  tempBuffer.offset = reqofs;
218  tempBuffer.length = URN_REQBUF_SZ;
219  tempBuffer.data = reqbuf;
221  tempBuffer,
223  this);
224 }
225 
226 void
228 {
229  const auto anUrn = new UrnState(ale);
230  anUrn->start (r, e);
231 }
232 
233 static int
234 url_entry_sort(const void *A, const void *B)
235 {
236  const url_entry *u1 = (const url_entry *)A;
237  const url_entry *u2 = (const url_entry *)B;
238 
239  if (u2->rtt == u1->rtt)
240  return 0;
241  else if (0 == u1->rtt)
242  return 1;
243  else if (0 == u2->rtt)
244  return -1;
245  else
246  return u1->rtt - u2->rtt;
247 }
248 
249 static void
251 {
252  urlres_e->unlock("urnHandleReplyError+res");
253  urnState->entry->unlock("urnHandleReplyError+prime");
254  delete urnState;
255 }
256 
257 /* TODO: use the clientStream support for this */
258 static void
260 {
261  UrnState *urnState = static_cast<UrnState *>(data);
262  StoreEntry *e = urnState->entry;
263  StoreEntry *urlres_e = urnState->urlres_e;
264  char *s = NULL;
265  size_t k;
266  HttpReply *rep;
267  url_entry *urls;
268  url_entry *u;
269  url_entry *min_u;
270  MemBuf *mb = NULL;
271  ErrorState *err;
272  int i;
273  int urlcnt = 0;
274  char *buf = urnState->reqbuf;
275  StoreIOBuffer tempBuffer;
276 
277  debugs(52, 3, "urnHandleReply: Called with size=" << result.length << ".");
278 
279  if (EBIT_TEST(urlres_e->flags, ENTRY_ABORTED) || result.length == 0 || result.flags.error) {
280  urnHandleReplyError(urnState, urlres_e);
281  return;
282  }
283 
284  /* Update reqofs to point to where in the buffer we'd be */
285  urnState->reqofs += result.length;
286 
287  /* Handle reqofs being bigger than normal */
288  if (urnState->reqofs >= URN_REQBUF_SZ) {
289  urnHandleReplyError(urnState, urlres_e);
290  return;
291  }
292 
293  /* If we haven't received the entire object (urn), copy more */
294  if (urlres_e->store_status == STORE_PENDING &&
295  urnState->reqofs < URN_REQBUF_SZ) {
296  tempBuffer.offset = urnState->reqofs;
297  tempBuffer.length = URN_REQBUF_SZ;
298  tempBuffer.data = urnState->reqbuf + urnState->reqofs;
299  storeClientCopy(urnState->sc, urlres_e,
300  tempBuffer,
302  urnState);
303  return;
304  }
305 
306  /* we know its STORE_OK */
307  k = headersEnd(buf, urnState->reqofs);
308 
309  if (0 == k) {
310  debugs(52, DBG_IMPORTANT, "urnHandleReply: didn't find end-of-headers for " << e->url() );
311  urnHandleReplyError(urnState, urlres_e);
312  return;
313  }
314 
315  s = buf + k;
316  assert(urlres_e->getReply());
317  rep = new HttpReply;
318  rep->parseCharBuf(buf, k);
319  debugs(52, 3, "reply exists, code=" << rep->sline.status() << ".");
320 
321  if (rep->sline.status() != Http::scOkay) {
322  debugs(52, 3, "urnHandleReply: failed.");
323  err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw());
324  err->url = xstrdup(e->url());
325  errorAppendEntry(e, err);
326  delete rep;
327  urnHandleReplyError(urnState, urlres_e);
328  return;
329  }
330 
331  delete rep;
332 
333  while (xisspace(*s))
334  ++s;
335 
336  urls = urnParseReply(s, urnState->request->method);
337 
338  if (!urls) { /* unknown URN error */
339  debugs(52, 3, "urnTranslateDone: unknown URN " << e->url());
340  err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw());
341  err->url = xstrdup(e->url());
342  errorAppendEntry(e, err);
343  urnHandleReplyError(urnState, urlres_e);
344  return;
345  }
346 
347  for (i = 0; urls[i].url; ++i)
348  ++urlcnt;
349 
350  debugs(53, 3, "urnFindMinRtt: Counted " << i << " URLs");
351 
352  min_u = urnFindMinRtt(urls, urnState->request->method, NULL);
353  qsort(urls, urlcnt, sizeof(*urls), url_entry_sort);
354  e->buffer();
355  mb = new MemBuf;
356  mb->init();
357  mb->appendf( "<TITLE>Select URL for %s</TITLE>\n"
358  "<STYLE type=\"text/css\"><!--BODY{background-color:#ffffff;font-family:verdana,sans-serif}--></STYLE>\n"
359  "<H2>Select URL for %s</H2>\n"
360  "<TABLE BORDER=\"0\" WIDTH=\"100%%\">\n", e->url(), e->url());
361 
362  for (i = 0; i < urlcnt; ++i) {
363  u = &urls[i];
364  debugs(52, 3, "URL {" << u->url << "}");
365  mb->appendf(
366  "<TR><TD><A HREF=\"%s\">%s</A></TD>", u->url, u->url);
367 
368  if (urls[i].rtt > 0)
369  mb->appendf(
370  "<TD align=\"right\">%4d <it>ms</it></TD>", u->rtt);
371  else
372  mb->appendf("<TD align=\"right\">Unknown</TD>");
373 
374  mb->appendf("<TD>%s</TD></TR>\n", u->flags.cached ? " [cached]" : " ");
375  }
376 
377  mb->appendf(
378  "</TABLE>"
379  "<HR noshade size=\"1px\">\n"
380  "<ADDRESS>\n"
381  "Generated by %s@%s\n"
382  "</ADDRESS>\n",
384  rep = new HttpReply;
385  rep->setHeaders(Http::scFound, NULL, "text/html", mb->contentSize(), 0, squid_curtime);
386 
387  if (urnState->flags.force_menu) {
388  debugs(51, 3, "urnHandleReply: forcing menu");
389  } else if (min_u) {
390  rep->header.putStr(Http::HdrType::LOCATION, min_u->url);
391  }
392 
393  rep->body.setMb(mb);
394  /* don't clean or delete mb; rep->body owns it now */
395  e->replaceHttpReply(rep);
396  e->complete();
397 
398  for (i = 0; i < urlcnt; ++i) {
399  safe_free(urls[i].url);
400  safe_free(urls[i].host);
401  }
402 
403  safe_free(urls);
404  /* mb was absorbed in httpBodySet call, so we must not clean it */
405  storeUnregister(urnState->sc, urlres_e, urnState);
406 
407  urnHandleReplyError(urnState, urlres_e);
408 }
409 
410 static url_entry *
411 urnParseReply(const char *inbuf, const HttpRequestMethod& m)
412 {
413  char *buf = xstrdup(inbuf);
414  char *token;
415  char *host;
416  url_entry *list;
417  url_entry *old;
418  int n = 32;
419  int i = 0;
420  debugs(52, 3, "urnParseReply");
421  list = (url_entry *)xcalloc(n + 1, sizeof(*list));
422 
423  for (token = strtok(buf, crlf); token; token = strtok(NULL, crlf)) {
424  debugs(52, 3, "urnParseReply: got '" << token << "'");
425 
426  if (i == n) {
427  old = list;
428  n <<= 2;
429  list = (url_entry *)xcalloc(n + 1, sizeof(*list));
430  memcpy(list, old, i * sizeof(*list));
431  safe_free(old);
432  }
433 
434  host = urlHostname(token);
435 
436  if (NULL == host)
437  continue;
438 
439 #if USE_ICMP
440  list[i].rtt = netdbHostRtt(host);
441 
442  if (0 == list[i].rtt) {
443  debugs(52, 3, "urnParseReply: Pinging " << host);
444  netdbPingSite(host);
445  }
446 #else
447  list[i].rtt = 0;
448 #endif
449 
450  list[i].url = xstrdup(token);
451  list[i].host = xstrdup(host);
452  // TODO: Use storeHas() or lock/unlock entry to avoid creating unlocked
453  // ones.
454  list[i].flags.cached = storeGetPublic(list[i].url, m) ? 1 : 0;
455  ++i;
456  }
457 
458  debugs(52, 3, "urnParseReply: Found " << i << " URLs");
459  return list;
460 }
461 
MasterXaction::Pointer masterXaction
the master transaction this request belongs to. Never nil.
Definition: HttpRequest.h:228
void urnStart(HttpRequest *r, StoreEntry *e, const AccessLogEntryPointer &ale)
Definition: urn.cc:227
char * urlHostname(const char *url)
Definition: Uri.cc:858
virtual void fillChecklist(ACLFilledChecklist &) const
configure the ACL checklist with the current transaction state
Definition: urn.cc:193
bool startCollapsingOn(const StoreEntry &, const bool doingRevalidation)
Definition: store_client.cc:65
char * url
Definition: urn.cc:68
#define assert(EX)
Definition: assert.h:17
void storeClientCopy(store_client *sc, StoreEntry *e, StoreIOBuffer copyInto, STCB *callback, void *data)
UrnState(const AccessLogEntry::Pointer &anAle)
Definition: urn.cc:37
int unlock(const char *context)
Definition: store.cc:457
void path(const char *p)
Definition: Uri.h:86
Definition: SBuf.h:86
static uint32 B
Definition: md4.c:43
#define xcalloc
Definition: membanger.c:57
store_client * sc
Definition: urn.cc:47
StoreEntry * entry
Definition: urn.cc:46
HttpRequestMethod method
Definition: HttpRequest.h:106
int i
Definition: membanger.c:49
#define xstrdup
char * getHost(const SBuf &urlpath)
Definition: urn.cc:135
virtual void buffer()
Definition: store.cc:1635
virtual ~UrnState()
Definition: urn.cc:83
#define safe_free(x)
Definition: xalloc.h:73
static STCB urnHandleReply
Definition: urn.cc:77
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
char * url
Definition: errorpage.h:148
StoreEntry * urlres_e
Definition: urn.cc:48
int netdbHostRtt(const char *host)
Definition: net_db.cc:1052
void created(StoreEntry *newEntry)
Definition: urn.cc:200
HttpReply const * getReply() const
Definition: store.cc:1670
static void urnHandleReplyError(UrnState *urnState, StoreEntry *urlres_e)
Definition: urn.cc:250
store_client * storeClientListAdd(StoreEntry *e, void *data)
#define xisspace(x)
Definition: xis.h:17
static const char *const crlf
Definition: urn.cc:79
char * p
Definition: membanger.c:43
struct url_entry::@152 flags
void SBufToCstring(char *d, const SBuf &s)
Definition: SBuf.h:741
const char * getMyHostname(void)
Definition: tools.cc:412
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:516
time_t squid_curtime
Definition: stub_time.cc:17
void replaceHttpReply(HttpReply *, bool andStartWriting=true)
Definition: store.cc:1782
static url_entry * urnFindMinRtt(url_entry *urls, const HttpRequestMethod &, int *rtt_ret)
Definition: urn.cc:89
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
void complete()
Definition: store.cc:1058
static void getPublic(StoreClient *aClient, const char *uri, const HttpRequestMethod &method)
Definition: store.cc:509
Definition: urn.cc:32
char * host
Definition: urn.cc:69
void const char HLPCB void * data
Definition: stub_helper.cc:16
int64_t offset
Definition: StoreIOBuffer.h:55
virtual LogTags * loggingTags()
Definition: urn.cc:60
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
void setHeaders(Http::StatusCode status, const char *reason, const char *ctype, int64_t clen, time_t lmt, time_t expires)
Definition: HttpReply.cc:161
const char * url() const
Definition: store.cc:1600
StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod &method)
Definition: store.cc:776
#define DBG_IMPORTANT
Definition: Debug.h:46
#define URN_REQBUF_SZ
Definition: urn.cc:30
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
static int url_entry_sort(const void *A, const void *B)
Definition: urn.cc:234
#define CBDATA_CLASS(type)
Definition: cbdata.h:302
AnyP::Uri url
the request URI
Definition: HttpRequest.h:107
int reqofs
Definition: urn.cc:56
bool force_menu
Definition: urn.cc:53
uint16_t flags
Definition: Store.h:208
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:192
#define APP_FULLNAME
Definition: version.h:22
char reqbuf[URN_REQBUF_SZ]
Definition: urn.cc:55
struct StoreIOBuffer::@146 flags
#define LOCAL_ARRAY(type, name, size)
Definition: leakcheck.h:18
static HttpRequest * FromUrl(const char *url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:523
void const char * buf
Definition: stub_helper.cc:16
class AccessLogEntry::CacheDetails cache
static uint32 A
Definition: md4.c:43
unsigned error
Definition: StoreIOBuffer.h:52
HttpHeader header
Definition: Message.h:75
int storeUnregister(store_client *sc, StoreEntry *e, void *data)
store_status_t store_status
Definition: Store.h:220
int cached
Definition: urn.cc:73
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1072
SBuf substr(size_type pos, size_type n=npos) const
Definition: SBuf.cc:586
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
void STCB(void *, StoreIOBuffer)
Definition: StoreClient.h:16
C * getRaw() const
Definition: RefCount.h:74
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
size_type find(char c, size_type startPos=0) const
Definition: SBuf.cc:594
static const size_type npos
Definition: SBuf.h:92
AccessLogEntry::Pointer ale
master transaction summary
Definition: urn.cc:64
Definition: MemBuf.h:23
bool parseCharBuf(const char *buf, ssize_t end)
Definition: Message.cc:148
void start(HttpRequest *, StoreEntry *)
Definition: urn.cc:177
void errorAppendEntry(StoreEntry *entry, ErrorState *err)
Definition: errorpage.cc:575
static void Start(const Comm::ConnectionPointer &client, StoreEntry *, HttpRequest *, const AccessLogEntryPointer &alp)
Initiates request forwarding to a peer or origin server.
Definition: FwdState.cc:317
void setRequest(HttpRequest *)
configure client request-related fields for the first time
HttpRequest::Pointer request
Definition: urn.cc:49
void netdbPingSite(const char *hostname)
Definition: net_db.cc:909
HttpRequest::Pointer urlres_r
Definition: urn.cc:50
#define SQUIDSBUFPH
Definition: SBuf.h:31
char * urlres
Definition: urn.cc:63
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:16
int rtt
Definition: urn.cc:70
static url_entry * urnParseReply(const char *inbuf, const HttpRequestMethod &)
Definition: urn.cc:411
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
void setUriResFromRequest(HttpRequest *)
Definition: urn.cc:147
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
#define NULL
Definition: types.h:166
A StoreEntry::getPublic*() caller.
Definition: StoreClient.h:23
struct UrnState::@151 flags
void lock(const char *context)
Definition: store.cc:433
Definition: urn.cc:67

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors