HttpHeader.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 55 HTTP Header */
10 
11 #include "squid.h"
12 #include "base/CharacterSet.h"
13 #include "base/EnumIterator.h"
14 #include "base64.h"
15 #include "globals.h"
17 #include "HttpHdrCc.h"
18 #include "HttpHdrContRange.h"
19 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
20 #include "HttpHeader.h"
21 #include "HttpHeaderFieldInfo.h"
22 #include "HttpHeaderStat.h"
23 #include "HttpHeaderTools.h"
24 #include "MemBuf.h"
25 #include "mgr/Registration.h"
26 #include "mime_header.h"
27 #include "profiler/Profiler.h"
28 #include "rfc1123.h"
29 #include "sbuf/StringConvert.h"
30 #include "SquidConfig.h"
31 #include "StatHist.h"
32 #include "Store.h"
33 #include "StrList.h"
34 #include "TimeOrTag.h"
35 #include "util.h"
36 
37 #include <algorithm>
38 
39 /* XXX: the whole set of API managing the entries vector should be rethought
40  * after the parse4r-ng effort is complete.
41  */
42 
43 /*
44  * On naming conventions:
45  *
46  * HTTP/1.1 defines message-header as
47  *
48  * message-header = field-name ":" [ field-value ] CRLF
49  * field-name = token
50  * field-value = *( field-content | LWS )
51  *
52  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
53  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
54  *
55  * HttpHeader is an object that represents all message-headers in a message.
56  * HttpHeader does not manage start-line.
57  *
58  * HttpHeader is implemented as a collection of header "entries".
59  * An entry is a (field_id, field_name, field_value) triplet.
60  */
61 
62 /*
63  * local constants and vars
64  */
65 
66 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
67 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
68 
69 /* request-only headers. Used for cachemgr */
70 static HttpHeaderMask RequestHeadersMask; /* set run-time using RequestHeaders */
71 
72 /* reply-only headers. Used for cachemgr */
73 static HttpHeaderMask ReplyHeadersMask; /* set run-time using ReplyHeaders */
74 
75 /* header accounting */
76 // NP: keep in sync with enum http_hdr_owner_type
78  HttpHeaderStat(/*hoNone*/ "all", NULL),
79 #if USE_HTCP
80  HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
81 #endif
82  HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
83  HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
84 #if USE_OPENSSL
85  /* hoErrorDetail */
86 #endif
87  /* hoEnd */
88 };
89 static int HttpHeaderStatCount = countof(HttpHeaderStats);
90 
91 static int HeaderEntryParsedCount = 0;
92 
93 /*
94  * forward declarations and local routines
95  */
96 
97 class StoreEntry;
98 
99 // update parse statistics for header id; if error is true also account
100 // for errors and write to debug log what happened
101 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
102 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
104 static void httpHeaderStoreReport(StoreEntry * e);
105 
106 /*
107  * Module initialization routines
108  */
109 
110 static void
112 {
113  Mgr::RegisterAction("http_headers",
114  "HTTP Header Statistics",
115  httpHeaderStoreReport, 0, 1);
116 }
117 
118 void
120 {
121  /* check that we have enough space for masks */
123 
124  // masks are needed for stats page still
125  for (auto h : WholeEnum<Http::HdrType>()) {
126  if (Http::HeaderLookupTable.lookup(h).request)
128  if (Http::HeaderLookupTable.lookup(h).reply)
130  }
131 
132  /* header stats initialized by class constructor */
134 
135  /* init dependent modules */
138 
140 }
141 
142 /*
143  * HttpHeader Implementation
144  */
145 
146 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
147 {
148  entries.reserve(32);
150 }
151 
152 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
153 {
154  assert(anOwner > hoNone && anOwner < hoEnd);
155  debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
156  entries.reserve(32);
158 }
159 
160 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
161 {
162  entries.reserve(other.entries.capacity());
164  update(&other); // will update the mask as well
165 }
166 
168 {
169  clean();
170 }
171 
172 HttpHeader &
174 {
175  if (this != &other) {
176  // we do not really care, but the caller probably does
177  assert(owner == other.owner);
178  clean();
179  update(&other); // will update the mask as well
180  len = other.len;
182  }
183  return *this;
184 }
185 
186 void
188 {
189 
190  assert(owner > hoNone && owner < hoEnd);
191  debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
192 
193  PROF_start(HttpHeaderClean);
194 
195  if (owner <= hoReply) {
196  /*
197  * An unfortunate bug. The entries array is initialized
198  * such that count is set to zero. httpHeaderClean() seems to
199  * be called both when 'hdr' is created, and destroyed. Thus,
200  * we accumulate a large number of zero counts for 'hdr' before
201  * it is ever used. Can't think of a good way to fix it, except
202  * adding a state variable that indicates whether or not 'hdr'
203  * has been used. As a hack, just never count zero-sized header
204  * arrays.
205  */
206  if (!entries.empty())
207  HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
208 
209  ++ HttpHeaderStats[owner].destroyedCount;
210 
211  HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
212  } // if (owner <= hoReply)
213 
214  for (HttpHeaderEntry *e : entries) {
215  if (e == nullptr)
216  continue;
217  if (!Http::any_valid_header(e->id)) {
218  debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
219  } else {
220  if (owner <= hoReply)
221  HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
222  delete e;
223  }
224  }
225 
226  entries.clear();
228  len = 0;
230  PROF_stop(HttpHeaderClean);
231 }
232 
233 /* append entries (also see httpHeaderUpdate) */
234 void
236 {
237  assert(src);
238  assert(src != this);
239  debugs(55, 7, "appending hdr: " << this << " += " << src);
240 
241  for (auto e : src->entries) {
242  if (e)
243  addEntry(e->clone());
244  }
245 }
246 
248 bool
250 {
251  for (const auto e: fresh->entries) {
252  if (!e || skipUpdateHeader(e->id))
253  continue;
254  String value;
255  if (!hasNamed(e->name, &value) ||
256  (value != fresh->getByName(e->name)))
257  return true;
258  }
259  return false;
260 }
261 
262 void
264 {
265  int count = 0;
267 
268  // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
269  while (HttpHeaderEntry *e = getEntry(&pos)) {
270  if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
271  delAt(pos, count);
272  }
273 }
274 
275 bool
277 {
278  // RFC 7234, section 4.3.4: use fields other from Warning for update
279  return id == Http::HdrType::WARNING;
280 }
281 
282 bool
284 {
285  assert(fresh);
286  assert(this != fresh);
287 
288  // Optimization: Finding whether a header field changed is expensive
289  // and probably not worth it except for collapsed revalidation needs.
291  return false;
292 
293  updateWarnings();
294 
295  const HttpHeaderEntry *e;
297 
298  while ((e = fresh->getEntry(&pos))) {
299  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
300 
301  if (skipUpdateHeader(e->id))
302  continue;
303 
304  if (e->id != Http::HdrType::OTHER)
305  delById(e->id);
306  else
307  delByName(e->name);
308  }
309 
310  pos = HttpHeaderInitPos;
311  while ((e = fresh->getEntry(&pos))) {
312  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
313 
314  if (skipUpdateHeader(e->id))
315  continue;
316 
317  debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
318 
319  addEntry(e->clone());
320  }
321  return true;
322 }
323 
324 bool
325 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
326 {
327  /*
328  * parse_start points to the first line of HTTP message *headers*,
329  * not including the request or status lines
330  */
331  const size_t end = headersEnd(*parse_start, l);
332 
333  if (end) {
334  *blk_start = *parse_start;
335  *blk_end = *parse_start + end - 1;
336  assert(**blk_end == '\n');
337  // Point blk_end to the first character after the last header field.
338  // In other words, blk_end should point to the CR?LF header terminator.
339  if (end > 1 && *(*blk_end - 1) == '\r')
340  --(*blk_end);
341  *parse_start += end;
342  }
343  return end;
344 }
345 
346 int
347 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz, Http::ContentLengthInterpreter &clen)
348 {
349  const char *parse_start = buf;
350  const char *blk_start, *blk_end;
351  hdr_sz = 0;
352 
353  if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
354  // XXX: do not parse non-isolated headers even if the connection is closed.
355  // Treat unterminated headers as "partial headers" framing errors.
356  if (!atEnd)
357  return 0;
358  blk_start = parse_start;
359  blk_end = blk_start + strlen(blk_start);
360  }
361 
362  if (parse(blk_start, blk_end - blk_start, clen)) {
363  hdr_sz = parse_start - buf;
364  return 1;
365  }
366  return -1;
367 }
368 
369 int
370 HttpHeader::parse(const char *header_start, size_t hdrLen, Http::ContentLengthInterpreter &clen)
371 {
372  const char *field_ptr = header_start;
373  const char *header_end = header_start + hdrLen; // XXX: remove
374  int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
375 
376  PROF_start(HttpHeaderParse);
377 
378  assert(header_start && header_end);
379  debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
380  ++ HttpHeaderStats[owner].parsedCount;
381 
382  char *nulpos;
383  if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
384  debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
385  getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
386  PROF_stop(HttpHeaderParse);
387  clean();
388  return 0;
389  }
390 
391  /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
392  * continuation lines start with a (single) space or tab */
393  while (field_ptr < header_end) {
394  const char *field_start = field_ptr;
395  const char *field_end;
396 
397  do {
398  const char *this_line = field_ptr;
399  field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
400 
401  if (!field_ptr) {
402  // missing <LF>
403  PROF_stop(HttpHeaderParse);
404  clean();
405  return 0;
406  }
407 
408  field_end = field_ptr;
409 
410  ++field_ptr; /* Move to next line */
411 
412  if (field_end > this_line && field_end[-1] == '\r') {
413  --field_end; /* Ignore CR LF */
414 
415  if (owner == hoRequest && field_end > this_line) {
416  bool cr_only = true;
417  for (const char *p = this_line; p < field_end && cr_only; ++p) {
418  if (*p != '\r')
419  cr_only = false;
420  }
421  if (cr_only) {
422  debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
423  "header field to prevent request smuggling attacks: {" <<
424  getStringPrefix(header_start, hdrLen) << "}");
425  PROF_stop(HttpHeaderParse);
426  clean();
427  return 0;
428  }
429  }
430  }
431 
432  /* Barf on stray CR characters */
433  if (memchr(this_line, '\r', field_end - this_line)) {
434  debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
435  getStringPrefix(field_start, field_end-field_start) << "}");
436 
438  char *p = (char *) this_line; /* XXX Warning! This destroys original header content and violates specifications somewhat */
439 
440  while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
441  *p = ' ';
442  ++p;
443  }
444  } else {
445  PROF_stop(HttpHeaderParse);
446  clean();
447  return 0;
448  }
449  }
450 
451  if (this_line + 1 == field_end && this_line > field_start) {
452  debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
453  getStringPrefix(header_start, hdrLen) << "}");
454  PROF_stop(HttpHeaderParse);
455  clean();
456  return 0;
457  }
458  } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
459 
460  if (field_start == field_end) {
461  if (field_ptr < header_end) {
462  debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
463  getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
464  PROF_stop(HttpHeaderParse);
465  clean();
466  return 0;
467  }
468 
469  break; /* terminating blank line */
470  }
471 
472  HttpHeaderEntry *e;
473  if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
474  debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
475  getStringPrefix(field_start, field_end-field_start) << "}");
476  debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
477 
479  continue;
480 
481  PROF_stop(HttpHeaderParse);
482  clean();
483  return 0;
484  }
485 
486  if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
487  delete e;
488 
490  continue; // clen has printed any necessary warnings
491 
492  PROF_stop(HttpHeaderParse);
493  clean();
494  return 0;
495  }
496 
497  /* AYJ 2017-05-23: I suspect we need to change this whitespace check to conform to the
498  updated WSP character set in RFC 7230/7231. For now I left it as the
499  characters in w_space which the previous code was using. */
501  if (e->id == Http::HdrType::OTHER && e->name.findFirstOf(wsp) != SBuf::npos) {
502  debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
503  getStringPrefix(field_start, field_end-field_start) << "}");
504 
506  delete e;
507  PROF_stop(HttpHeaderParse);
508  clean();
509  return 0;
510  }
511  }
512 
513  addEntry(e);
514  }
515 
516  if (clen.headerWideProblem) {
517  debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
518  " Content-Length field values in" <<
519  Raw("header", header_start, hdrLen));
520  }
521 
522  if (clen.prohibitedAndIgnored()) {
523  // RFC 7230 section 3.3.2: A server MUST NOT send a Content-Length
524  // header field in any response with a status code of 1xx (Informational)
525  // or 204 (No Content). And RFC 7230 3.3.3#1 tells recipients to ignore
526  // such Content-Lengths.
528  debugs(55, 3, "Content-Length is " << clen.prohibitedAndIgnored());
529  } else if (chunked()) {
530  // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
531  // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
533  // and clen state becomes irrelevant
534  } else if (clen.sawBad) {
535  // ensure our callers do not accidentally see bad Content-Length values
537  conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
538  } else if (clen.needsSanitizing) {
539  // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
540  // ensure our callers see a clean Content-Length value or none at all
542  if (clen.sawGood) {
544  debugs(55, 5, "sanitized Content-Length to be " << clen.value);
545  }
546  }
547 
548  PROF_stop(HttpHeaderParse);
549  return 1; /* even if no fields where found, it is a valid header */
550 }
551 
552 /* packs all the entries using supplied packer */
553 void
554 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
555 {
557  const HttpHeaderEntry *e;
558  assert(p);
559  debugs(55, 7, this << " into " << p <<
560  (mask_sensitive_info ? " while masking" : ""));
561  /* pack all entries one by one */
562  while ((e = getEntry(&pos))) {
563  if (!mask_sensitive_info) {
564  e->packInto(p);
565  continue;
566  }
567 
568  bool maskThisEntry = false;
569  switch (e->id) {
572  maskThisEntry = true;
573  break;
574 
577  maskThisEntry = (cmd->value == "PASS");
578  break;
579 
580  default:
581  break;
582  }
583  if (maskThisEntry) {
584  p->append(e->name.rawContent(), e->name.length());
585  p->append(": ** NOT DISPLAYED **\r\n", 23);
586  } else {
587  e->packInto(p);
588  }
589 
590  }
591  /* Pack in the "special" entries */
592 
593  /* Cache-Control */
594 }
595 
596 /* returns next valid entry */
599 {
600  assert(pos);
601  assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
602 
603  for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
604  if (entries[*pos])
605  return static_cast<HttpHeaderEntry*>(entries[*pos]);
606  }
607 
608  return NULL;
609 }
610 
611 /*
612  * returns a pointer to a specified entry if any
613  * note that we return one entry so it does not make much sense to ask for
614  * "list" headers
615  */
618 {
620  assert(!Http::HeaderLookupTable.lookup(id).list);
621 
622  /* check mask first */
623 
624  if (!CBIT_TEST(mask, id))
625  return NULL;
626 
627  /* looks like we must have it, do linear search */
628  for (auto e : entries) {
629  if (e && e->id == id)
630  return e;
631  }
632 
633  /* hm.. we thought it was there, but it was not found */
634  assert(false);
635  return nullptr; /* not reached */
636 }
637 
638 /*
639  * same as httpHeaderFindEntry
640  */
643 {
645  assert(!Http::HeaderLookupTable.lookup(id).list);
646 
647  /* check mask first */
648  if (!CBIT_TEST(mask, id))
649  return NULL;
650 
651  for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
652  if (*e && (*e)->id == id)
653  return *e;
654  }
655 
656  /* hm.. we thought it was there, but it was not found */
657  assert(false);
658  return nullptr; /* not reached */
659 }
660 
661 int
663 {
664  int count = 0;
666  httpHeaderMaskInit(&mask, 0); /* temporal inconsistency */
667  debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
668 
669  while (const HttpHeaderEntry *e = getEntry(&pos)) {
670  if (!e->name.caseCmp(name))
671  delAt(pos, count);
672  else
673  CBIT_SET(mask, e->id);
674  }
675 
676  return count;
677 }
678 
679 /* deletes all entries with a given id, returns the #entries deleted */
680 int
682 {
683  debugs(55, 8, this << " del-by-id " << id);
685 
686  if (!CBIT_TEST(mask, id))
687  return 0;
688 
689  int count = 0;
690 
692  while (HttpHeaderEntry *e = getEntry(&pos)) {
693  if (e->id == id)
694  delAt(pos, count); // deletes e
695  }
696 
697  CBIT_CLR(mask, id);
698  assert(count);
699  return count;
700 }
701 
702 /*
703  * deletes an entry at pos and leaves a gap; leaving a gap makes it
704  * possible to iterate(search) and delete fields at the same time
705  * NOTE: Does not update the header mask. Caller must follow up with
706  * a call to refreshMask() if headers_deleted was incremented.
707  */
708 void
709 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
710 {
711  HttpHeaderEntry *e;
712  assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
713  e = static_cast<HttpHeaderEntry*>(entries[pos]);
714  entries[pos] = NULL;
715  /* decrement header length, allow for ": " and crlf */
716  len -= e->name.length() + 2 + e->value.size() + 2;
717  assert(len >= 0);
718  delete e;
719  ++headers_deleted;
720 }
721 
722 /*
723  * Compacts the header storage
724  */
725 void
727 {
728  // TODO: optimize removal, or possibly make it so that's not needed.
729  entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
730  entries.end());
731 }
732 
733 /*
734  * Refreshes the header mask. Required after delAt() calls.
735  */
736 void
738 {
740  debugs(55, 7, "refreshing the mask in hdr " << this);
741  for (auto e : entries) {
742  if (e)
743  CBIT_SET(mask, e->id);
744  }
745 }
746 
747 /* appends an entry;
748  * does not call e->clone() so one should not reuse "*e"
749  */
750 void
752 {
753  assert(e);
755  assert(e->name.length());
756 
757  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
758 
759  if (e->id != Http::HdrType::BAD_HDR) {
760  if (CBIT_TEST(mask, e->id)) {
761  ++ headerStatsTable[e->id].repCount;
762  } else {
763  CBIT_SET(mask, e->id);
764  }
765  }
766 
767  entries.push_back(e);
768 
769  /* increment header length, allow for ": " and crlf */
770  len += e->name.length() + 2 + e->value.size() + 2;
771 }
772 
773 /* inserts an entry;
774  * does not call e->clone() so one should not reuse "*e"
775  */
776 void
778 {
779  assert(e);
781 
782  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
783 
784  // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
785  if (CBIT_TEST(mask, e->id)) {
786  ++ headerStatsTable[e->id].repCount;
787  } else {
788  CBIT_SET(mask, e->id);
789  }
790 
791  entries.insert(entries.begin(),e);
792 
793  /* increment header length, allow for ": " and crlf */
794  len += e->name.length() + 2 + e->value.size() + 2;
795 }
796 
797 bool
799 {
800  debugs(55, 9, this << " joining for id " << id);
801  /* only fields from ListHeaders array can be "listed" */
802  assert(Http::HeaderLookupTable.lookup(id).list);
803 
804  if (!CBIT_TEST(mask, id))
805  return false;
806 
807  for (auto e: entries) {
808  if (e && e->id == id)
809  strListAdd(s, e->value.termedBuf(), ',');
810  }
811 
812  /*
813  * note: we might get an empty (size==0) string if there was an "empty"
814  * header. This results in an empty length String, which may have a NULL
815  * buffer.
816  */
817  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
818  if (!s->size())
819  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
820  else
821  debugs(55, 6, this << ": joined for id " << id << ": " << s);
822 
823  return true;
824 }
825 
826 /* return a list of entries with the same id separated by ',' and ws */
827 String
829 {
830  HttpHeaderEntry *e;
832  debugs(55, 9, this << "joining for id " << id);
833  /* only fields from ListHeaders array can be "listed" */
834  assert(Http::HeaderLookupTable.lookup(id).list);
835 
836  if (!CBIT_TEST(mask, id))
837  return String();
838 
839  String s;
840 
841  while ((e = getEntry(&pos))) {
842  if (e->id == id)
843  strListAdd(&s, e->value.termedBuf(), ',');
844  }
845 
846  /*
847  * note: we might get an empty (size==0) string if there was an "empty"
848  * header. This results in an empty length String, which may have a NULL
849  * buffer.
850  */
851  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
852  if (!s.size())
853  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
854  else
855  debugs(55, 6, this << ": joined for id " << id << ": " << s);
856 
857  return s;
858 }
859 
860 /* return a string or list of entries with the same id separated by ',' and ws */
861 String
863 {
864  HttpHeaderEntry *e;
865 
866  if (Http::HeaderLookupTable.lookup(id).list)
867  return getList(id);
868 
869  if ((e = findEntry(id)))
870  return e->value;
871 
872  return String();
873 }
874 
875 /*
876  * Returns the value of the specified header and/or an undefined String.
877  */
878 String
879 HttpHeader::getByName(const char *name) const
880 {
881  String result;
882  // ignore presence: return undefined string if an empty header is present
883  (void)hasNamed(name, strlen(name), &result);
884  return result;
885 }
886 
887 String
888 HttpHeader::getByName(const SBuf &name) const
889 {
890  String result;
891  // ignore presence: return undefined string if an empty header is present
892  (void)hasNamed(name, &result);
893  return result;
894 }
895 
896 String
898 {
899  String result;
900  (void)getByIdIfPresent(id, &result);
901  return result;
902 }
903 
904 bool
905 HttpHeader::hasNamed(const SBuf &s, String *result) const
906 {
907  return hasNamed(s.rawContent(), s.length(), result);
908 }
909 
910 bool
912 {
913  if (id == Http::HdrType::BAD_HDR)
914  return false;
915  if (!has(id))
916  return false;
917  if (result)
918  *result = getStrOrList(id);
919  return true;
920 }
921 
922 bool
923 HttpHeader::hasNamed(const char *name, unsigned int namelen, String *result) const
924 {
925  Http::HdrType id;
927  HttpHeaderEntry *e;
928 
929  assert(name);
930 
931  /* First try the quick path */
932  id = Http::HeaderLookupTable.lookup(name,namelen).id;
933 
934  if (id != Http::HdrType::BAD_HDR) {
935  if (getByIdIfPresent(id, result))
936  return true;
937  }
938 
939  /* Sorry, an unknown header name. Do linear search */
940  bool found = false;
941  while ((e = getEntry(&pos))) {
942  if (e->id == Http::HdrType::OTHER && e->name.length() == namelen && e->name.caseCmp(name, namelen) == 0) {
943  found = true;
944  if (!result)
945  break;
946  strListAdd(result, e->value.termedBuf(), ',');
947  }
948  }
949 
950  return found;
951 }
952 
953 /*
954  * Returns a the value of the specified list member, if any.
955  */
956 String
957 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
958 {
959  String header;
960  const char *pos = NULL;
961  const char *item;
962  int ilen;
963  int mlen = strlen(member);
964 
965  assert(name);
966 
967  header = getByName(name);
968 
969  String result;
970 
971  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
972  if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
973  result.append(item + mlen + 1, ilen - mlen - 1);
974  break;
975  }
976  }
977 
978  return result;
979 }
980 
981 /*
982  * returns a the value of the specified list member, if any.
983  */
984 String
985 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
986 {
987  String header;
988  const char *pos = NULL;
989  const char *item;
990  int ilen;
991  int mlen = strlen(member);
992 
994 
995  header = getStrOrList(id);
996  String result;
997 
998  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
999  if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1000  result.append(item + mlen + 1, ilen - mlen - 1);
1001  break;
1002  }
1003  }
1004 
1005  header.clean();
1006  return result;
1007 }
1008 
1009 /* test if a field is present */
1010 int
1012 {
1014  debugs(55, 9, this << " lookup for " << id);
1015  return CBIT_TEST(mask, id);
1016 }
1017 
1018 void
1020 {
1021  // TODO: do not add Via header for messages where Squid itself
1022  // generated the message (i.e., Downloader or ESI) there should be no Via header added at all.
1023 
1024  if (Config.onoff.via) {
1025  SBuf buf;
1026  // RFC 7230 section 5.7.1.: protocol-name is omitted when
1027  // the received protocol is HTTP.
1030  buf.appendf("%s/", AnyP::ProtocolType_str[ver.protocol]);
1031  buf.appendf("%d.%d %s", ver.major, ver.minor, ThisCache);
1032  const HttpHeader *hdr = from ? from : this;
1033  SBuf strVia = StringToSBuf(hdr->getList(Http::HdrType::VIA));
1034  if (!strVia.isEmpty())
1035  strVia.append(", ", 2);
1036  strVia.append(buf);
1037  // XXX: putStr() still suffers from String size limits
1038  Must(strVia.length() < String::SizeMaxXXX());
1040  putStr(Http::HdrType::VIA, strVia.c_str());
1041  }
1042 }
1043 
1044 void
1046 {
1048  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1049  assert(number >= 0);
1050  addEntry(new HttpHeaderEntry(id, SBuf(), xitoa(number)));
1051 }
1052 
1053 void
1055 {
1057  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1058  assert(number >= 0);
1059  addEntry(new HttpHeaderEntry(id, SBuf(), xint64toa(number)));
1060 }
1061 
1062 void
1064 {
1066  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1067  assert(htime >= 0);
1068  addEntry(new HttpHeaderEntry(id, SBuf(), mkrfc1123(htime)));
1069 }
1070 
1071 void
1072 HttpHeader::putStr(Http::HdrType id, const char *str)
1073 {
1075  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1076  assert(str);
1077  addEntry(new HttpHeaderEntry(id, SBuf(), str));
1078 }
1079 
1080 void
1081 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1082 {
1083  assert(auth_scheme && realm);
1084  httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1085 }
1086 
1087 void
1089 {
1090  assert(cc);
1091  /* remove old directives if any */
1093  /* pack into mb */
1094  MemBuf mb;
1095  mb.init();
1096  cc->packInto(&mb);
1097  /* put */
1099  /* cleanup */
1100  mb.clean();
1101 }
1102 
1103 void
1105 {
1106  assert(cr);
1107  /* remove old directives if any */
1109  /* pack into mb */
1110  MemBuf mb;
1111  mb.init();
1112  httpHdrContRangePackInto(cr, &mb);
1113  /* put */
1115  /* cleanup */
1116  mb.clean();
1117 }
1118 
1119 void
1121 {
1122  assert(range);
1123  /* remove old directives if any */
1125  /* pack into mb */
1126  MemBuf mb;
1127  mb.init();
1128  range->packInto(&mb);
1129  /* put */
1131  /* cleanup */
1132  mb.clean();
1133 }
1134 
1135 void
1137 {
1138  assert(sc);
1139  /* remove old directives if any */
1141  /* pack into mb */
1142  MemBuf mb;
1143  mb.init();
1144  sc->packInto(&mb);
1145  /* put */
1147  /* cleanup */
1148  mb.clean();
1149 }
1150 
1151 void
1152 HttpHeader::putWarning(const int code, const char *const text)
1153 {
1154  char buf[512];
1155  snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1157 }
1158 
1159 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1160 void
1161 HttpHeader::putExt(const char *name, const char *value)
1162 {
1163  assert(name && value);
1164  debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1165  addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, SBuf(name), value));
1166 }
1167 
1168 int
1170 {
1172  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1173  HttpHeaderEntry *e;
1174 
1175  if ((e = findEntry(id)))
1176  return e->getInt();
1177 
1178  return -1;
1179 }
1180 
1181 int64_t
1183 {
1185  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1186  HttpHeaderEntry *e;
1187 
1188  if ((e = findEntry(id)))
1189  return e->getInt64();
1190 
1191  return -1;
1192 }
1193 
1194 time_t
1196 {
1197  HttpHeaderEntry *e;
1198  time_t value = -1;
1200  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1201 
1202  if ((e = findEntry(id))) {
1203  value = parse_rfc1123(e->value.termedBuf());
1204  httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1205  }
1206 
1207  return value;
1208 }
1209 
1210 /* sync with httpHeaderGetLastStr */
1211 const char *
1213 {
1214  HttpHeaderEntry *e;
1216  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1217 
1218  if ((e = findEntry(id))) {
1219  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1220  return e->value.termedBuf();
1221  }
1222 
1223  return NULL;
1224 }
1225 
1226 /* unusual */
1227 const char *
1229 {
1230  HttpHeaderEntry *e;
1232  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1233 
1234  if ((e = findLastEntry(id))) {
1235  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1236  return e->value.termedBuf();
1237  }
1238 
1239  return NULL;
1240 }
1241 
1242 HttpHdrCc *
1244 {
1246  return NULL;
1247  PROF_start(HttpHeader_getCc);
1248 
1249  String s;
1251 
1252  HttpHdrCc *cc=new HttpHdrCc();
1253 
1254  if (!cc->parse(s)) {
1255  delete cc;
1256  cc = NULL;
1257  }
1258 
1259  ++ HttpHeaderStats[owner].ccParsedCount;
1260 
1261  if (cc)
1262  httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1263 
1265 
1266  PROF_stop(HttpHeader_getCc);
1267 
1268  return cc;
1269 }
1270 
1271 HttpHdrRange *
1273 {
1274  HttpHdrRange *r = NULL;
1275  HttpHeaderEntry *e;
1276  /* some clients will send "Request-Range" _and_ *matching* "Range"
1277  * who knows, some clients might send Request-Range only;
1278  * this "if" should work correctly in both cases;
1279  * hopefully no clients send mismatched headers! */
1280 
1281  if ((e = findEntry(Http::HdrType::RANGE)) ||
1284  httpHeaderNoteParsedEntry(e->id, e->value, !r);
1285  }
1286 
1287  return r;
1288 }
1289 
1290 HttpHdrSc *
1292 {
1294  return NULL;
1295 
1296  String s;
1297 
1299 
1301 
1302  ++ HttpHeaderStats[owner].ccParsedCount;
1303 
1304  if (sc)
1305  sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1306 
1308 
1309  return sc;
1310 }
1311 
1314 {
1315  HttpHdrContRange *cr = NULL;
1316  HttpHeaderEntry *e;
1317 
1320  httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1321  }
1322 
1323  return cr;
1324 }
1325 
1326 const char *
1327 HttpHeader::getAuth(Http::HdrType id, const char *auth_scheme) const
1328 {
1329  const char *field;
1330  int l;
1331  assert(auth_scheme);
1332  field = getStr(id);
1333 
1334  if (!field) /* no authorization field */
1335  return NULL;
1336 
1337  l = strlen(auth_scheme);
1338 
1339  if (!l || strncasecmp(field, auth_scheme, l)) /* wrong scheme */
1340  return NULL;
1341 
1342  field += l;
1343 
1344  if (!xisspace(*field)) /* wrong scheme */
1345  return NULL;
1346 
1347  /* skip white space */
1348  for (; field && xisspace(*field); ++field);
1349 
1350  if (!*field) /* no authorization cookie */
1351  return NULL;
1352 
1353  static char decodedAuthToken[8192];
1354  struct base64_decode_ctx ctx;
1355  base64_decode_init(&ctx);
1356  size_t decodedLen = 0;
1357  if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), field) ||
1358  !base64_decode_final(&ctx)) {
1359  return NULL;
1360  }
1361  decodedAuthToken[decodedLen] = '\0';
1362  return decodedAuthToken;
1363 }
1364 
1365 ETag
1367 {
1368  ETag etag = {NULL, -1};
1369  HttpHeaderEntry *e;
1370  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag); /* must be of an appropriate type */
1371 
1372  if ((e = findEntry(id)))
1373  etagParseInit(&etag, e->value.termedBuf());
1374 
1375  return etag;
1376 }
1377 
1378 TimeOrTag
1380 {
1381  TimeOrTag tot;
1382  HttpHeaderEntry *e;
1383  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag); /* must be of an appropriate type */
1384  memset(&tot, 0, sizeof(tot));
1385 
1386  if ((e = findEntry(id))) {
1387  const char *str = e->value.termedBuf();
1388  /* try as an ETag */
1389 
1390  if (etagParseInit(&tot.tag, str)) {
1391  tot.valid = tot.tag.str != NULL;
1392  tot.time = -1;
1393  } else {
1394  /* or maybe it is time? */
1395  tot.time = parse_rfc1123(str);
1396  tot.valid = tot.time >= 0;
1397  tot.tag.str = NULL;
1398  }
1399  }
1400 
1401  assert(tot.time < 0 || !tot.tag.str); /* paranoid */
1402  return tot;
1403 }
1404 
1405 /*
1406  * HttpHeaderEntry
1407  */
1408 
1409 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const SBuf &aName, const char *aValue)
1410 {
1412  id = anId;
1413 
1414  if (id != Http::HdrType::OTHER)
1416  else
1417  name = aName;
1418 
1419  value = aValue;
1420 
1421  if (id != Http::HdrType::BAD_HDR)
1422  ++ headerStatsTable[id].aliveCount;
1423 
1424  debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1425 }
1426 
1428 {
1429  debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1430 
1431  if (id != Http::HdrType::BAD_HDR) {
1432  assert(headerStatsTable[id].aliveCount);
1433  -- headerStatsTable[id].aliveCount;
1434  id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1435  }
1436 
1437 }
1438 
1439 /* parses and inits header entry, returns true/false */
1441 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1442 {
1443  /* note: name_start == field_start */
1444  const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1445  int name_len = name_end ? name_end - field_start :0;
1446  const char *value_start = field_start + name_len + 1; /* skip ':' */
1447  /* note: value_end == field_end */
1448 
1450 
1451  /* do we have a valid field name within this field? */
1452 
1453  if (!name_len || name_end > field_end)
1454  return NULL;
1455 
1456  if (name_len > 65534) {
1457  /* String must be LESS THAN 64K and it adds a terminating NULL */
1458  debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1459  return NULL;
1460  }
1461 
1462  if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1463  debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1464  "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1465 
1466  while (name_len > 0 && xisspace(field_start[name_len - 1]))
1467  --name_len;
1468 
1469  if (!name_len)
1470  return NULL;
1471  }
1472 
1473  /* now we know we can parse it */
1474 
1475  debugs(55, 9, "parsing HttpHeaderEntry: near '" << getStringPrefix(field_start, field_end-field_start) << "'");
1476 
1477  /* is it a "known" field? */
1478  Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1479  debugs(55, 9, "got hdr-id=" << id);
1480 
1481  SBuf theName;
1482 
1483  String value;
1484 
1485  if (id == Http::HdrType::BAD_HDR)
1486  id = Http::HdrType::OTHER;
1487 
1488  /* set field name */
1489  if (id == Http::HdrType::OTHER)
1490  theName.append(field_start, name_len);
1491  else
1492  theName = Http::HeaderLookupTable.lookup(id).name;
1493 
1494  /* trim field value */
1495  while (value_start < field_end && xisspace(*value_start))
1496  ++value_start;
1497 
1498  while (value_start < field_end && xisspace(field_end[-1]))
1499  --field_end;
1500 
1501  if (field_end - value_start > 65534) {
1502  /* String must be LESS THAN 64K and it adds a terminating NULL */
1503  debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << theName << "' header of " << (field_end - value_start) << " bytes");
1504  return NULL;
1505  }
1506 
1507  /* set field value */
1508  value.limitInit(value_start, field_end - value_start);
1509 
1510  if (id != Http::HdrType::BAD_HDR)
1511  ++ headerStatsTable[id].seenCount;
1512 
1513  debugs(55, 9, "parsed HttpHeaderEntry: '" << theName << ": " << value << "'");
1514 
1515  return new HttpHeaderEntry(id, theName, value.termedBuf());
1516 }
1517 
1520 {
1521  return new HttpHeaderEntry(id, name, value.termedBuf());
1522 }
1523 
1524 void
1526 {
1527  assert(p);
1528  p->append(name.rawContent(), name.length());
1529  p->append(": ", 2);
1530  p->append(value.rawBuf(), value.size());
1531  p->append("\r\n", 2);
1532 }
1533 
1534 int
1536 {
1537  int val = -1;
1538  int ok = httpHeaderParseInt(value.termedBuf(), &val);
1539  httpHeaderNoteParsedEntry(id, value, ok == 0);
1540  /* XXX: Should we check ok - ie
1541  * return ok ? -1 : value;
1542  */
1543  return val;
1544 }
1545 
1546 int64_t
1548 {
1549  int64_t val = -1;
1550  const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1551  httpHeaderNoteParsedEntry(id, value, !ok);
1552  return val; // remains -1 if !ok (XXX: bad method API)
1553 }
1554 
1555 static void
1557 {
1558  if (id != Http::HdrType::BAD_HDR)
1559  ++ headerStatsTable[id].parsCount;
1560 
1561  if (error) {
1562  if (id != Http::HdrType::BAD_HDR)
1563  ++ headerStatsTable[id].errCount;
1564  debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1565  }
1566 }
1567 
1568 /*
1569  * Reports
1570  */
1571 
1572 /* tmp variable used to pass stat info to dumpers */
1573 extern const HttpHeaderStat *dump_stat; /* argh! */
1574 const HttpHeaderStat *dump_stat = NULL;
1575 
1576 void
1577 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1578 {
1579  const int id = static_cast<int>(val);
1580  const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1581  const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1582  int visible = count > 0;
1583  /* for entries with zero count, list only those that belong to current type of message */
1584 
1585  if (!visible && valid_id && dump_stat->owner_mask)
1586  visible = CBIT_TEST(*dump_stat->owner_mask, id);
1587 
1588  if (visible)
1589  storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1590  id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1591 }
1592 
1593 static void
1594 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1595 {
1596  if (count)
1597  storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1598  idx, (int) val, count,
1599  xpercent(count, dump_stat->destroyedCount));
1600 }
1601 
1602 static void
1604 {
1605  assert(hs);
1606  assert(e);
1607 
1608  dump_stat = hs;
1609  storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1610  storeAppendPrintf(e, "\nField type distribution\n");
1611  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1612  "id", "name", "count", "#/header");
1614  storeAppendPrintf(e, "\nCache-control directives distribution\n");
1615  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1616  "id", "name", "count", "#/cc_field");
1618  storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1619  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1620  "id", "name", "count", "#/sc_field");
1622  storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1623  storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1624  "id", "#flds", "count", "%total");
1626  storeAppendPrintf(e, "\n");
1627  dump_stat = NULL;
1628 }
1629 
1630 void
1632 {
1633  int i;
1634  assert(e);
1635 
1636  HttpHeaderStats[0].parsedCount =
1637  HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1638  HttpHeaderStats[0].ccParsedCount =
1639  HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1640  HttpHeaderStats[0].destroyedCount =
1641  HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1642  HttpHeaderStats[0].busyDestroyedCount =
1643  HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1644 
1645  for (i = 1; i < HttpHeaderStatCount; ++i) {
1646  httpHeaderStatDump(HttpHeaderStats + i, e);
1647  }
1648 
1649  /* field stats for all messages */
1650  storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1651 
1652  storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1653  "id", "name", "#alive", "%err", "%repeat");
1654 
1655  // scan heaaderTable and output
1656  for (auto h : WholeEnum<Http::HdrType>()) {
1657  auto stats = headerStatsTable[h];
1658  storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1659  Http::HeaderLookupTable.lookup(h).id,
1660  Http::HeaderLookupTable.lookup(h).name,
1661  stats.aliveCount,
1662  xpercent(stats.errCount, stats.parsCount),
1663  xpercent(stats.repCount, stats.seenCount));
1664  }
1665 
1666  storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1667  HttpHeaderStats[hoRequest].parsedCount,
1668  HttpHeaderStats[hoReply].parsedCount,
1669  HttpHeaderStats[0].parsedCount);
1670  storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1671 }
1672 
1673 int
1674 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1675 {
1676  int result = 0;
1677  const char *pos = NULL;
1678  const char *item;
1679  int ilen;
1680  int mlen = strlen(member);
1681 
1683 
1684  String header (getStrOrList(id));
1685 
1686  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1687  if (strncasecmp(item, member, mlen) == 0
1688  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1689  result = 1;
1690  break;
1691  }
1692  }
1693 
1694  return result;
1695 }
1696 
1697 int
1698 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1699 {
1700  int result = 0;
1701  const char *pos = NULL;
1702  const char *item;
1703  int ilen;
1704  int mlen = strlen(member);
1705 
1706  assert(name);
1707 
1708  String header (getByName(name));
1709 
1710  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1711  if (strncasecmp(item, member, mlen) == 0
1712  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1713  result = 1;
1714  break;
1715  }
1716  }
1717 
1718  return result;
1719 }
1720 
1721 void
1723 {
1725 
1726  const HttpHeaderEntry *e;
1728  int headers_deleted = 0;
1729  while ((e = getEntry(&pos))) {
1730  Http::HdrType id = e->id;
1731  if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1732  delAt(pos, headers_deleted);
1733  CBIT_CLR(mask, id);
1734  }
1735  }
1736 }
1737 
1738 void
1740 {
1742  /* anything that matches Connection list member will be deleted */
1743  String strConnection;
1744 
1745  (void) getList(Http::HdrType::CONNECTION, &strConnection);
1746  const HttpHeaderEntry *e;
1748  /*
1749  * think: on-average-best nesting of the two loops (hdrEntry
1750  * and strListItem) @?@
1751  */
1752  /*
1753  * maybe we should delete standard stuff ("keep-alive","close")
1754  * from strConnection first?
1755  */
1756 
1757  int headers_deleted = 0;
1758  while ((e = getEntry(&pos))) {
1759  if (strListIsMember(&strConnection, e->name, ','))
1760  delAt(pos, headers_deleted);
1761  }
1762  if (headers_deleted)
1763  refreshMask();
1764  }
1765 }
1766 
int delById(Http::HdrType id)
Definition: HttpHeader.cc:681
int caseCmp(const SBuf &S, const size_type n) const
shorthand version for case-insensitive compare()
Definition: SBuf.h:272
HttpHdrSc * httpHdrScParseCreate(const String &str)
Definition: HttpHdrSc.cc:60
void removeConnectionHeaderEntries()
Definition: HttpHeader.cc:1739
int has(Http::HdrType id) const
Definition: HttpHeader.cc:1011
bool httpHeaderParseOffset(const char *start, int64_t *value, char **endPtr)
void count(double val)
Definition: StatHist.cc:57
void httpHeaderFieldStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHeader.cc:1577
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:77
void limitInit(const char *str, int len)
Definition: String.cc:94
bool needUpdate(const HttpHeader *fresh) const
check whether the fresh header has any new/changed updatable fields
Definition: HttpHeader.cc:249
void httpHdrScInitModule(void)
Definition: HttpHdrSc.cc:49
#define assert(EX)
Definition: assert.h:17
String getListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:985
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:239
time_t parse_rfc1123(const char *str)
Definition: rfc1123.c:159
const char * prohibitedAndIgnored() const
static int sc[16]
Definition: smbdes.c:121
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
const char * mkrfc1123(time_t)
Definition: rfc1123.c:202
http_hdr_owner_type
Definition: HttpHeader.h:31
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1243
SQUIDCEXTERN double xpercent(double part, double whole)
Definition: util.c:54
#define CBIT_TEST(mask, bit)
Definition: defines.h:114
static const CharacterSet LF
Definition: CharacterSet.h:89
SQUIDCEXTERN const char * xitoa(int num)
Definition: util.c:79
bool hasNamed(const SBuf &s, String *value=0) const
Definition: HttpHeader.cc:905
Definition: ETag.h:17
std::vector< HttpHeaderEntry *, PoolingAllocator< HttpHeaderEntry * > > entries
Definition: HttpHeader.h:156
int type
Definition: errorpage.cc:78
Definition: SBuf.h:86
void putContRange(const HttpHdrContRange *cr)
Definition: HttpHeader.cc:1104
int collapsed_forwarding
Definition: SquidConfig.h:326
int etagParseInit(ETag *etag, const char *str)
Definition: ETag.cc:29
bool any_HdrType_enum_value(const Http::HdrType id)
match any known header type, including OTHER and BAD
time_t time
Definition: TimeOrTag.h:21
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
void error(char *format,...)
int delByName(const SBuf &name)
Definition: HttpHeader.cc:662
int i
Definition: membanger.c:49
void removeHopByHopEntries()
Definition: HttpHeader.cc:1722
SBuf & append(const SBuf &S)
Definition: SBuf.cc:195
static bool Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
Definition: HttpHeader.cc:325
HttpHeaderMask mask
Definition: HttpHeader.h:157
void putExt(const char *name, const char *value)
Definition: HttpHeader.cc:1161
class Ping::pingStats_ stats
void base64_decode_init(struct base64_decode_ctx *ctx)
Definition: base64.c:54
void httpHdrCcStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrCc.cc:330
static size_type SizeMaxXXX()
Definition: SquidString.h:70
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:598
void strListAdd(String *str, const char *item, char del)
Definition: StrList.cc:19
bool conflictingContentLength_
found different Content-Length fields
Definition: HttpHeader.h:177
int64_t getInt64() const
Definition: HttpHeader.cc:1547
#define xisspace(x)
Definition: xis.h:17
HttpHdrRange * getRange() const
Definition: HttpHeader.cc:1272
bool isEmpty() const
Definition: SBuf.h:420
#define DBG_CRITICAL
Definition: Debug.h:45
char * p
Definition: membanger.c:43
int getInt() const
Definition: HttpHeader.cc:1535
String getByName(const SBuf &name) const
Definition: HttpHeader.cc:888
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:37
int getInt(Http::HdrType id) const
Definition: HttpHeader.cc:1169
size_type size() const
Definition: SquidString.h:72
void append(char const *buf, int len)
Definition: String.cc:161
virtual void append(const char *buf, int size)=0
Appends a c-string to existing packed data.
#define CBIT_SET(mask, bit)
Definition: defines.h:112
static void httpHeaderFldsPerHdrDumper(StoreEntry *sentry, int idx, double val, double, int count)
Definition: HttpHeader.cc:1594
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
ETag tag
Definition: TimeOrTag.h:20
bool skipUpdateHeader(const Http::HdrType id) const
Definition: HttpHeader.cc:276
static const CharacterSet WSP
Definition: CharacterSet.h:95
Http::HdrType id
Definition: HttpHeader.h:63
const char * headerWideProblem
worst header-wide problem found (or nil)
int hasByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:1698
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
void delAt(HttpHeaderPos pos, int &headers_deleted)
Definition: HttpHeader.cc:709
void packInto(Packable *p) const
void append(const HttpHeader *src)
Definition: HttpHeader.cc:235
TimeOrTag getTimeOrTag(Http::HdrType id) const
Definition: HttpHeader.cc:1379
unsigned int major
major version number
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:404
const char * getAuth(Http::HdrType id, const char *auth_scheme) const
Definition: HttpHeader.cc:1327
int relaxed_header_parser
Definition: SquidConfig.h:318
const char * getLastStr(Http::HdrType id) const
Definition: HttpHeader.cc:1228
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
static HttpHeaderStat HttpHeaderStats[]
Definition: HttpHeader.cc:77
String getByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:957
void httpHdrScStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrSc.cc:301
#define DBG_IMPORTANT
Definition: Debug.h:46
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
char const * rawBuf() const
Definition: SquidString.h:85
int parse(const char *header_start, size_t len, Http::ContentLengthInterpreter &interpreter)
Definition: HttpHeader.cc:370
HttpHdrSc * getSc() const
Definition: HttpHeader.cc:1291
const HeaderTableRecord & lookup(const char *buf, const std::size_t len) const
look record type up by name (C-string and length)
HttpHeaderEntry * findEntry(Http::HdrType id) const
Definition: HttpHeader.cc:617
int64_t getInt64(Http::HdrType id) const
Definition: HttpHeader.cc:1182
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1063
static HttpHeaderMask ReplyHeadersMask
Definition: HttpHeader.cc:73
HttpHeaderEntry(Http::HdrType id, const SBuf &name, const char *value)
Definition: HttpHeader.cc:1409
char const * termedBuf() const
Definition: SquidString.h:91
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:828
static void httpHeaderRegisterWithCacheManager(void)
Definition: HttpHeader.cc:111
void updateStats(StatHist *) const
Definition: HttpHdrSc.cc:277
#define countof(arr)
Definition: defines.h:186
const char * label
static const CharacterSet CR
Definition: CharacterSet.h:77
http_hdr_owner_type owner
Definition: HttpHeader.h:158
char HttpHeaderMask[12]
bool parse(const String &s)
parse a header-string and fill in appropriate values.
Definition: HttpHdrCc.cc:94
const char * c_str()
Definition: SBuf.cc:526
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:17
void putSc(HttpHdrSc *sc)
Definition: HttpHeader.cc:1136
void putWarning(const int code, const char *const text)
add a Warning header
Definition: HttpHeader.cc:1152
static HttpHdrRange * ParseCreate(const String *range_spec)
bool update(HttpHeader const *fresh)
Definition: HttpHeader.cc:283
void refreshMask()
Definition: HttpHeader.cc:737
const char * getStringPrefix(const char *str, size_t sz)
void putInt(Http::HdrType id, int number)
Definition: HttpHeader.cc:1045
StatHist ccTypeDistr
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1674
HttpHdrContRange * httpHdrContRangeParseCreate(const char *str)
HttpHeaderEntry * findLastEntry(Http::HdrType id) const
Definition: HttpHeader.cc:642
String getById(Http::HdrType id) const
Definition: HttpHeader.cc:897
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=0)
Definition: HttpHeader.cc:1019
unsigned char code
Definition: html_quote.c:20
void const char * buf
Definition: stub_helper.cc:16
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
struct SquidConfig::@112 onoff
String getStrOrList(Http::HdrType id) const
Definition: HttpHeader.cc:862
static int HeaderEntryParsedCount
Definition: HttpHeader.cc:91
void httpHeaderInitModule(void)
Definition: HttpHeader.cc:119
HTTP per header statistics.
void packInto(Packable *p) const
Definition: HttpHeader.cc:1525
void clean()
Definition: MemBuf.cc:113
void httpHdrCcInitModule(void)
Module initialization hook.
Definition: HttpHdrCc.cc:60
StatHist fieldTypeDistr
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1054
size_type findFirstOf(const CharacterSet &set, size_type startPos=0) const
Definition: SBuf.cc:733
char * buf
Definition: MemBuf.h:134
SQUIDCEXTERN const char * xint64toa(int64_t num)
Definition: util.c:88
#define CBIT_CLR(mask, bit)
Definition: defines.h:113
number
Definition: testStatHist.cc:16
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1072
void clean()
Definition: HttpHeader.cc:187
int valid
Definition: TimeOrTag.h:22
const HeaderLookupTable_t HeaderLookupTable
bool any_valid_header(const Http::HdrType id)
match any valid header type, including OTHER but not BAD
#define PROF_start(probename)
Definition: Profiler.h:62
void putCc(const HttpHdrCc *cc)
Definition: HttpHeader.cc:1088
SQUIDCEXTERN double xdiv(double nom, double denom)
Definition: util.c:72
void httpHdrContRangePackInto(const HttpHdrContRange *range, Packable *p)
void updateWarnings()
Definition: HttpHeader.cc:263
void packInto(Packable *p) const
Definition: HttpHdrSc.cc:250
static void httpHeaderStatDump(const HttpHeaderStat *hs, StoreEntry *e)
Definition: HttpHeader.cc:1603
char ThisCache[RFC2181_MAXHOSTNAMELEN<< 1]
HttpHeader & operator=(const HttpHeader &other)
Definition: HttpHeader.cc:173
void compact()
Definition: HttpHeader.cc:726
bool chunked() const
whether message uses chunked Transfer-Encoding
Definition: HttpHeader.h:188
static const size_type npos
Definition: SBuf.h:92
std::vector< HttpHeaderFieldStat > headerStatsTable(Http::HdrType::enumEnd_)
void packInto(Packable *p) const
Definition: HttpHdrCc.cc:246
void httpHdrCcUpdateStats(const HttpHdrCc *cc, StatHist *hist)
Definition: HttpHdrCc.cc:320
Definition: MemBuf.h:23
bool any_registered_header(const Http::HdrType id)
time_t getTime(Http::HdrType id) const
Definition: HttpHeader.cc:1195
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:554
const char * ProtocolType_str[]
static HttpHeaderEntry * parse(const char *field_start, const char *field_end)
Definition: HttpHeader.cc:1441
const char * str
quoted-string
Definition: ETag.h:20
bool sawBad
whether a malformed Content-Length value was present
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1212
static int HttpHeaderStatCount
Definition: HttpHeader.cc:89
void clean()
Definition: String.cc:125
Definition: Debug.h:184
HttpHdrContRange * getContRange() const
Definition: HttpHeader.cc:1313
unsigned int minor
minor version number
#define PROF_stop(probename)
Definition: Profiler.h:63
SBuf StringToSBuf(const String &s)
create a new SBuf from a String by copying contents
Definition: StringConvert.h:17
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:751
static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error)
Definition: HttpHeader.cc:1556
void dump(StoreEntry *sentry, StatHistBinDumper *bd) const
Definition: StatHist.cc:173
void insertEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:777
void httpHeaderMaskInit(HttpHeaderMask *mask, int value)
int base64_decode_update(struct base64_decode_ctx *ctx, size_t *dst_length, uint8_t *dst, size_t src_length, const char *src)
Definition: base64.c:129
const HttpHeaderStat * dump_stat
Definition: HttpHeader.cc:1574
ProtocolType protocol
which protocol this version is for
StatHist hdrUCountDistr
const char * rawContent() const
Definition: SBuf.cc:519
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:16
void putRange(const HttpHdrRange *range)
Definition: HttpHeader.cc:1120
int base64_decode_final(struct base64_decode_ctx *ctx)
Definition: base64.c:159
HttpHeaderMask * owner_mask
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:875
char const * visible_appname_string
StatHist scTypeDistr
static HttpHeaderMask RequestHeadersMask
Definition: HttpHeader.cc:70
void putAuth(const char *auth_scheme, const char *realm)
Definition: HttpHeader.cc:1081
class SquidConfig Config
Definition: SquidConfig.cc:12
#define NULL
Definition: types.h:166
const char * text
Definition: errorpage.cc:79
static void httpHeaderStoreReport(StoreEntry *e)
Definition: HttpHeader.cc:1631
int httpHeaderParseInt(const char *start, int *value)
#define false
Definition: GnuRegex.c:233
bool getByIdIfPresent(Http::HdrType id, String *result) const
Definition: HttpHeader.cc:911
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1519
ETag getETag(Http::HdrType id) const
Definition: HttpHeader.cc:1366

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors