HttpHeader.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 55 HTTP Header */
10 
11 #include "squid.h"
12 #include "base/CharacterSet.h"
13 #include "base/EnumIterator.h"
14 #include "base/Raw.h"
15 #include "base64.h"
16 #include "globals.h"
18 #include "HttpHdrCc.h"
19 #include "HttpHdrContRange.h"
20 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
21 #include "HttpHeader.h"
22 #include "HttpHeaderFieldInfo.h"
23 #include "HttpHeaderStat.h"
24 #include "HttpHeaderTools.h"
25 #include "MemBuf.h"
26 #include "mgr/Registration.h"
27 #include "mime_header.h"
28 #include "sbuf/StringConvert.h"
29 #include "SquidConfig.h"
30 #include "StatHist.h"
31 #include "Store.h"
32 #include "StrList.h"
33 #include "time/gadgets.h"
34 #include "TimeOrTag.h"
35 #include "util.h"
36 
37 #include <algorithm>
38 #include <array>
39 
40 /* XXX: the whole set of API managing the entries vector should be rethought
41  * after the parse4r-ng effort is complete.
42  */
43 
44 /*
45  * On naming conventions:
46  *
47  * HTTP/1.1 defines message-header as
48  *
49  * message-header = field-name ":" [ field-value ] CRLF
50  * field-name = token
51  * field-value = *( field-content | LWS )
52  *
53  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
54  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
55  *
56  * HttpHeader is an object that represents all message-headers in a message.
57  * HttpHeader does not manage start-line.
58  *
59  * HttpHeader is implemented as a collection of header "entries".
60  * An entry is a (field_id, field_name, field_value) triplet.
61  */
62 
63 /*
64  * local constants and vars
65  */
66 
67 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
68 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
69 
70 /* request-only headers. Used for cachemgr */
71 static HttpHeaderMask RequestHeadersMask; /* set run-time using RequestHeaders */
72 
73 /* reply-only headers. Used for cachemgr */
74 static HttpHeaderMask ReplyHeadersMask; /* set run-time using ReplyHeaders */
75 
76 /* header accounting */
77 // NP: keep in sync with enum http_hdr_owner_type
78 static std::array<HttpHeaderStat, hoEnd> HttpHeaderStats = {
79  HttpHeaderStat(/*hoNone*/ "all", NULL),
80 #if USE_HTCP
81  HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
82 #endif
83  HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
84  HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
85 #if USE_OPENSSL
86  , HttpHeaderStat(/*hoErrorDetail*/ "error detail templates", nullptr)
87 #endif
88  /* hoEnd */
89 };
90 
91 static int HeaderEntryParsedCount = 0;
92 
93 /*
94  * forward declarations and local routines
95  */
96 
97 class StoreEntry;
98 
99 // update parse statistics for header id; if error is true also account
100 // for errors and write to debug log what happened
101 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
102 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
104 static void httpHeaderStoreReport(StoreEntry * e);
105 
106 /*
107  * Module initialization routines
108  */
109 
110 static void
112 {
113  Mgr::RegisterAction("http_headers",
114  "HTTP Header Statistics",
115  httpHeaderStoreReport, 0, 1);
116 }
117 
118 void
120 {
121  /* check that we have enough space for masks */
123 
124  // masks are needed for stats page still
125  for (auto h : WholeEnum<Http::HdrType>()) {
126  if (Http::HeaderLookupTable.lookup(h).request)
128  if (Http::HeaderLookupTable.lookup(h).reply)
130  }
131 
132  assert(HttpHeaderStats[0].label && "httpHeaderInitModule() called via main()");
133  assert(HttpHeaderStats[hoEnd-1].label && "HttpHeaderStats created with all elements");
134 
135  /* init dependent modules */
138 
140 }
141 
142 /*
143  * HttpHeader Implementation
144  */
145 
146 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
147 {
148  entries.reserve(32);
150 }
151 
152 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
153 {
154  assert(anOwner > hoNone && anOwner < hoEnd);
155  debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
156  entries.reserve(32);
158 }
159 
160 // XXX: Delete as unused, expensive, and violating copy semantics by skipping Warnings
161 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
162 {
163  entries.reserve(other.entries.capacity());
165  update(&other); // will update the mask as well
166 }
167 
169 {
170  clean();
171 }
172 
173 // XXX: Delete as unused, expensive, and violating assignment semantics by skipping Warnings
174 HttpHeader &
176 {
177  if (this != &other) {
178  // we do not really care, but the caller probably does
179  assert(owner == other.owner);
180  clean();
181  update(&other); // will update the mask as well
182  len = other.len;
185  }
186  return *this;
187 }
188 
189 void
191 {
192 
193  assert(owner > hoNone && owner < hoEnd);
194  debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
195 
196  if (owner <= hoReply) {
197  /*
198  * An unfortunate bug. The entries array is initialized
199  * such that count is set to zero. httpHeaderClean() seems to
200  * be called both when 'hdr' is created, and destroyed. Thus,
201  * we accumulate a large number of zero counts for 'hdr' before
202  * it is ever used. Can't think of a good way to fix it, except
203  * adding a state variable that indicates whether or not 'hdr'
204  * has been used. As a hack, just never count zero-sized header
205  * arrays.
206  */
207  if (!entries.empty())
208  HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
209 
210  ++ HttpHeaderStats[owner].destroyedCount;
211 
212  HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
213  } // if (owner <= hoReply)
214 
215  for (HttpHeaderEntry *e : entries) {
216  if (e == nullptr)
217  continue;
218  if (!Http::any_valid_header(e->id)) {
219  debugs(55, DBG_CRITICAL, "ERROR: Squid BUG: invalid entry (" << e->id << "). Ignored.");
220  } else {
221  if (owner <= hoReply)
222  HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
223  delete e;
224  }
225  }
226 
227  entries.clear();
229  len = 0;
231  teUnsupported_ = false;
232 }
233 
234 /* append entries (also see httpHeaderUpdate) */
235 void
237 {
238  assert(src);
239  assert(src != this);
240  debugs(55, 7, "appending hdr: " << this << " += " << src);
241 
242  for (auto e : src->entries) {
243  if (e)
244  addEntry(e->clone());
245  }
246 }
247 
248 bool
250 {
251  // our 1xx Warnings must be removed
252  for (const auto e: entries) {
253  // TODO: Move into HttpHeaderEntry::is1xxWarning() before official commit.
254  if (e && e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1))
255  return true;
256  }
257 
258  for (const auto e: fresh->entries) {
259  if (!e || skipUpdateHeader(e->id))
260  continue;
261  String value;
262  if (!hasNamed(e->name, &value) ||
263  (value != fresh->getByName(e->name)))
264  return true;
265  }
266  return false;
267 }
268 
269 void
271 {
272  int count = 0;
274 
275  // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
276  while (HttpHeaderEntry *e = getEntry(&pos)) {
277  if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
278  delAt(pos, count);
279  }
280 }
281 
282 bool
284 {
285  return
286  // RFC 7234, section 4.3.4: use header fields other than Warning
287  (id == Http::HdrType::WARNING) ||
288  // TODO: Consider updating Vary headers after comparing the magnitude of
289  // the required changes (and/or cache losses) with compliance gains.
290  (id == Http::HdrType::VARY);
291 }
292 
293 void
295 {
296  assert(fresh);
297  assert(this != fresh);
298 
299  updateWarnings();
300 
301  const HttpHeaderEntry *e;
303 
304  while ((e = fresh->getEntry(&pos))) {
305  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
306 
307  if (skipUpdateHeader(e->id))
308  continue;
309 
310  if (e->id != Http::HdrType::OTHER)
311  delById(e->id);
312  else
313  delByName(e->name);
314  }
315 
316  pos = HttpHeaderInitPos;
317  while ((e = fresh->getEntry(&pos))) {
318  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
319 
320  if (skipUpdateHeader(e->id))
321  continue;
322 
323  debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
324 
325  addEntry(e->clone());
326  }
327 }
328 
329 bool
330 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
331 {
332  /*
333  * parse_start points to the first line of HTTP message *headers*,
334  * not including the request or status lines
335  */
336  const size_t end = headersEnd(*parse_start, l);
337 
338  if (end) {
339  *blk_start = *parse_start;
340  *blk_end = *parse_start + end - 1;
341  assert(**blk_end == '\n');
342  // Point blk_end to the first character after the last header field.
343  // In other words, blk_end should point to the CR?LF header terminator.
344  if (end > 1 && *(*blk_end - 1) == '\r')
345  --(*blk_end);
346  *parse_start += end;
347  }
348  return end;
349 }
350 
351 int
352 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz, Http::ContentLengthInterpreter &clen)
353 {
354  const char *parse_start = buf;
355  const char *blk_start, *blk_end;
356  hdr_sz = 0;
357 
358  if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
359  // XXX: do not parse non-isolated headers even if the connection is closed.
360  // Treat unterminated headers as "partial headers" framing errors.
361  if (!atEnd)
362  return 0;
363  blk_start = parse_start;
364  blk_end = blk_start + strlen(blk_start);
365  }
366 
367  if (parse(blk_start, blk_end - blk_start, clen)) {
368  hdr_sz = parse_start - buf;
369  return 1;
370  }
371  return -1;
372 }
373 
374 // XXX: callers treat this return as boolean.
375 // XXX: A better mechanism is needed to signal different types of error.
376 // lexicon, syntax, semantics, validation, access policy - are all (ab)using 'return 0'
377 int
378 HttpHeader::parse(const char *header_start, size_t hdrLen, Http::ContentLengthInterpreter &clen)
379 {
380  const char *field_ptr = header_start;
381  const char *header_end = header_start + hdrLen; // XXX: remove
382  int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
383 
384  assert(header_start && header_end);
385  debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
386  ++ HttpHeaderStats[owner].parsedCount;
387 
388  char *nulpos;
389  if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
390  debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
391  getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
392  clean();
393  return 0;
394  }
395 
396  /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
397  * continuation lines start with a (single) space or tab */
398  while (field_ptr < header_end) {
399  const char *field_start = field_ptr;
400  const char *field_end;
401 
402  const char *hasBareCr = nullptr;
403  size_t lines = 0;
404  do {
405  const char *this_line = field_ptr;
406  field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
407  ++lines;
408 
409  if (!field_ptr) {
410  // missing <LF>
411  clean();
412  return 0;
413  }
414 
415  field_end = field_ptr;
416 
417  ++field_ptr; /* Move to next line */
418 
419  if (field_end > this_line && field_end[-1] == '\r') {
420  --field_end; /* Ignore CR LF */
421 
422  if (owner == hoRequest && field_end > this_line) {
423  bool cr_only = true;
424  for (const char *p = this_line; p < field_end && cr_only; ++p) {
425  if (*p != '\r')
426  cr_only = false;
427  }
428  if (cr_only) {
429  debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
430  "header field to prevent request smuggling attacks: {" <<
431  getStringPrefix(header_start, hdrLen) << "}");
432  clean();
433  return 0;
434  }
435  }
436  }
437 
438  /* Barf on stray CR characters */
439  if (memchr(this_line, '\r', field_end - this_line)) {
440  hasBareCr = "bare CR";
441  debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
442  getStringPrefix(field_start, field_end-field_start) << "}");
443 
445  char *p = (char *) this_line; /* XXX Warning! This destroys original header content and violates specifications somewhat */
446 
447  while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
448  *p = ' ';
449  ++p;
450  }
451  } else {
452  clean();
453  return 0;
454  }
455  }
456 
457  if (this_line + 1 == field_end && this_line > field_start) {
458  debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
459  getStringPrefix(header_start, hdrLen) << "}");
460  clean();
461  return 0;
462  }
463  } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
464 
465  if (field_start == field_end) {
466  if (field_ptr < header_end) {
467  debugs(55, warnOnError, "WARNING: unparsable HTTP header field near {" <<
468  getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
469  clean();
470  return 0;
471  }
472 
473  break; /* terminating blank line */
474  }
475 
476  const auto e = HttpHeaderEntry::parse(field_start, field_end, owner);
477  if (!e) {
478  debugs(55, warnOnError, "WARNING: unparsable HTTP header field {" <<
479  getStringPrefix(field_start, field_end-field_start) << "}");
480  debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
481 
482  clean();
483  return 0;
484  }
485 
486  if (lines > 1 || hasBareCr) {
487  const auto framingHeader = (e->id == Http::HdrType::CONTENT_LENGTH || e->id == Http::HdrType::TRANSFER_ENCODING);
488  if (framingHeader) {
489  if (!hasBareCr) // already warned about bare CRs
490  debugs(55, warnOnError, "WARNING: obs-fold in framing-sensitive " << e->name << ": " << e->value);
491  delete e;
492  clean();
493  return 0;
494  }
495  }
496 
497  if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
498  delete e;
499 
501  continue; // clen has printed any necessary warnings
502 
503  clean();
504  return 0;
505  }
506 
507  addEntry(e);
508  }
509 
510  if (clen.headerWideProblem) {
511  debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
512  " Content-Length field values in" <<
513  Raw("header", header_start, hdrLen));
514  }
515 
516  String rawTe;
517  if (clen.prohibitedAndIgnored()) {
518  // prohibitedAndIgnored() includes trailer header blocks
519  // being parsed as a case to forbid/ignore these headers.
520 
521  // RFC 7230 section 3.3.2: A server MUST NOT send a Content-Length
522  // header field in any response with a status code of 1xx (Informational)
523  // or 204 (No Content). And RFC 7230 3.3.3#1 tells recipients to ignore
524  // such Content-Lengths.
526  debugs(55, 3, "Content-Length is " << clen.prohibitedAndIgnored());
527 
528  // The same RFC 7230 3.3.3#1-based logic applies to Transfer-Encoding
529  // banned by RFC 7230 section 3.3.1.
531  debugs(55, 3, "Transfer-Encoding is " << clen.prohibitedAndIgnored());
532 
534  // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
535  // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
537  // and clen state becomes irrelevant
538 
539  if (rawTe.caseCmp("chunked") == 0) {
540  ; // leave header present for chunked() method
541  } else if (rawTe.caseCmp("identity") == 0) { // deprecated. no coding
543  } else {
544  // This also rejects multiple encodings until we support them properly.
545  debugs(55, warnOnError, "WARNING: unsupported Transfer-Encoding used by client: " << rawTe);
546  teUnsupported_ = true;
547  }
548 
549  } else if (clen.sawBad) {
550  // ensure our callers do not accidentally see bad Content-Length values
552  conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
553  } else if (clen.needsSanitizing) {
554  // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
555  // ensure our callers see a clean Content-Length value or none at all
557  if (clen.sawGood) {
559  debugs(55, 5, "sanitized Content-Length to be " << clen.value);
560  }
561  }
562 
563  return 1; /* even if no fields where found, it is a valid header */
564 }
565 
566 /* packs all the entries using supplied packer */
567 void
568 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
569 {
571  const HttpHeaderEntry *e;
572  assert(p);
573  debugs(55, 7, this << " into " << p <<
574  (mask_sensitive_info ? " while masking" : ""));
575  /* pack all entries one by one */
576  while ((e = getEntry(&pos))) {
577  if (!mask_sensitive_info) {
578  e->packInto(p);
579  continue;
580  }
581 
582  bool maskThisEntry = false;
583  switch (e->id) {
586  maskThisEntry = true;
587  break;
588 
591  maskThisEntry = (cmd->value == "PASS");
592  break;
593 
594  default:
595  break;
596  }
597  if (maskThisEntry) {
598  p->append(e->name.rawContent(), e->name.length());
599  p->append(": ** NOT DISPLAYED **\r\n", 23);
600  } else {
601  e->packInto(p);
602  }
603 
604  }
605  /* Pack in the "special" entries */
606 
607  /* Cache-Control */
608 }
609 
610 /* returns next valid entry */
613 {
614  assert(pos);
615  assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
616 
617  for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
618  if (entries[*pos])
619  return static_cast<HttpHeaderEntry*>(entries[*pos]);
620  }
621 
622  return NULL;
623 }
624 
625 /*
626  * returns a pointer to a specified entry if any
627  * note that we return one entry so it does not make much sense to ask for
628  * "list" headers
629  */
632 {
634  assert(!Http::HeaderLookupTable.lookup(id).list);
635 
636  /* check mask first */
637 
638  if (!CBIT_TEST(mask, id))
639  return NULL;
640 
641  /* looks like we must have it, do linear search */
642  for (auto e : entries) {
643  if (e && e->id == id)
644  return e;
645  }
646 
647  /* hm.. we thought it was there, but it was not found */
648  assert(false);
649  return nullptr; /* not reached */
650 }
651 
652 /*
653  * same as httpHeaderFindEntry
654  */
657 {
659  assert(!Http::HeaderLookupTable.lookup(id).list);
660 
661  /* check mask first */
662  if (!CBIT_TEST(mask, id))
663  return NULL;
664 
665  for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
666  if (*e && (*e)->id == id)
667  return *e;
668  }
669 
670  /* hm.. we thought it was there, but it was not found */
671  assert(false);
672  return nullptr; /* not reached */
673 }
674 
675 int
677 {
678  int count = 0;
680  httpHeaderMaskInit(&mask, 0); /* temporal inconsistency */
681  debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
682 
683  while (const HttpHeaderEntry *e = getEntry(&pos)) {
684  if (!e->name.caseCmp(name))
685  delAt(pos, count);
686  else
687  CBIT_SET(mask, e->id);
688  }
689 
690  return count;
691 }
692 
693 /* deletes all entries with a given id, returns the #entries deleted */
694 int
696 {
697  debugs(55, 8, this << " del-by-id " << id);
699 
700  if (!CBIT_TEST(mask, id))
701  return 0;
702 
703  int count = 0;
704 
706  while (HttpHeaderEntry *e = getEntry(&pos)) {
707  if (e->id == id)
708  delAt(pos, count); // deletes e
709  }
710 
711  CBIT_CLR(mask, id);
712  assert(count);
713  return count;
714 }
715 
716 /*
717  * deletes an entry at pos and leaves a gap; leaving a gap makes it
718  * possible to iterate(search) and delete fields at the same time
719  * NOTE: Does not update the header mask. Caller must follow up with
720  * a call to refreshMask() if headers_deleted was incremented.
721  */
722 void
723 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
724 {
725  HttpHeaderEntry *e;
726  assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
727  e = static_cast<HttpHeaderEntry*>(entries[pos]);
728  entries[pos] = NULL;
729  /* decrement header length, allow for ": " and crlf */
730  len -= e->name.length() + 2 + e->value.size() + 2;
731  assert(len >= 0);
732  delete e;
733  ++headers_deleted;
734 }
735 
736 /*
737  * Compacts the header storage
738  */
739 void
741 {
742  // TODO: optimize removal, or possibly make it so that's not needed.
743  entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
744  entries.end());
745 }
746 
747 /*
748  * Refreshes the header mask. Required after delAt() calls.
749  */
750 void
752 {
754  debugs(55, 7, "refreshing the mask in hdr " << this);
755  for (auto e : entries) {
756  if (e)
757  CBIT_SET(mask, e->id);
758  }
759 }
760 
761 /* appends an entry;
762  * does not call e->clone() so one should not reuse "*e"
763  */
764 void
766 {
767  assert(e);
769  assert(e->name.length());
770 
771  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
772 
773  if (e->id != Http::HdrType::BAD_HDR) {
774  if (CBIT_TEST(mask, e->id)) {
775  ++ headerStatsTable[e->id].repCount;
776  } else {
777  CBIT_SET(mask, e->id);
778  }
779  }
780 
781  entries.push_back(e);
782 
783  /* increment header length, allow for ": " and crlf */
784  len += e->name.length() + 2 + e->value.size() + 2;
785 }
786 
787 /* inserts an entry;
788  * does not call e->clone() so one should not reuse "*e"
789  */
790 void
792 {
793  assert(e);
795 
796  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
797 
798  // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
799  if (CBIT_TEST(mask, e->id)) {
800  ++ headerStatsTable[e->id].repCount;
801  } else {
802  CBIT_SET(mask, e->id);
803  }
804 
805  entries.insert(entries.begin(),e);
806 
807  /* increment header length, allow for ": " and crlf */
808  len += e->name.length() + 2 + e->value.size() + 2;
809 }
810 
811 bool
813 {
814  debugs(55, 9, this << " joining for id " << id);
815  /* only fields from ListHeaders array can be "listed" */
816  assert(Http::HeaderLookupTable.lookup(id).list);
817 
818  if (!CBIT_TEST(mask, id))
819  return false;
820 
821  for (auto e: entries) {
822  if (e && e->id == id)
823  strListAdd(s, e->value.termedBuf(), ',');
824  }
825 
826  /*
827  * note: we might get an empty (size==0) string if there was an "empty"
828  * header. This results in an empty length String, which may have a NULL
829  * buffer.
830  */
831  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
832  if (!s->size())
833  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
834  else
835  debugs(55, 6, this << ": joined for id " << id << ": " << s);
836 
837  return true;
838 }
839 
840 /* return a list of entries with the same id separated by ',' and ws */
841 String
843 {
844  HttpHeaderEntry *e;
846  debugs(55, 9, this << "joining for id " << id);
847  /* only fields from ListHeaders array can be "listed" */
848  assert(Http::HeaderLookupTable.lookup(id).list);
849 
850  if (!CBIT_TEST(mask, id))
851  return String();
852 
853  String s;
854 
855  while ((e = getEntry(&pos))) {
856  if (e->id == id)
857  strListAdd(&s, e->value.termedBuf(), ',');
858  }
859 
860  /*
861  * note: we might get an empty (size==0) string if there was an "empty"
862  * header. This results in an empty length String, which may have a NULL
863  * buffer.
864  */
865  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
866  if (!s.size())
867  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
868  else
869  debugs(55, 6, this << ": joined for id " << id << ": " << s);
870 
871  return s;
872 }
873 
874 /* return a string or list of entries with the same id separated by ',' and ws */
875 String
877 {
878  HttpHeaderEntry *e;
879 
880  if (Http::HeaderLookupTable.lookup(id).list)
881  return getList(id);
882 
883  if ((e = findEntry(id)))
884  return e->value;
885 
886  return String();
887 }
888 
889 /*
890  * Returns the value of the specified header and/or an undefined String.
891  */
892 String
893 HttpHeader::getByName(const char *name) const
894 {
895  String result;
896  // ignore presence: return undefined string if an empty header is present
897  (void)hasNamed(name, strlen(name), &result);
898  return result;
899 }
900 
901 String
902 HttpHeader::getByName(const SBuf &name) const
903 {
904  String result;
905  // ignore presence: return undefined string if an empty header is present
906  (void)hasNamed(name, &result);
907  return result;
908 }
909 
910 String
912 {
913  String result;
914  (void)getByIdIfPresent(id, &result);
915  return result;
916 }
917 
918 bool
919 HttpHeader::hasNamed(const SBuf &s, String *result) const
920 {
921  return hasNamed(s.rawContent(), s.length(), result);
922 }
923 
924 bool
926 {
927  if (id == Http::HdrType::BAD_HDR)
928  return false;
929  if (!has(id))
930  return false;
931  if (result)
932  *result = getStrOrList(id);
933  return true;
934 }
935 
936 bool
937 HttpHeader::hasNamed(const char *name, unsigned int namelen, String *result) const
938 {
939  Http::HdrType id;
941  HttpHeaderEntry *e;
942 
943  assert(name);
944 
945  /* First try the quick path */
946  id = Http::HeaderLookupTable.lookup(name,namelen).id;
947 
948  if (id != Http::HdrType::BAD_HDR) {
949  if (getByIdIfPresent(id, result))
950  return true;
951  }
952 
953  /* Sorry, an unknown header name. Do linear search */
954  bool found = false;
955  while ((e = getEntry(&pos))) {
956  if (e->id == Http::HdrType::OTHER && e->name.length() == namelen && e->name.caseCmp(name, namelen) == 0) {
957  found = true;
958  if (!result)
959  break;
960  strListAdd(result, e->value.termedBuf(), ',');
961  }
962  }
963 
964  return found;
965 }
966 
967 /*
968  * Returns a the value of the specified list member, if any.
969  */
970 SBuf
971 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
972 {
973  assert(name);
974  const auto header = getByName(name);
975  return ::getListMember(header, member, separator);
976 }
977 
978 /*
979  * returns a the value of the specified list member, if any.
980  */
981 SBuf
982 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
983 {
985  const auto header = getStrOrList(id);
986  return ::getListMember(header, member, separator);
987 }
988 
989 /* test if a field is present */
990 int
992 {
994  debugs(55, 9, this << " lookup for " << id);
995  return CBIT_TEST(mask, id);
996 }
997 
998 void
1000 {
1001  // TODO: do not add Via header for messages where Squid itself
1002  // generated the message (i.e., Downloader or ESI) there should be no Via header added at all.
1003 
1004  if (Config.onoff.via) {
1005  SBuf buf;
1006  // RFC 7230 section 5.7.1.: protocol-name is omitted when
1007  // the received protocol is HTTP.
1010  buf.appendf("%s/", AnyP::ProtocolType_str[ver.protocol]);
1011  buf.appendf("%d.%d %s", ver.major, ver.minor, ThisCache);
1012  const HttpHeader *hdr = from ? from : this;
1013  SBuf strVia = StringToSBuf(hdr->getList(Http::HdrType::VIA));
1014  if (!strVia.isEmpty())
1015  strVia.append(", ", 2);
1016  strVia.append(buf);
1017  // XXX: putStr() still suffers from String size limits
1018  Must(strVia.length() < String::SizeMaxXXX());
1020  putStr(Http::HdrType::VIA, strVia.c_str());
1021  }
1022 }
1023 
1024 void
1026 {
1028  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1029  assert(number >= 0);
1030  addEntry(new HttpHeaderEntry(id, SBuf(), xitoa(number)));
1031 }
1032 
1033 void
1035 {
1037  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1038  assert(number >= 0);
1040 }
1041 
1042 void
1044 {
1046  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1047  assert(htime >= 0);
1048  addEntry(new HttpHeaderEntry(id, SBuf(), Time::FormatRfc1123(htime)));
1049 }
1050 
1051 void
1052 HttpHeader::putStr(Http::HdrType id, const char *str)
1053 {
1055  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1056  assert(str);
1057  addEntry(new HttpHeaderEntry(id, SBuf(), str));
1058 }
1059 
1060 void
1061 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1062 {
1063  assert(auth_scheme && realm);
1064  httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1065 }
1066 
1067 void
1069 {
1070  assert(cc);
1071  /* remove old directives if any */
1073  /* pack into mb */
1074  MemBuf mb;
1075  mb.init();
1076  cc->packInto(&mb);
1077  /* put */
1079  /* cleanup */
1080  mb.clean();
1081 }
1082 
1083 void
1085 {
1086  assert(cr);
1087  /* remove old directives if any */
1089  /* pack into mb */
1090  MemBuf mb;
1091  mb.init();
1092  httpHdrContRangePackInto(cr, &mb);
1093  /* put */
1095  /* cleanup */
1096  mb.clean();
1097 }
1098 
1099 void
1101 {
1102  assert(range);
1103  /* remove old directives if any */
1105  /* pack into mb */
1106  MemBuf mb;
1107  mb.init();
1108  range->packInto(&mb);
1109  /* put */
1111  /* cleanup */
1112  mb.clean();
1113 }
1114 
1115 void
1117 {
1118  assert(sc);
1119  /* remove old directives if any */
1121  /* pack into mb */
1122  MemBuf mb;
1123  mb.init();
1124  sc->packInto(&mb);
1125  /* put */
1127  /* cleanup */
1128  mb.clean();
1129 }
1130 
1131 void
1132 HttpHeader::putWarning(const int code, const char *const text)
1133 {
1134  char buf[512];
1135  snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1137 }
1138 
1139 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1140 void
1141 HttpHeader::putExt(const char *name, const char *value)
1142 {
1143  assert(name && value);
1144  debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1145  addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, SBuf(name), value));
1146 }
1147 
1148 int
1150 {
1152  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1153  HttpHeaderEntry *e;
1154 
1155  if ((e = findEntry(id)))
1156  return e->getInt();
1157 
1158  return -1;
1159 }
1160 
1161 int64_t
1163 {
1165  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1166  HttpHeaderEntry *e;
1167 
1168  if ((e = findEntry(id)))
1169  return e->getInt64();
1170 
1171  return -1;
1172 }
1173 
1174 time_t
1176 {
1177  HttpHeaderEntry *e;
1178  time_t value = -1;
1180  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1181 
1182  if ((e = findEntry(id))) {
1183  value = Time::ParseRfc1123(e->value.termedBuf());
1184  httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1185  }
1186 
1187  return value;
1188 }
1189 
1190 /* sync with httpHeaderGetLastStr */
1191 const char *
1193 {
1194  HttpHeaderEntry *e;
1196  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1197 
1198  if ((e = findEntry(id))) {
1199  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1200  return e->value.termedBuf();
1201  }
1202 
1203  return NULL;
1204 }
1205 
1206 /* unusual */
1207 const char *
1209 {
1210  HttpHeaderEntry *e;
1212  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1213 
1214  if ((e = findLastEntry(id))) {
1215  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1216  return e->value.termedBuf();
1217  }
1218 
1219  return NULL;
1220 }
1221 
1222 HttpHdrCc *
1224 {
1226  return NULL;
1227 
1228  String s;
1230 
1231  HttpHdrCc *cc=new HttpHdrCc();
1232 
1233  if (!cc->parse(s)) {
1234  delete cc;
1235  cc = NULL;
1236  }
1237 
1238  ++ HttpHeaderStats[owner].ccParsedCount;
1239 
1240  if (cc)
1241  httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1242 
1244 
1245  return cc;
1246 }
1247 
1248 HttpHdrRange *
1250 {
1251  HttpHdrRange *r = NULL;
1252  HttpHeaderEntry *e;
1253  /* some clients will send "Request-Range" _and_ *matching* "Range"
1254  * who knows, some clients might send Request-Range only;
1255  * this "if" should work correctly in both cases;
1256  * hopefully no clients send mismatched headers! */
1257 
1258  if ((e = findEntry(Http::HdrType::RANGE)) ||
1261  httpHeaderNoteParsedEntry(e->id, e->value, !r);
1262  }
1263 
1264  return r;
1265 }
1266 
1267 HttpHdrSc *
1269 {
1271  return NULL;
1272 
1273  String s;
1274 
1276 
1278 
1279  ++ HttpHeaderStats[owner].ccParsedCount;
1280 
1281  if (sc)
1282  sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1283 
1285 
1286  return sc;
1287 }
1288 
1291 {
1292  HttpHdrContRange *cr = NULL;
1293  HttpHeaderEntry *e;
1294 
1297  httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1298  }
1299 
1300  return cr;
1301 }
1302 
1303 SBuf
1304 HttpHeader::getAuthToken(Http::HdrType id, const char *auth_scheme) const
1305 {
1306  const char *field;
1307  int l;
1308  assert(auth_scheme);
1309  field = getStr(id);
1310 
1311  static const SBuf nil;
1312  if (!field) /* no authorization field */
1313  return nil;
1314 
1315  l = strlen(auth_scheme);
1316 
1317  if (!l || strncasecmp(field, auth_scheme, l)) /* wrong scheme */
1318  return nil;
1319 
1320  field += l;
1321 
1322  if (!xisspace(*field)) /* wrong scheme */
1323  return nil;
1324 
1325  /* skip white space */
1326  for (; field && xisspace(*field); ++field);
1327 
1328  if (!*field) /* no authorization cookie */
1329  return nil;
1330 
1331  const auto fieldLen = strlen(field);
1332  SBuf result;
1333  char *decodedAuthToken = result.rawAppendStart(BASE64_DECODE_LENGTH(fieldLen));
1334  struct base64_decode_ctx ctx;
1335  base64_decode_init(&ctx);
1336  size_t decodedLen = 0;
1337  if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), fieldLen, field) ||
1338  !base64_decode_final(&ctx)) {
1339  return nil;
1340  }
1341  result.rawAppendFinish(decodedAuthToken, decodedLen);
1342  return result;
1343 }
1344 
1345 ETag
1347 {
1348  ETag etag = {NULL, -1};
1349  HttpHeaderEntry *e;
1350  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag); /* must be of an appropriate type */
1351 
1352  if ((e = findEntry(id)))
1353  etagParseInit(&etag, e->value.termedBuf());
1354 
1355  return etag;
1356 }
1357 
1358 TimeOrTag
1360 {
1361  TimeOrTag tot;
1362  HttpHeaderEntry *e;
1363  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag); /* must be of an appropriate type */
1364  memset(&tot, 0, sizeof(tot));
1365 
1366  if ((e = findEntry(id))) {
1367  const char *str = e->value.termedBuf();
1368  /* try as an ETag */
1369 
1370  if (etagParseInit(&tot.tag, str)) {
1371  tot.valid = tot.tag.str != NULL;
1372  tot.time = -1;
1373  } else {
1374  /* or maybe it is time? */
1375  tot.time = Time::ParseRfc1123(str);
1376  tot.valid = tot.time >= 0;
1377  tot.tag.str = NULL;
1378  }
1379  }
1380 
1381  assert(tot.time < 0 || !tot.tag.str); /* paranoid */
1382  return tot;
1383 }
1384 
1385 /*
1386  * HttpHeaderEntry
1387  */
1388 
1389 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const SBuf &aName, const char *aValue)
1390 {
1392  id = anId;
1393 
1394  if (id != Http::HdrType::OTHER)
1396  else
1397  name = aName;
1398 
1399  value = aValue;
1400 
1401  if (id != Http::HdrType::BAD_HDR)
1402  ++ headerStatsTable[id].aliveCount;
1403 
1404  debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1405 }
1406 
1408 {
1409  debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1410 
1411  if (id != Http::HdrType::BAD_HDR) {
1412  assert(headerStatsTable[id].aliveCount);
1413  -- headerStatsTable[id].aliveCount;
1414  id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1415  }
1416 
1417 }
1418 
1419 /* parses and inits header entry, returns true/false */
1421 HttpHeaderEntry::parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
1422 {
1423  /* note: name_start == field_start */
1424  const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1425  int name_len = name_end ? name_end - field_start :0;
1426  const char *value_start = field_start + name_len + 1; /* skip ':' */
1427  /* note: value_end == field_end */
1428 
1430 
1431  /* do we have a valid field name within this field? */
1432 
1433  if (!name_len || name_end > field_end)
1434  return NULL;
1435 
1436  if (name_len > 65534) {
1437  /* String must be LESS THAN 64K and it adds a terminating NULL */
1438  // TODO: update this to show proper name_len in Raw markup, but not print all that
1439  debugs(55, 2, "ignoring huge header field (" << Raw("field_start", field_start, 100) << "...)");
1440  return NULL;
1441  }
1442 
1443  /*
1444  * RFC 7230 section 3.2.4:
1445  * "No whitespace is allowed between the header field-name and colon.
1446  * ...
1447  * A server MUST reject any received request message that contains
1448  * whitespace between a header field-name and colon with a response code
1449  * of 400 (Bad Request). A proxy MUST remove any such whitespace from a
1450  * response message before forwarding the message downstream."
1451  */
1452  if (xisspace(field_start[name_len - 1])) {
1453 
1454  if (msgType == hoRequest)
1455  return nullptr;
1456 
1457  // for now, also let relaxed parser remove this BWS from any non-HTTP messages
1458  const bool stripWhitespace = (msgType == hoReply) ||
1460  if (!stripWhitespace)
1461  return nullptr; // reject if we cannot strip
1462 
1463  debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1464  "WARNING: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1465 
1466  while (name_len > 0 && xisspace(field_start[name_len - 1]))
1467  --name_len;
1468 
1469  if (!name_len) {
1470  debugs(55, 2, "found header with only whitespace for name");
1471  return NULL;
1472  }
1473  }
1474 
1475  /* RFC 7230 section 3.2:
1476  *
1477  * header-field = field-name ":" OWS field-value OWS
1478  * field-name = token
1479  * token = 1*TCHAR
1480  */
1481  for (const char *pos = field_start; pos < (field_start+name_len); ++pos) {
1482  if (!CharacterSet::TCHAR[*pos]) {
1483  debugs(55, 2, "found header with invalid characters in " <<
1484  Raw("field-name", field_start, min(name_len,100)) << "...");
1485  return nullptr;
1486  }
1487  }
1488 
1489  /* now we know we can parse it */
1490 
1491  debugs(55, 9, "parsing HttpHeaderEntry: near '" << getStringPrefix(field_start, field_end-field_start) << "'");
1492 
1493  /* is it a "known" field? */
1494  Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1495  debugs(55, 9, "got hdr-id=" << id);
1496 
1497  SBuf theName;
1498 
1499  String value;
1500 
1501  if (id == Http::HdrType::BAD_HDR)
1502  id = Http::HdrType::OTHER;
1503 
1504  /* set field name */
1505  if (id == Http::HdrType::OTHER)
1506  theName.append(field_start, name_len);
1507  else
1508  theName = Http::HeaderLookupTable.lookup(id).name;
1509 
1510  /* trim field value */
1511  while (value_start < field_end && xisspace(*value_start))
1512  ++value_start;
1513 
1514  while (value_start < field_end && xisspace(field_end[-1]))
1515  --field_end;
1516 
1517  if (field_end - value_start > 65534) {
1518  /* String must be LESS THAN 64K and it adds a terminating NULL */
1519  debugs(55, 2, "WARNING: found '" << theName << "' header of " << (field_end - value_start) << " bytes");
1520  return NULL;
1521  }
1522 
1523  /* set field value */
1524  value.assign(value_start, field_end - value_start);
1525 
1526  if (id != Http::HdrType::BAD_HDR)
1527  ++ headerStatsTable[id].seenCount;
1528 
1529  debugs(55, 9, "parsed HttpHeaderEntry: '" << theName << ": " << value << "'");
1530 
1531  return new HttpHeaderEntry(id, theName, value.termedBuf());
1532 }
1533 
1536 {
1537  return new HttpHeaderEntry(id, name, value.termedBuf());
1538 }
1539 
1540 void
1542 {
1543  assert(p);
1544  p->append(name.rawContent(), name.length());
1545  p->append(": ", 2);
1546  p->append(value.rawBuf(), value.size());
1547  p->append("\r\n", 2);
1548 }
1549 
1550 int
1552 {
1553  int val = -1;
1554  int ok = httpHeaderParseInt(value.termedBuf(), &val);
1555  httpHeaderNoteParsedEntry(id, value, ok == 0);
1556  /* XXX: Should we check ok - ie
1557  * return ok ? -1 : value;
1558  */
1559  return val;
1560 }
1561 
1562 int64_t
1564 {
1565  int64_t val = -1;
1566  const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1567  httpHeaderNoteParsedEntry(id, value, !ok);
1568  return val; // remains -1 if !ok (XXX: bad method API)
1569 }
1570 
1571 static void
1573 {
1574  if (id != Http::HdrType::BAD_HDR)
1575  ++ headerStatsTable[id].parsCount;
1576 
1577  if (error) {
1578  if (id != Http::HdrType::BAD_HDR)
1579  ++ headerStatsTable[id].errCount;
1580  debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1581  }
1582 }
1583 
1584 /*
1585  * Reports
1586  */
1587 
1588 /* tmp variable used to pass stat info to dumpers */
1589 extern const HttpHeaderStat *dump_stat; /* argh! */
1591 
1592 static void
1593 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1594 {
1595  const int id = static_cast<int>(val);
1596  const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1597  const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1598  int visible = count > 0;
1599  /* for entries with zero count, list only those that belong to current type of message */
1600 
1601  if (!visible && valid_id && dump_stat->owner_mask)
1602  visible = CBIT_TEST(*dump_stat->owner_mask, id);
1603 
1604  if (visible)
1605  storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1606  id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1607 }
1608 
1609 static void
1610 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1611 {
1612  if (count)
1613  storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1614  idx, (int) val, count,
1615  xpercent(count, dump_stat->destroyedCount));
1616 }
1617 
1618 static void
1620 {
1621  assert(hs);
1622  assert(e);
1623 
1624  if (!hs->owner_mask)
1625  return; // these HttpHeaderStat objects were not meant to be dumped here
1626 
1627  dump_stat = hs;
1628  storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1629  storeAppendPrintf(e, "\nField type distribution\n");
1630  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1631  "id", "name", "count", "#/header");
1633  storeAppendPrintf(e, "\nCache-control directives distribution\n");
1634  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1635  "id", "name", "count", "#/cc_field");
1637  storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1638  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1639  "id", "name", "count", "#/sc_field");
1641  storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1642  storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1643  "id", "#flds", "count", "%total");
1645  storeAppendPrintf(e, "\n");
1646  dump_stat = NULL;
1647 }
1648 
1649 void
1651 {
1652  assert(e);
1653 
1654  HttpHeaderStats[0].parsedCount =
1655  HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1656  HttpHeaderStats[0].ccParsedCount =
1657  HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1658  HttpHeaderStats[0].destroyedCount =
1659  HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1660  HttpHeaderStats[0].busyDestroyedCount =
1661  HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1662 
1663  for (const auto &stats: HttpHeaderStats)
1665 
1666  /* field stats for all messages */
1667  storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1668 
1669  storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1670  "id", "name", "#alive", "%err", "%repeat");
1671 
1672  // scan heaaderTable and output
1673  for (auto h : WholeEnum<Http::HdrType>()) {
1674  auto stats = headerStatsTable[h];
1675  storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1676  Http::HeaderLookupTable.lookup(h).id,
1677  Http::HeaderLookupTable.lookup(h).name,
1678  stats.aliveCount,
1679  xpercent(stats.errCount, stats.parsCount),
1680  xpercent(stats.repCount, stats.seenCount));
1681  }
1682 
1683  storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1684  HttpHeaderStats[hoRequest].parsedCount,
1685  HttpHeaderStats[hoReply].parsedCount,
1686  HttpHeaderStats[0].parsedCount);
1687  storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1688 }
1689 
1690 int
1691 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1692 {
1693  int result = 0;
1694  const char *pos = NULL;
1695  const char *item;
1696  int ilen;
1697  int mlen = strlen(member);
1698 
1700 
1701  String header (getStrOrList(id));
1702 
1703  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1704  if (strncasecmp(item, member, mlen) == 0
1705  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1706  result = 1;
1707  break;
1708  }
1709  }
1710 
1711  return result;
1712 }
1713 
1714 int
1715 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1716 {
1717  int result = 0;
1718  const char *pos = NULL;
1719  const char *item;
1720  int ilen;
1721  int mlen = strlen(member);
1722 
1723  assert(name);
1724 
1725  String header (getByName(name));
1726 
1727  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1728  if (strncasecmp(item, member, mlen) == 0
1729  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1730  result = 1;
1731  break;
1732  }
1733  }
1734 
1735  return result;
1736 }
1737 
1738 void
1740 {
1742 
1743  const HttpHeaderEntry *e;
1745  int headers_deleted = 0;
1746  while ((e = getEntry(&pos))) {
1747  Http::HdrType id = e->id;
1748  if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1749  delAt(pos, headers_deleted);
1750  CBIT_CLR(mask, id);
1751  }
1752  }
1753 }
1754 
1755 void
1757 {
1759  /* anything that matches Connection list member will be deleted */
1760  String strConnection;
1761 
1762  (void) getList(Http::HdrType::CONNECTION, &strConnection);
1763  const HttpHeaderEntry *e;
1765  /*
1766  * think: on-average-best nesting of the two loops (hdrEntry
1767  * and strListItem) @?@
1768  */
1769  /*
1770  * maybe we should delete standard stuff ("keep-alive","close")
1771  * from strConnection first?
1772  */
1773 
1774  int headers_deleted = 0;
1775  while ((e = getEntry(&pos))) {
1776  if (strListIsMember(&strConnection, e->name, ','))
1777  delAt(pos, headers_deleted);
1778  }
1779  if (headers_deleted)
1780  refreshMask();
1781  }
1782 }
1783 
int caseCmp(char const *) const
Definition: String.cc:285
void append(const HttpHeader *src)
Definition: HttpHeader.cc:236
SQUIDCEXTERN double xpercent(double part, double whole)
Definition: util.c:40
bool any_registered_header(const Http::HdrType id)
void refreshMask()
Definition: HttpHeader.cc:751
char * buf
Definition: MemBuf.h:134
@ SURROGATE_CONTROL
class Ping::pingStats_ stats
@ hoReply
Definition: HttpHeader.h:37
unsigned int major
major version number
static HttpHeaderEntry * parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
Definition: HttpHeader.cc:1421
HttpHdrRange * getRange() const
Definition: HttpHeader.cc:1249
int delByName(const SBuf &name)
Definition: HttpHeader.cc:676
int relaxed_header_parser
Definition: SquidConfig.h:321
#define DBG_CRITICAL
Definition: Stream.h:40
Definition: ETag.h:18
http_hdr_owner_type owner
Definition: HttpHeader.h:173
SBuf getListMember(const String &list, const char *key, const char delimiter)
Definition: StrList.cc:144
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1192
void httpHdrScStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrSc.cc:272
void removeHopByHopEntries()
Definition: HttpHeader.cc:1739
bool teUnsupported_
Definition: HttpHeader.h:194
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1691
void packInto(Packable *p) const
Definition: HttpHeader.cc:1541
bool isEmpty() const
Definition: SBuf.h:431
HttpHeader & operator=(const HttpHeader &other)
Definition: HttpHeader.cc:175
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
HttpHeaderMask mask
Definition: HttpHeader.h:172
#define CBIT_CLR(mask, bit)
Definition: defines.h:75
HTTP per header statistics.
String getById(Http::HdrType id) const
Definition: HttpHeader.cc:911
HttpHeaderEntry * findLastEntry(Http::HdrType id) const
Definition: HttpHeader.cc:656
@ PROTO_NONE
Definition: ProtocolType.h:24
void httpHdrCcInitModule(void)
Module initialization hook.
Definition: HttpHdrCc.cc:61
void base64_decode_init(struct base64_decode_ctx *ctx)
Definition: base64.c:54
int etagParseInit(ETag *etag, const char *str)
Definition: ETag.cc:29
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:830
void error(char *format,...)
unsigned int minor
minor version number
bool conflictingContentLength_
Definition: HttpHeader.h:191
int parse(const char *header_start, size_t len, Http::ContentLengthInterpreter &interpreter)
Definition: HttpHeader.cc:378
const HttpHeaderStat * dump_stat
Definition: HttpHeader.cc:1590
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:93
const char * ProtocolType_str[]
Definition: SBuf.h:94
StatHist hdrUCountDistr
virtual void append(const char *buf, int size)=0
Appends a c-string to existing packed data.
const char * str
quoted-string
Definition: ETag.h:20
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:842
void rawAppendFinish(const char *start, size_type actualSize)
Definition: SBuf.cc:144
bool needUpdate(const HttpHeader *fresh) const
Definition: HttpHeader.cc:249
int type
Definition: errorpage.cc:152
time_t ParseRfc1123(const char *)
Convert from RFC 1123 style time: "www, DD MMM YYYY hh:mm:ss ZZZ".
Definition: rfc1123.cc:159
static HttpHeaderMask ReplyHeadersMask
Definition: HttpHeader.cc:74
void httpHdrCcUpdateStats(const HttpHdrCc *cc, StatHist *hist)
Definition: HttpHdrCc.cc:321
static bool Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
Definition: HttpHeader.cc:330
String getStrOrList(Http::HdrType id) const
Definition: HttpHeader.cc:876
time_t time
Definition: TimeOrTag.h:21
int hasByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:1715
char ThisCache[RFC2181_MAXHOSTNAMELEN<< 1]
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
ProtocolType protocol
which protocol this version is for
@ PROTO_UNKNOWN
Definition: ProtocolType.h:43
@ CONTENT_LENGTH
number
Definition: testStatHist.cc:16
StatHist fieldTypeDistr
char * rawAppendStart(size_type anticipatedSize)
Definition: SBuf.cc:136
static void httpHeaderFldsPerHdrDumper(StoreEntry *sentry, int idx, double val, double, int count)
Definition: HttpHeader.cc:1610
bool sawBad
whether a malformed Content-Length value was present
bool skipUpdateHeader(const Http::HdrType id) const
Definition: HttpHeader.cc:283
bool any_HdrType_enum_value(const Http::HdrType id)
match any known header type, including OTHER and BAD
static int HeaderEntryParsedCount
Definition: HttpHeader.cc:91
@ WWW_AUTHENTICATE
static std::array< HttpHeaderStat, hoEnd > HttpHeaderStats
Definition: HttpHeader.cc:78
@ hoNone
Definition: HttpHeader.h:32
Definition: Raw.h:21
const char * FormatRfc1123(time_t)
Definition: rfc1123.cc:202
TimeOrTag getTimeOrTag(Http::HdrType id) const
Definition: HttpHeader.cc:1359
void strListAdd(String &str, const char *item, const size_t itemSize, const char delimiter)
Appends the given item of a given size to a delimiter-separated list in str.
Definition: StrList.cc:18
static size_type SizeMaxXXX()
Definition: SquidString.h:71
void putContRange(const HttpHdrContRange *cr)
Definition: HttpHeader.cc:1084
bool hasNamed(const SBuf &s, String *value=0) const
Definition: HttpHeader.cc:919
int base64_decode_final(struct base64_decode_ctx *ctx)
Definition: base64.c:159
int64_t getInt64() const
Definition: HttpHeader.cc:1563
#define NULL
Definition: types.h:166
const char * rawContent() const
Definition: SBuf.cc:509
StatHist scTypeDistr
SQUIDCEXTERN const char * xint64toa(int64_t num)
Definition: util.c:69
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
bool parse(const String &s)
parse a header-string and fill in appropriate values.
Definition: HttpHdrCc.cc:95
const char * getLastStr(Http::HdrType id) const
Definition: HttpHeader.cc:1208
char const * visible_appname_string
void dump(StoreEntry *sentry, StatHistBinDumper *bd) const
Definition: StatHist.cc:171
std::vector< HttpHeaderFieldStat > headerStatsTable(Http::HdrType::enumEnd_)
const char * text
Definition: errorpage.cc:153
char HttpHeaderMask[12]
void putAuth(const char *auth_scheme, const char *realm)
Definition: HttpHeader.cc:1061
std::vector< HttpHeaderEntry *, PoolingAllocator< HttpHeaderEntry * > > entries
Definition: HttpHeader.h:171
void putWarning(const int code, const char *const text)
add a Warning header
Definition: HttpHeader.cc:1132
Definition: MemBuf.h:24
SBuf StringToSBuf(const String &s)
create a new SBuf from a String by copying contents
Definition: StringConvert.h:17
static const CharacterSet TCHAR
Definition: CharacterSet.h:105
HttpHdrSc * getSc() const
Definition: HttpHeader.cc:1268
void insertEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:791
void clean()
Definition: MemBuf.cc:110
unsigned char code
Definition: html_quote.c:20
bool httpHeaderParseOffset(const char *start, int64_t *value, char **endPtr)
int delById(Http::HdrType id)
Definition: HttpHeader.cc:695
Http::HdrType id
Definition: HttpHeader.h:63
HttpHdrSc * httpHdrScParseCreate(const String &str)
Definition: HttpHdrSc.cc:60
void httpHeaderInitModule(void)
Definition: HttpHeader.cc:119
void putExt(const char *name, const char *value)
Definition: HttpHeader.cc:1141
int64_t getInt64(Http::HdrType id) const
Definition: HttpHeader.cc:1162
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:765
time_t getTime(Http::HdrType id) const
Definition: HttpHeader.cc:1175
SBuf getAuthToken(Http::HdrType id, const char *auth_scheme) const
Definition: HttpHeader.cc:1304
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1043
#define assert(EX)
Definition: assert.h:19
int base64_decode_update(struct base64_decode_ctx *ctx, size_t *dst_length, uint8_t *dst, size_t src_length, const char *src)
Definition: base64.c:129
char const * termedBuf() const
Definition: SquidString.h:92
void httpHdrScInitModule(void)
Definition: HttpHdrSc.cc:49
#define CBIT_TEST(mask, bit)
Definition: defines.h:76
bool any_valid_header(const Http::HdrType id)
match any valid header type, including OTHER but not BAD
int getInt(Http::HdrType id) const
Definition: HttpHeader.cc:1149
@ hoEnd
Definition: HttpHeader.h:41
@ TRANSFER_ENCODING
int valid
Definition: TimeOrTag.h:22
const HeaderTableRecord & lookup(const char *buf, const std::size_t len) const
look record type up by name (C-string and length)
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:568
HttpHeaderMask * owner_mask
ETag tag
Definition: TimeOrTag.h:20
static int sc[16]
Definition: smbdes.c:121
const char * c_str()
Definition: SBuf.cc:516
HttpHdrContRange * httpHdrContRangeParseCreate(const char *str)
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:415
static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error)
Definition: HttpHeader.cc:1572
SBuf & append(const SBuf &S)
Definition: SBuf.cc:185
SQUIDCEXTERN const char * xitoa(int num)
Definition: util.c:60
void putCc(const HttpHdrCc *cc)
Definition: HttpHeader.cc:1068
void removeConnectionHeaderEntries()
Definition: HttpHeader.cc:1756
ETag getETag(Http::HdrType id) const
Definition: HttpHeader.cc:1346
const HeaderLookupTable_t HeaderLookupTable
const char * getStringPrefix(const char *str, size_t sz)
String getByName(const SBuf &name) const
Definition: HttpHeader.cc:902
@ PROTO_HTTPS
Definition: ProtocolType.h:27
void putInt(Http::HdrType id, int number)
Definition: HttpHeader.cc:1025
HttpHeaderEntry * findEntry(Http::HdrType id) const
Definition: HttpHeader.cc:631
static HttpHdrRange * ParseCreate(const String *range_spec)
@ PROTO_HTTP
Definition: ProtocolType.h:25
static void httpHeaderRegisterWithCacheManager(void)
Definition: HttpHeader.cc:111
void httpHdrCcStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrCc.cc:331
struct SquidConfig::@110 onoff
SBuf getByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:971
int has(Http::HdrType id) const
Definition: HttpHeader.cc:991
bool getByIdIfPresent(Http::HdrType id, String *result) const
Definition: HttpHeader.cc:925
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1052
size_type size() const
Definition: SquidString.h:73
static void httpHeaderFieldStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHeader.cc:1593
int getInt() const
Definition: HttpHeader.cc:1551
#define Must(condition)
Definition: TextException.h:71
void assign(const char *str, int len)
Definition: String.cc:89
int httpHeaderParseInt(const char *start, int *value)
#define DBG_IMPORTANT
Definition: Stream.h:41
void updateWarnings()
Definition: HttpHeader.cc:270
void Controller::create() STUB void Controller Controller nil
HttpHeaderEntry(Http::HdrType id, const SBuf &name, const char *value)
Definition: HttpHeader.cc:1389
const char * label
SQUIDCEXTERN double xdiv(double nom, double denom)
Definition: util.c:53
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:229
static void httpHeaderStoreReport(StoreEntry *e)
Definition: HttpHeader.cc:1650
SBuf getListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:982
static HttpHeaderMask RequestHeadersMask
Definition: HttpHeader.cc:71
void compact()
Definition: HttpHeader.cc:740
void update(const HttpHeader *fresh)
Definition: HttpHeader.cc:294
void putSc(HttpHdrSc *sc)
Definition: HttpHeader.cc:1116
#define CBIT_SET(mask, bit)
Definition: defines.h:74
int caseCmp(const SBuf &S, const size_type n) const
shorthand version for case-insensitive compare()
Definition: SBuf.h:283
void delAt(HttpHeaderPos pos, int &headers_deleted)
Definition: HttpHeader.cc:723
const char * headerWideProblem
worst header-wide problem found (or nil)
#define xisspace(x)
Definition: xis.h:17
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:612
StatHist ccTypeDistr
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
void httpHdrContRangePackInto(const HttpHdrContRange *range, Packable *p)
void httpHeaderMaskInit(HttpHeaderMask *mask, int value)
#define false
Definition: GnuRegex.c:233
void packInto(Packable *p) const
Definition: HttpHdrCc.cc:247
HttpHdrContRange * getContRange() const
Definition: HttpHeader.cc:1290
A const & min(A const &lhs, A const &rhs)
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=0)
Definition: HttpHeader.cc:999
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:17
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
@ hoRequest
Definition: HttpHeader.h:36
char const * rawBuf() const
Definition: SquidString.h:86
void putRange(const HttpHdrRange *range)
Definition: HttpHeader.cc:1100
void clean()
Definition: HttpHeader.cc:190
static void httpHeaderStatDump(const HttpHeaderStat *hs, StoreEntry *e)
Definition: HttpHeader.cc:1619
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1223
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1034
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1535
#define BASE64_DECODE_LENGTH(length)
Definition: base64.h:120
class SquidConfig Config
Definition: SquidConfig.cc:12
http_hdr_owner_type
Definition: HttpHeader.h:31
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:86
@ PROXY_AUTHORIZATION
void packInto(Packable *p) const

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors