HttpHeader.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 55 HTTP Header */
10 
11 #include "squid.h"
12 #include "base/CharacterSet.h"
13 #include "base/EnumIterator.h"
14 #include "base64.h"
15 #include "globals.h"
17 #include "HttpHdrCc.h"
18 #include "HttpHdrContRange.h"
19 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
20 #include "HttpHeader.h"
21 #include "HttpHeaderFieldInfo.h"
22 #include "HttpHeaderStat.h"
23 #include "HttpHeaderTools.h"
24 #include "MemBuf.h"
25 #include "mgr/Registration.h"
26 #include "mime_header.h"
27 #include "profiler/Profiler.h"
28 #include "rfc1123.h"
29 #include "sbuf/StringConvert.h"
30 #include "SquidConfig.h"
31 #include "StatHist.h"
32 #include "Store.h"
33 #include "StrList.h"
34 #include "TimeOrTag.h"
35 #include "util.h"
36 
37 #include <algorithm>
38 
39 /* XXX: the whole set of API managing the entries vector should be rethought
40  * after the parse4r-ng effort is complete.
41  */
42 
43 /*
44  * On naming conventions:
45  *
46  * HTTP/1.1 defines message-header as
47  *
48  * message-header = field-name ":" [ field-value ] CRLF
49  * field-name = token
50  * field-value = *( field-content | LWS )
51  *
52  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
53  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
54  *
55  * HttpHeader is an object that represents all message-headers in a message.
56  * HttpHeader does not manage start-line.
57  *
58  * HttpHeader is implemented as a collection of header "entries".
59  * An entry is a (field_id, field_name, field_value) triplet.
60  */
61 
62 /*
63  * local constants and vars
64  */
65 
66 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
67 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
68 
69 /* request-only headers. Used for cachemgr */
70 static HttpHeaderMask RequestHeadersMask; /* set run-time using RequestHeaders */
71 
72 /* reply-only headers. Used for cachemgr */
73 static HttpHeaderMask ReplyHeadersMask; /* set run-time using ReplyHeaders */
74 
75 /* header accounting */
76 // NP: keep in sync with enum http_hdr_owner_type
78  HttpHeaderStat(/*hoNone*/ "all", NULL),
79 #if USE_HTCP
80  HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
81 #endif
82  HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
83  HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
84 #if USE_OPENSSL
85  /* hoErrorDetail */
86 #endif
87  /* hoEnd */
88 };
90 
91 static int HeaderEntryParsedCount = 0;
92 
93 /*
94  * forward declarations and local routines
95  */
96 
97 class StoreEntry;
98 
99 // update parse statistics for header id; if error is true also account
100 // for errors and write to debug log what happened
101 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
102 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
104 static void httpHeaderStoreReport(StoreEntry * e);
105 
106 /*
107  * Module initialization routines
108  */
109 
110 static void
112 {
113  Mgr::RegisterAction("http_headers",
114  "HTTP Header Statistics",
115  httpHeaderStoreReport, 0, 1);
116 }
117 
118 void
120 {
121  /* check that we have enough space for masks */
123 
124  // masks are needed for stats page still
125  for (auto h : WholeEnum<Http::HdrType>()) {
126  if (Http::HeaderLookupTable.lookup(h).request)
128  if (Http::HeaderLookupTable.lookup(h).reply)
130  }
131 
132  /* header stats initialized by class constructor */
134 
135  /* init dependent modules */
138 
140 }
141 
142 /*
143  * HttpHeader Implementation
144  */
145 
146 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
147 {
148  entries.reserve(32);
150 }
151 
152 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
153 {
154  assert(anOwner > hoNone && anOwner < hoEnd);
155  debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
156  entries.reserve(32);
158 }
159 
160 // XXX: Delete as unused, expensive, and violating copy semantics by skipping Warnings
161 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
162 {
163  entries.reserve(other.entries.capacity());
165  update(&other); // will update the mask as well
166 }
167 
169 {
170  clean();
171 }
172 
173 // XXX: Delete as unused, expensive, and violating assignment semantics by skipping Warnings
174 HttpHeader &
176 {
177  if (this != &other) {
178  // we do not really care, but the caller probably does
179  assert(owner == other.owner);
180  clean();
181  update(&other); // will update the mask as well
182  len = other.len;
185  }
186  return *this;
187 }
188 
189 void
191 {
192 
193  assert(owner > hoNone && owner < hoEnd);
194  debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
195 
196  PROF_start(HttpHeaderClean);
197 
198  if (owner <= hoReply) {
199  /*
200  * An unfortunate bug. The entries array is initialized
201  * such that count is set to zero. httpHeaderClean() seems to
202  * be called both when 'hdr' is created, and destroyed. Thus,
203  * we accumulate a large number of zero counts for 'hdr' before
204  * it is ever used. Can't think of a good way to fix it, except
205  * adding a state variable that indicates whether or not 'hdr'
206  * has been used. As a hack, just never count zero-sized header
207  * arrays.
208  */
209  if (!entries.empty())
211 
213 
215  } // if (owner <= hoReply)
216 
217  for (HttpHeaderEntry *e : entries) {
218  if (e == nullptr)
219  continue;
220  if (!Http::any_valid_header(e->id)) {
221  debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
222  } else {
223  if (owner <= hoReply)
225  delete e;
226  }
227  }
228 
229  entries.clear();
231  len = 0;
233  teUnsupported_ = false;
234  PROF_stop(HttpHeaderClean);
235 }
236 
237 /* append entries (also see httpHeaderUpdate) */
238 void
240 {
241  assert(src);
242  assert(src != this);
243  debugs(55, 7, "appending hdr: " << this << " += " << src);
244 
245  for (auto e : src->entries) {
246  if (e)
247  addEntry(e->clone());
248  }
249 }
250 
251 bool
253 {
254  // our 1xx Warnings must be removed
255  for (const auto e: entries) {
256  // TODO: Move into HttpHeaderEntry::is1xxWarning() before official commit.
257  if (e && e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1))
258  return true;
259  }
260 
261  for (const auto e: fresh->entries) {
262  if (!e || skipUpdateHeader(e->id))
263  continue;
264  String value;
265  if (!hasNamed(e->name, &value) ||
266  (value != fresh->getByName(e->name)))
267  return true;
268  }
269  return false;
270 }
271 
272 void
274 {
275  int count = 0;
277 
278  // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
279  while (HttpHeaderEntry *e = getEntry(&pos)) {
280  if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
281  delAt(pos, count);
282  }
283 }
284 
285 bool
287 {
288  return
289  // RFC 7234, section 4.3.4: use header fields other than Warning
290  (id == Http::HdrType::WARNING) ||
291  // TODO: Consider updating Vary headers after comparing the magnitude of
292  // the required changes (and/or cache losses) with compliance gains.
293  (id == Http::HdrType::VARY);
294 }
295 
296 void
298 {
299  assert(fresh);
300  assert(this != fresh);
301 
302  updateWarnings();
303 
304  const HttpHeaderEntry *e;
306 
307  while ((e = fresh->getEntry(&pos))) {
308  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
309 
310  if (skipUpdateHeader(e->id))
311  continue;
312 
313  if (e->id != Http::HdrType::OTHER)
314  delById(e->id);
315  else
316  delByName(e->name);
317  }
318 
319  pos = HttpHeaderInitPos;
320  while ((e = fresh->getEntry(&pos))) {
321  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
322 
323  if (skipUpdateHeader(e->id))
324  continue;
325 
326  debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
327 
328  addEntry(e->clone());
329  }
330 }
331 
332 bool
333 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
334 {
335  /*
336  * parse_start points to the first line of HTTP message *headers*,
337  * not including the request or status lines
338  */
339  const size_t end = headersEnd(*parse_start, l);
340 
341  if (end) {
342  *blk_start = *parse_start;
343  *blk_end = *parse_start + end - 1;
344  assert(**blk_end == '\n');
345  // Point blk_end to the first character after the last header field.
346  // In other words, blk_end should point to the CR?LF header terminator.
347  if (end > 1 && *(*blk_end - 1) == '\r')
348  --(*blk_end);
349  *parse_start += end;
350  }
351  return end;
352 }
353 
354 int
355 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz, Http::ContentLengthInterpreter &clen)
356 {
357  const char *parse_start = buf;
358  const char *blk_start, *blk_end;
359  hdr_sz = 0;
360 
361  if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
362  // XXX: do not parse non-isolated headers even if the connection is closed.
363  // Treat unterminated headers as "partial headers" framing errors.
364  if (!atEnd)
365  return 0;
366  blk_start = parse_start;
367  blk_end = blk_start + strlen(blk_start);
368  }
369 
370  if (parse(blk_start, blk_end - blk_start, clen)) {
371  hdr_sz = parse_start - buf;
372  return 1;
373  }
374  return -1;
375 }
376 
377 // XXX: callers treat this return as boolean.
378 // XXX: A better mechanism is needed to signal different types of error.
379 // lexicon, syntax, semantics, validation, access policy - are all (ab)using 'return 0'
380 int
381 HttpHeader::parse(const char *header_start, size_t hdrLen, Http::ContentLengthInterpreter &clen)
382 {
383  const char *field_ptr = header_start;
384  const char *header_end = header_start + hdrLen; // XXX: remove
385  int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
386 
387  PROF_start(HttpHeaderParse);
388 
389  assert(header_start && header_end);
390  debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
392 
393  char *nulpos;
394  if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
395  debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
396  getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
397  PROF_stop(HttpHeaderParse);
398  clean();
399  return 0;
400  }
401 
402  /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
403  * continuation lines start with a (single) space or tab */
404  while (field_ptr < header_end) {
405  const char *field_start = field_ptr;
406  const char *field_end;
407 
408  const char *hasBareCr = nullptr;
409  size_t lines = 0;
410  do {
411  const char *this_line = field_ptr;
412  field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
413  ++lines;
414 
415  if (!field_ptr) {
416  // missing <LF>
417  PROF_stop(HttpHeaderParse);
418  clean();
419  return 0;
420  }
421 
422  field_end = field_ptr;
423 
424  ++field_ptr; /* Move to next line */
425 
426  if (field_end > this_line && field_end[-1] == '\r') {
427  --field_end; /* Ignore CR LF */
428 
429  if (owner == hoRequest && field_end > this_line) {
430  bool cr_only = true;
431  for (const char *p = this_line; p < field_end && cr_only; ++p) {
432  if (*p != '\r')
433  cr_only = false;
434  }
435  if (cr_only) {
436  debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
437  "header field to prevent request smuggling attacks: {" <<
438  getStringPrefix(header_start, hdrLen) << "}");
439  PROF_stop(HttpHeaderParse);
440  clean();
441  return 0;
442  }
443  }
444  }
445 
446  /* Barf on stray CR characters */
447  if (memchr(this_line, '\r', field_end - this_line)) {
448  hasBareCr = "bare CR";
449  debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
450  getStringPrefix(field_start, field_end-field_start) << "}");
451 
453  char *p = (char *) this_line; /* XXX Warning! This destroys original header content and violates specifications somewhat */
454 
455  while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
456  *p = ' ';
457  ++p;
458  }
459  } else {
460  PROF_stop(HttpHeaderParse);
461  clean();
462  return 0;
463  }
464  }
465 
466  if (this_line + 1 == field_end && this_line > field_start) {
467  debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
468  getStringPrefix(header_start, hdrLen) << "}");
469  PROF_stop(HttpHeaderParse);
470  clean();
471  return 0;
472  }
473  } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
474 
475  if (field_start == field_end) {
476  if (field_ptr < header_end) {
477  debugs(55, warnOnError, "WARNING: unparsable HTTP header field near {" <<
478  getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
479  PROF_stop(HttpHeaderParse);
480  clean();
481  return 0;
482  }
483 
484  break; /* terminating blank line */
485  }
486 
487  const auto e = HttpHeaderEntry::parse(field_start, field_end, owner);
488  if (!e) {
489  debugs(55, warnOnError, "WARNING: unparsable HTTP header field {" <<
490  getStringPrefix(field_start, field_end-field_start) << "}");
491  debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
492 
493  PROF_stop(HttpHeaderParse);
494  clean();
495  return 0;
496  }
497 
498  if (lines > 1 || hasBareCr) {
499  const auto framingHeader = (e->id == Http::HdrType::CONTENT_LENGTH || e->id == Http::HdrType::TRANSFER_ENCODING);
500  if (framingHeader) {
501  if (!hasBareCr) // already warned about bare CRs
502  debugs(55, warnOnError, "WARNING: obs-fold in framing-sensitive " << e->name << ": " << e->value);
503  delete e;
504  PROF_stop(HttpHeaderParse);
505  clean();
506  return 0;
507  }
508  }
509 
510  if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
511  delete e;
512 
514  continue; // clen has printed any necessary warnings
515 
516  PROF_stop(HttpHeaderParse);
517  clean();
518  return 0;
519  }
520 
521  addEntry(e);
522  }
523 
524  if (clen.headerWideProblem) {
525  debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
526  " Content-Length field values in" <<
527  Raw("header", header_start, hdrLen));
528  }
529 
530  String rawTe;
531  if (clen.prohibitedAndIgnored()) {
532  // prohibitedAndIgnored() includes trailer header blocks
533  // being parsed as a case to forbid/ignore these headers.
534 
535  // RFC 7230 section 3.3.2: A server MUST NOT send a Content-Length
536  // header field in any response with a status code of 1xx (Informational)
537  // or 204 (No Content). And RFC 7230 3.3.3#1 tells recipients to ignore
538  // such Content-Lengths.
540  debugs(55, 3, "Content-Length is " << clen.prohibitedAndIgnored());
541 
542  // RFC 7230 section 3.3.1 has the same criteria forbid Transfer-Encoding
544  debugs(55, 3, "Transfer-Encoding is " << clen.prohibitedAndIgnored());
545  teUnsupported_ = true;
546  }
547 
549  // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
550  // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
552  // and clen state becomes irrelevant
553 
554  if (rawTe.caseCmp("chunked") == 0) {
555  ; // leave header present for chunked() method
556  } else if (rawTe.caseCmp("identity") == 0) { // deprecated. no coding
558  } else {
559  // This also rejects multiple encodings until we support them properly.
560  debugs(55, warnOnError, "WARNING: unsupported Transfer-Encoding used by client: " << rawTe);
561  teUnsupported_ = true;
562  }
563 
564  } else if (clen.sawBad) {
565  // ensure our callers do not accidentally see bad Content-Length values
567  conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
568  } else if (clen.needsSanitizing) {
569  // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
570  // ensure our callers see a clean Content-Length value or none at all
572  if (clen.sawGood) {
574  debugs(55, 5, "sanitized Content-Length to be " << clen.value);
575  }
576  }
577 
578  PROF_stop(HttpHeaderParse);
579  return 1; /* even if no fields where found, it is a valid header */
580 }
581 
582 /* packs all the entries using supplied packer */
583 void
584 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
585 {
587  const HttpHeaderEntry *e;
588  assert(p);
589  debugs(55, 7, this << " into " << p <<
590  (mask_sensitive_info ? " while masking" : ""));
591  /* pack all entries one by one */
592  while ((e = getEntry(&pos))) {
593  if (!mask_sensitive_info) {
594  e->packInto(p);
595  continue;
596  }
597 
598  bool maskThisEntry = false;
599  switch (e->id) {
602  maskThisEntry = true;
603  break;
604 
607  maskThisEntry = (cmd->value == "PASS");
608  break;
609 
610  default:
611  break;
612  }
613  if (maskThisEntry) {
614  p->append(e->name.rawContent(), e->name.length());
615  p->append(": ** NOT DISPLAYED **\r\n", 23);
616  } else {
617  e->packInto(p);
618  }
619 
620  }
621  /* Pack in the "special" entries */
622 
623  /* Cache-Control */
624 }
625 
626 /* returns next valid entry */
629 {
630  assert(pos);
631  assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
632 
633  for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
634  if (entries[*pos])
635  return static_cast<HttpHeaderEntry*>(entries[*pos]);
636  }
637 
638  return NULL;
639 }
640 
641 /*
642  * returns a pointer to a specified entry if any
643  * note that we return one entry so it does not make much sense to ask for
644  * "list" headers
645  */
648 {
650  assert(!Http::HeaderLookupTable.lookup(id).list);
651 
652  /* check mask first */
653 
654  if (!CBIT_TEST(mask, id))
655  return NULL;
656 
657  /* looks like we must have it, do linear search */
658  for (auto e : entries) {
659  if (e && e->id == id)
660  return e;
661  }
662 
663  /* hm.. we thought it was there, but it was not found */
664  assert(false);
665  return nullptr; /* not reached */
666 }
667 
668 /*
669  * same as httpHeaderFindEntry
670  */
673 {
675  assert(!Http::HeaderLookupTable.lookup(id).list);
676 
677  /* check mask first */
678  if (!CBIT_TEST(mask, id))
679  return NULL;
680 
681  for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
682  if (*e && (*e)->id == id)
683  return *e;
684  }
685 
686  /* hm.. we thought it was there, but it was not found */
687  assert(false);
688  return nullptr; /* not reached */
689 }
690 
691 int
693 {
694  int count = 0;
696  httpHeaderMaskInit(&mask, 0); /* temporal inconsistency */
697  debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
698 
699  while (const HttpHeaderEntry *e = getEntry(&pos)) {
700  if (!e->name.caseCmp(name))
701  delAt(pos, count);
702  else
703  CBIT_SET(mask, e->id);
704  }
705 
706  return count;
707 }
708 
709 /* deletes all entries with a given id, returns the #entries deleted */
710 int
712 {
713  debugs(55, 8, this << " del-by-id " << id);
715 
716  if (!CBIT_TEST(mask, id))
717  return 0;
718 
719  int count = 0;
720 
722  while (HttpHeaderEntry *e = getEntry(&pos)) {
723  if (e->id == id)
724  delAt(pos, count); // deletes e
725  }
726 
727  CBIT_CLR(mask, id);
728  assert(count);
729  return count;
730 }
731 
732 /*
733  * deletes an entry at pos and leaves a gap; leaving a gap makes it
734  * possible to iterate(search) and delete fields at the same time
735  * NOTE: Does not update the header mask. Caller must follow up with
736  * a call to refreshMask() if headers_deleted was incremented.
737  */
738 void
739 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
740 {
741  HttpHeaderEntry *e;
742  assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
743  e = static_cast<HttpHeaderEntry*>(entries[pos]);
744  entries[pos] = NULL;
745  /* decrement header length, allow for ": " and crlf */
746  len -= e->name.length() + 2 + e->value.size() + 2;
747  assert(len >= 0);
748  delete e;
749  ++headers_deleted;
750 }
751 
752 /*
753  * Compacts the header storage
754  */
755 void
757 {
758  // TODO: optimize removal, or possibly make it so that's not needed.
759  entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
760  entries.end());
761 }
762 
763 /*
764  * Refreshes the header mask. Required after delAt() calls.
765  */
766 void
768 {
770  debugs(55, 7, "refreshing the mask in hdr " << this);
771  for (auto e : entries) {
772  if (e)
773  CBIT_SET(mask, e->id);
774  }
775 }
776 
777 /* appends an entry;
778  * does not call e->clone() so one should not reuse "*e"
779  */
780 void
782 {
783  assert(e);
785  assert(e->name.length());
786 
787  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
788 
789  if (e->id != Http::HdrType::BAD_HDR) {
790  if (CBIT_TEST(mask, e->id)) {
791  ++ headerStatsTable[e->id].repCount;
792  } else {
793  CBIT_SET(mask, e->id);
794  }
795  }
796 
797  entries.push_back(e);
798 
799  /* increment header length, allow for ": " and crlf */
800  len += e->name.length() + 2 + e->value.size() + 2;
801 }
802 
803 /* inserts an entry;
804  * does not call e->clone() so one should not reuse "*e"
805  */
806 void
808 {
809  assert(e);
811 
812  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
813 
814  // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
815  if (CBIT_TEST(mask, e->id)) {
816  ++ headerStatsTable[e->id].repCount;
817  } else {
818  CBIT_SET(mask, e->id);
819  }
820 
821  entries.insert(entries.begin(),e);
822 
823  /* increment header length, allow for ": " and crlf */
824  len += e->name.length() + 2 + e->value.size() + 2;
825 }
826 
827 bool
829 {
830  debugs(55, 9, this << " joining for id " << id);
831  /* only fields from ListHeaders array can be "listed" */
832  assert(Http::HeaderLookupTable.lookup(id).list);
833 
834  if (!CBIT_TEST(mask, id))
835  return false;
836 
837  for (auto e: entries) {
838  if (e && e->id == id)
839  strListAdd(s, e->value.termedBuf(), ',');
840  }
841 
842  /*
843  * note: we might get an empty (size==0) string if there was an "empty"
844  * header. This results in an empty length String, which may have a NULL
845  * buffer.
846  */
847  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
848  if (!s->size())
849  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
850  else
851  debugs(55, 6, this << ": joined for id " << id << ": " << s);
852 
853  return true;
854 }
855 
856 /* return a list of entries with the same id separated by ',' and ws */
857 String
859 {
860  HttpHeaderEntry *e;
862  debugs(55, 9, this << "joining for id " << id);
863  /* only fields from ListHeaders array can be "listed" */
864  assert(Http::HeaderLookupTable.lookup(id).list);
865 
866  if (!CBIT_TEST(mask, id))
867  return String();
868 
869  String s;
870 
871  while ((e = getEntry(&pos))) {
872  if (e->id == id)
873  strListAdd(&s, e->value.termedBuf(), ',');
874  }
875 
876  /*
877  * note: we might get an empty (size==0) string if there was an "empty"
878  * header. This results in an empty length String, which may have a NULL
879  * buffer.
880  */
881  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
882  if (!s.size())
883  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
884  else
885  debugs(55, 6, this << ": joined for id " << id << ": " << s);
886 
887  return s;
888 }
889 
890 /* return a string or list of entries with the same id separated by ',' and ws */
891 String
893 {
894  HttpHeaderEntry *e;
895 
896  if (Http::HeaderLookupTable.lookup(id).list)
897  return getList(id);
898 
899  if ((e = findEntry(id)))
900  return e->value;
901 
902  return String();
903 }
904 
905 /*
906  * Returns the value of the specified header and/or an undefined String.
907  */
908 String
909 HttpHeader::getByName(const char *name) const
910 {
911  String result;
912  // ignore presence: return undefined string if an empty header is present
913  (void)hasNamed(name, strlen(name), &result);
914  return result;
915 }
916 
917 String
918 HttpHeader::getByName(const SBuf &name) const
919 {
920  String result;
921  // ignore presence: return undefined string if an empty header is present
922  (void)hasNamed(name, &result);
923  return result;
924 }
925 
926 String
928 {
929  String result;
930  (void)getByIdIfPresent(id, &result);
931  return result;
932 }
933 
934 bool
935 HttpHeader::hasNamed(const SBuf &s, String *result) const
936 {
937  return hasNamed(s.rawContent(), s.length(), result);
938 }
939 
940 bool
942 {
943  if (id == Http::HdrType::BAD_HDR)
944  return false;
945  if (!has(id))
946  return false;
947  if (result)
948  *result = getStrOrList(id);
949  return true;
950 }
951 
952 bool
953 HttpHeader::hasNamed(const char *name, unsigned int namelen, String *result) const
954 {
955  Http::HdrType id;
957  HttpHeaderEntry *e;
958 
959  assert(name);
960 
961  /* First try the quick path */
962  id = Http::HeaderLookupTable.lookup(name,namelen).id;
963 
964  if (id != Http::HdrType::BAD_HDR) {
965  if (getByIdIfPresent(id, result))
966  return true;
967  }
968 
969  /* Sorry, an unknown header name. Do linear search */
970  bool found = false;
971  while ((e = getEntry(&pos))) {
972  if (e->id == Http::HdrType::OTHER && e->name.length() == namelen && e->name.caseCmp(name, namelen) == 0) {
973  found = true;
974  if (!result)
975  break;
976  strListAdd(result, e->value.termedBuf(), ',');
977  }
978  }
979 
980  return found;
981 }
982 
983 /*
984  * Returns a the value of the specified list member, if any.
985  */
986 SBuf
987 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
988 {
989  assert(name);
990  const auto header = getByName(name);
991  return ::getListMember(header, member, separator);
992 }
993 
994 /*
995  * returns a the value of the specified list member, if any.
996  */
997 SBuf
998 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
999 {
1001  const auto header = getStrOrList(id);
1002  return ::getListMember(header, member, separator);
1003 }
1004 
1005 /* test if a field is present */
1006 int
1008 {
1010  debugs(55, 9, this << " lookup for " << id);
1011  return CBIT_TEST(mask, id);
1012 }
1013 
1014 void
1016 {
1017  // TODO: do not add Via header for messages where Squid itself
1018  // generated the message (i.e., Downloader or ESI) there should be no Via header added at all.
1019 
1020  if (Config.onoff.via) {
1021  SBuf buf;
1022  // RFC 7230 section 5.7.1.: protocol-name is omitted when
1023  // the received protocol is HTTP.
1026  buf.appendf("%s/", AnyP::ProtocolType_str[ver.protocol]);
1027  buf.appendf("%d.%d %s", ver.major, ver.minor, ThisCache);
1028  const HttpHeader *hdr = from ? from : this;
1029  SBuf strVia = StringToSBuf(hdr->getList(Http::HdrType::VIA));
1030  if (!strVia.isEmpty())
1031  strVia.append(", ", 2);
1032  strVia.append(buf);
1033  // XXX: putStr() still suffers from String size limits
1034  Must(strVia.length() < String::SizeMaxXXX());
1036  putStr(Http::HdrType::VIA, strVia.c_str());
1037  }
1038 }
1039 
1040 void
1042 {
1044  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1045  assert(number >= 0);
1046  addEntry(new HttpHeaderEntry(id, SBuf(), xitoa(number)));
1047 }
1048 
1049 void
1051 {
1053  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1054  assert(number >= 0);
1056 }
1057 
1058 void
1060 {
1062  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1063  assert(htime >= 0);
1064  addEntry(new HttpHeaderEntry(id, SBuf(), mkrfc1123(htime)));
1065 }
1066 
1067 void
1068 HttpHeader::putStr(Http::HdrType id, const char *str)
1069 {
1071  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1072  assert(str);
1073  addEntry(new HttpHeaderEntry(id, SBuf(), str));
1074 }
1075 
1076 void
1077 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1078 {
1079  assert(auth_scheme && realm);
1080  httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1081 }
1082 
1083 void
1085 {
1086  assert(cc);
1087  /* remove old directives if any */
1089  /* pack into mb */
1090  MemBuf mb;
1091  mb.init();
1092  cc->packInto(&mb);
1093  /* put */
1095  /* cleanup */
1096  mb.clean();
1097 }
1098 
1099 void
1101 {
1102  assert(cr);
1103  /* remove old directives if any */
1105  /* pack into mb */
1106  MemBuf mb;
1107  mb.init();
1108  httpHdrContRangePackInto(cr, &mb);
1109  /* put */
1111  /* cleanup */
1112  mb.clean();
1113 }
1114 
1115 void
1117 {
1118  assert(range);
1119  /* remove old directives if any */
1121  /* pack into mb */
1122  MemBuf mb;
1123  mb.init();
1124  range->packInto(&mb);
1125  /* put */
1127  /* cleanup */
1128  mb.clean();
1129 }
1130 
1131 void
1133 {
1134  assert(sc);
1135  /* remove old directives if any */
1137  /* pack into mb */
1138  MemBuf mb;
1139  mb.init();
1140  sc->packInto(&mb);
1141  /* put */
1143  /* cleanup */
1144  mb.clean();
1145 }
1146 
1147 void
1148 HttpHeader::putWarning(const int code, const char *const text)
1149 {
1150  char buf[512];
1151  snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1153 }
1154 
1155 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1156 void
1157 HttpHeader::putExt(const char *name, const char *value)
1158 {
1159  assert(name && value);
1160  debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1161  addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, SBuf(name), value));
1162 }
1163 
1164 int
1166 {
1168  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1169  HttpHeaderEntry *e;
1170 
1171  if ((e = findEntry(id)))
1172  return e->getInt();
1173 
1174  return -1;
1175 }
1176 
1177 int64_t
1179 {
1181  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1182  HttpHeaderEntry *e;
1183 
1184  if ((e = findEntry(id)))
1185  return e->getInt64();
1186 
1187  return -1;
1188 }
1189 
1190 time_t
1192 {
1193  HttpHeaderEntry *e;
1194  time_t value = -1;
1196  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1197 
1198  if ((e = findEntry(id))) {
1199  value = parse_rfc1123(e->value.termedBuf());
1200  httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1201  }
1202 
1203  return value;
1204 }
1205 
1206 /* sync with httpHeaderGetLastStr */
1207 const char *
1209 {
1210  HttpHeaderEntry *e;
1212  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1213 
1214  if ((e = findEntry(id))) {
1215  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1216  return e->value.termedBuf();
1217  }
1218 
1219  return NULL;
1220 }
1221 
1222 /* unusual */
1223 const char *
1225 {
1226  HttpHeaderEntry *e;
1228  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1229 
1230  if ((e = findLastEntry(id))) {
1231  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1232  return e->value.termedBuf();
1233  }
1234 
1235  return NULL;
1236 }
1237 
1238 HttpHdrCc *
1240 {
1242  return NULL;
1243  PROF_start(HttpHeader_getCc);
1244 
1245  String s;
1247 
1248  HttpHdrCc *cc=new HttpHdrCc();
1249 
1250  if (!cc->parse(s)) {
1251  delete cc;
1252  cc = NULL;
1253  }
1254 
1256 
1257  if (cc)
1258  httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1259 
1261 
1262  PROF_stop(HttpHeader_getCc);
1263 
1264  return cc;
1265 }
1266 
1267 HttpHdrRange *
1269 {
1270  HttpHdrRange *r = NULL;
1271  HttpHeaderEntry *e;
1272  /* some clients will send "Request-Range" _and_ *matching* "Range"
1273  * who knows, some clients might send Request-Range only;
1274  * this "if" should work correctly in both cases;
1275  * hopefully no clients send mismatched headers! */
1276 
1277  if ((e = findEntry(Http::HdrType::RANGE)) ||
1280  httpHeaderNoteParsedEntry(e->id, e->value, !r);
1281  }
1282 
1283  return r;
1284 }
1285 
1286 HttpHdrSc *
1288 {
1290  return NULL;
1291 
1292  String s;
1293 
1295 
1297 
1299 
1300  if (sc)
1301  sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1302 
1304 
1305  return sc;
1306 }
1307 
1310 {
1311  HttpHdrContRange *cr = NULL;
1312  HttpHeaderEntry *e;
1313 
1316  httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1317  }
1318 
1319  return cr;
1320 }
1321 
1322 SBuf
1323 HttpHeader::getAuthToken(Http::HdrType id, const char *auth_scheme) const
1324 {
1325  const char *field;
1326  int l;
1327  assert(auth_scheme);
1328  field = getStr(id);
1329 
1330  static const SBuf nil;
1331  if (!field) /* no authorization field */
1332  return nil;
1333 
1334  l = strlen(auth_scheme);
1335 
1336  if (!l || strncasecmp(field, auth_scheme, l)) /* wrong scheme */
1337  return nil;
1338 
1339  field += l;
1340 
1341  if (!xisspace(*field)) /* wrong scheme */
1342  return nil;
1343 
1344  /* skip white space */
1345  for (; field && xisspace(*field); ++field);
1346 
1347  if (!*field) /* no authorization cookie */
1348  return nil;
1349 
1350  const auto fieldLen = strlen(field);
1351  SBuf result;
1352  char *decodedAuthToken = result.rawAppendStart(BASE64_DECODE_LENGTH(fieldLen));
1353  struct base64_decode_ctx ctx;
1354  base64_decode_init(&ctx);
1355  size_t decodedLen = 0;
1356  if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), fieldLen, field) ||
1357  !base64_decode_final(&ctx)) {
1358  return nil;
1359  }
1360  result.rawAppendFinish(decodedAuthToken, decodedLen);
1361  return result;
1362 }
1363 
1364 ETag
1366 {
1367  ETag etag = {NULL, -1};
1368  HttpHeaderEntry *e;
1369  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag); /* must be of an appropriate type */
1370 
1371  if ((e = findEntry(id)))
1372  etagParseInit(&etag, e->value.termedBuf());
1373 
1374  return etag;
1375 }
1376 
1377 TimeOrTag
1379 {
1380  TimeOrTag tot;
1381  HttpHeaderEntry *e;
1382  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag); /* must be of an appropriate type */
1383  memset(&tot, 0, sizeof(tot));
1384 
1385  if ((e = findEntry(id))) {
1386  const char *str = e->value.termedBuf();
1387  /* try as an ETag */
1388 
1389  if (etagParseInit(&tot.tag, str)) {
1390  tot.valid = tot.tag.str != NULL;
1391  tot.time = -1;
1392  } else {
1393  /* or maybe it is time? */
1394  tot.time = parse_rfc1123(str);
1395  tot.valid = tot.time >= 0;
1396  tot.tag.str = NULL;
1397  }
1398  }
1399 
1400  assert(tot.time < 0 || !tot.tag.str); /* paranoid */
1401  return tot;
1402 }
1403 
1404 /*
1405  * HttpHeaderEntry
1406  */
1407 
1408 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const SBuf &aName, const char *aValue)
1409 {
1411  id = anId;
1412 
1413  if (id != Http::HdrType::OTHER)
1415  else
1416  name = aName;
1417 
1418  value = aValue;
1419 
1420  if (id != Http::HdrType::BAD_HDR)
1421  ++ headerStatsTable[id].aliveCount;
1422 
1423  debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1424 }
1425 
1427 {
1428  debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1429 
1430  if (id != Http::HdrType::BAD_HDR) {
1431  assert(headerStatsTable[id].aliveCount);
1432  -- headerStatsTable[id].aliveCount;
1433  id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1434  }
1435 
1436 }
1437 
1438 /* parses and inits header entry, returns true/false */
1440 HttpHeaderEntry::parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
1441 {
1442  /* note: name_start == field_start */
1443  const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1444  int name_len = name_end ? name_end - field_start :0;
1445  const char *value_start = field_start + name_len + 1; /* skip ':' */
1446  /* note: value_end == field_end */
1447 
1449 
1450  /* do we have a valid field name within this field? */
1451 
1452  if (!name_len || name_end > field_end)
1453  return NULL;
1454 
1455  if (name_len > 65534) {
1456  /* String must be LESS THAN 64K and it adds a terminating NULL */
1457  // TODO: update this to show proper name_len in Raw markup, but not print all that
1458  debugs(55, 2, "ignoring huge header field (" << Raw("field_start", field_start, 100) << "...)");
1459  return NULL;
1460  }
1461 
1462  /*
1463  * RFC 7230 section 3.2.4:
1464  * "No whitespace is allowed between the header field-name and colon.
1465  * ...
1466  * A server MUST reject any received request message that contains
1467  * whitespace between a header field-name and colon with a response code
1468  * of 400 (Bad Request). A proxy MUST remove any such whitespace from a
1469  * response message before forwarding the message downstream."
1470  */
1471  if (xisspace(field_start[name_len - 1])) {
1472 
1473  if (msgType == hoRequest)
1474  return nullptr;
1475 
1476  // for now, also let relaxed parser remove this BWS from any non-HTTP messages
1477  const bool stripWhitespace = (msgType == hoReply) ||
1479  if (!stripWhitespace)
1480  return nullptr; // reject if we cannot strip
1481 
1482  debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1483  "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1484 
1485  while (name_len > 0 && xisspace(field_start[name_len - 1]))
1486  --name_len;
1487 
1488  if (!name_len) {
1489  debugs(55, 2, "found header with only whitespace for name");
1490  return NULL;
1491  }
1492  }
1493 
1494  /* RFC 7230 section 3.2:
1495  *
1496  * header-field = field-name ":" OWS field-value OWS
1497  * field-name = token
1498  * token = 1*TCHAR
1499  */
1500  for (const char *pos = field_start; pos < (field_start+name_len); ++pos) {
1501  if (!CharacterSet::TCHAR[*pos]) {
1502  debugs(55, 2, "found header with invalid characters in " <<
1503  Raw("field-name", field_start, min(name_len,100)) << "...");
1504  return nullptr;
1505  }
1506  }
1507 
1508  /* now we know we can parse it */
1509 
1510  debugs(55, 9, "parsing HttpHeaderEntry: near '" << getStringPrefix(field_start, field_end-field_start) << "'");
1511 
1512  /* is it a "known" field? */
1513  Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1514  debugs(55, 9, "got hdr-id=" << id);
1515 
1516  SBuf theName;
1517 
1518  String value;
1519 
1520  if (id == Http::HdrType::BAD_HDR)
1521  id = Http::HdrType::OTHER;
1522 
1523  /* set field name */
1524  if (id == Http::HdrType::OTHER)
1525  theName.append(field_start, name_len);
1526  else
1527  theName = Http::HeaderLookupTable.lookup(id).name;
1528 
1529  /* trim field value */
1530  while (value_start < field_end && xisspace(*value_start))
1531  ++value_start;
1532 
1533  while (value_start < field_end && xisspace(field_end[-1]))
1534  --field_end;
1535 
1536  if (field_end - value_start > 65534) {
1537  /* String must be LESS THAN 64K and it adds a terminating NULL */
1538  debugs(55, 2, "WARNING: found '" << theName << "' header of " << (field_end - value_start) << " bytes");
1539  return NULL;
1540  }
1541 
1542  /* set field value */
1543  value.assign(value_start, field_end - value_start);
1544 
1545  if (id != Http::HdrType::BAD_HDR)
1546  ++ headerStatsTable[id].seenCount;
1547 
1548  debugs(55, 9, "parsed HttpHeaderEntry: '" << theName << ": " << value << "'");
1549 
1550  return new HttpHeaderEntry(id, theName, value.termedBuf());
1551 }
1552 
1555 {
1556  return new HttpHeaderEntry(id, name, value.termedBuf());
1557 }
1558 
1559 void
1561 {
1562  assert(p);
1563  p->append(name.rawContent(), name.length());
1564  p->append(": ", 2);
1565  p->append(value.rawBuf(), value.size());
1566  p->append("\r\n", 2);
1567 }
1568 
1569 int
1571 {
1572  int val = -1;
1573  int ok = httpHeaderParseInt(value.termedBuf(), &val);
1574  httpHeaderNoteParsedEntry(id, value, ok == 0);
1575  /* XXX: Should we check ok - ie
1576  * return ok ? -1 : value;
1577  */
1578  return val;
1579 }
1580 
1581 int64_t
1583 {
1584  int64_t val = -1;
1585  const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1586  httpHeaderNoteParsedEntry(id, value, !ok);
1587  return val; // remains -1 if !ok (XXX: bad method API)
1588 }
1589 
1590 static void
1592 {
1593  if (id != Http::HdrType::BAD_HDR)
1594  ++ headerStatsTable[id].parsCount;
1595 
1596  if (error) {
1597  if (id != Http::HdrType::BAD_HDR)
1598  ++ headerStatsTable[id].errCount;
1599  debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1600  }
1601 }
1602 
1603 /*
1604  * Reports
1605  */
1606 
1607 /* tmp variable used to pass stat info to dumpers */
1608 extern const HttpHeaderStat *dump_stat; /* argh! */
1610 
1611 void
1612 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1613 {
1614  const int id = static_cast<int>(val);
1615  const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1616  const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1617  int visible = count > 0;
1618  /* for entries with zero count, list only those that belong to current type of message */
1619 
1620  if (!visible && valid_id && dump_stat->owner_mask)
1621  visible = CBIT_TEST(*dump_stat->owner_mask, id);
1622 
1623  if (visible)
1624  storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1625  id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1626 }
1627 
1628 static void
1629 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1630 {
1631  if (count)
1632  storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1633  idx, (int) val, count,
1634  xpercent(count, dump_stat->destroyedCount));
1635 }
1636 
1637 static void
1639 {
1640  assert(hs);
1641  assert(e);
1642 
1643  dump_stat = hs;
1644  storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1645  storeAppendPrintf(e, "\nField type distribution\n");
1646  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1647  "id", "name", "count", "#/header");
1649  storeAppendPrintf(e, "\nCache-control directives distribution\n");
1650  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1651  "id", "name", "count", "#/cc_field");
1653  storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1654  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1655  "id", "name", "count", "#/sc_field");
1657  storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1658  storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1659  "id", "#flds", "count", "%total");
1661  storeAppendPrintf(e, "\n");
1662  dump_stat = NULL;
1663 }
1664 
1665 void
1667 {
1668  int i;
1669  assert(e);
1670 
1679 
1680  for (i = 1; i < HttpHeaderStatCount; ++i) {
1682  }
1683 
1684  /* field stats for all messages */
1685  storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1686 
1687  storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1688  "id", "name", "#alive", "%err", "%repeat");
1689 
1690  // scan heaaderTable and output
1691  for (auto h : WholeEnum<Http::HdrType>()) {
1692  auto stats = headerStatsTable[h];
1693  storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1694  Http::HeaderLookupTable.lookup(h).id,
1695  Http::HeaderLookupTable.lookup(h).name,
1696  stats.aliveCount,
1697  xpercent(stats.errCount, stats.parsCount),
1698  xpercent(stats.repCount, stats.seenCount));
1699  }
1700 
1701  storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1702  HttpHeaderStats[hoRequest].parsedCount,
1703  HttpHeaderStats[hoReply].parsedCount,
1704  HttpHeaderStats[0].parsedCount);
1705  storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1706 }
1707 
1708 int
1709 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1710 {
1711  int result = 0;
1712  const char *pos = NULL;
1713  const char *item;
1714  int ilen;
1715  int mlen = strlen(member);
1716 
1718 
1719  String header (getStrOrList(id));
1720 
1721  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1722  if (strncasecmp(item, member, mlen) == 0
1723  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1724  result = 1;
1725  break;
1726  }
1727  }
1728 
1729  return result;
1730 }
1731 
1732 int
1733 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1734 {
1735  int result = 0;
1736  const char *pos = NULL;
1737  const char *item;
1738  int ilen;
1739  int mlen = strlen(member);
1740 
1741  assert(name);
1742 
1743  String header (getByName(name));
1744 
1745  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1746  if (strncasecmp(item, member, mlen) == 0
1747  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1748  result = 1;
1749  break;
1750  }
1751  }
1752 
1753  return result;
1754 }
1755 
1756 void
1758 {
1760 
1761  const HttpHeaderEntry *e;
1763  int headers_deleted = 0;
1764  while ((e = getEntry(&pos))) {
1765  Http::HdrType id = e->id;
1766  if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1767  delAt(pos, headers_deleted);
1768  CBIT_CLR(mask, id);
1769  }
1770  }
1771 }
1772 
1773 void
1775 {
1777  /* anything that matches Connection list member will be deleted */
1778  String strConnection;
1779 
1780  (void) getList(Http::HdrType::CONNECTION, &strConnection);
1781  const HttpHeaderEntry *e;
1783  /*
1784  * think: on-average-best nesting of the two loops (hdrEntry
1785  * and strListItem) @?@
1786  */
1787  /*
1788  * maybe we should delete standard stuff ("keep-alive","close")
1789  * from strConnection first?
1790  */
1791 
1792  int headers_deleted = 0;
1793  while ((e = getEntry(&pos))) {
1794  if (strListIsMember(&strConnection, e->name, ','))
1795  delAt(pos, headers_deleted);
1796  }
1797  if (headers_deleted)
1798  refreshMask();
1799  }
1800 }
1801 
const SQUIDCEXTERN char * xitoa(int num)
Definition: util.c:79
static HttpHeaderStat HttpHeaderStats[]
Definition: HttpHeader.cc:77
int caseCmp(char const *) const
Definition: String.cc:299
void append(const HttpHeader *src)
Definition: HttpHeader.cc:239
SQUIDCEXTERN double xpercent(double part, double whole)
Definition: util.c:54
bool any_registered_header(const Http::HdrType id)
void refreshMask()
Definition: HttpHeader.cc:767
char * buf
Definition: MemBuf.h:134
@ SURROGATE_CONTROL
class Ping::pingStats_ stats
@ hoReply
Definition: HttpHeader.h:37
unsigned int major
major version number
static HttpHeaderEntry * parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
Definition: HttpHeader.cc:1440
HttpHdrRange * getRange() const
Definition: HttpHeader.cc:1268
int delByName(const SBuf &name)
Definition: HttpHeader.cc:692
int relaxed_header_parser
Definition: SquidConfig.h:321
void httpHeaderFieldStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHeader.cc:1612
#define countof(arr)
Definition: defines.h:186
const char * rawBuf() const
Definition: SquidString.h:85
Definition: ETag.h:17
http_hdr_owner_type owner
Definition: HttpHeader.h:173
SBuf getListMember(const String &list, const char *key, const char delimiter)
Definition: StrList.cc:144
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1208
void httpHdrScStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrSc.cc:272
void removeHopByHopEntries()
Definition: HttpHeader.cc:1757
bool teUnsupported_
Definition: HttpHeader.h:194
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1709
void packInto(Packable *p) const
Definition: HttpHeader.cc:1560
bool isEmpty() const
Definition: SBuf.h:420
HttpHeader & operator=(const HttpHeader &other)
Definition: HttpHeader.cc:175
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
HttpHeaderMask mask
Definition: HttpHeader.h:172
#define CBIT_CLR(mask, bit)
Definition: defines.h:113
HTTP per header statistics.
String getById(Http::HdrType id) const
Definition: HttpHeader.cc:927
HttpHeaderEntry * findLastEntry(Http::HdrType id) const
Definition: HttpHeader.cc:672
@ PROTO_NONE
Definition: ProtocolType.h:24
void httpHdrCcInitModule(void)
Module initialization hook.
Definition: HttpHdrCc.cc:60
void base64_decode_init(struct base64_decode_ctx *ctx)
Definition: base64.c:54
int etagParseInit(ETag *etag, const char *str)
Definition: ETag.cc:29
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:901
void error(char *format,...)
unsigned int minor
minor version number
bool conflictingContentLength_
Definition: HttpHeader.h:191
int parse(const char *header_start, size_t len, Http::ContentLengthInterpreter &interpreter)
Definition: HttpHeader.cc:381
const HttpHeaderStat * dump_stat
Definition: HttpHeader.cc:1609
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
const char * ProtocolType_str[]
Definition: SBuf.h:86
StatHist hdrUCountDistr
virtual void append(const char *buf, int size)=0
Appends a c-string to existing packed data.
const char * str
quoted-string
Definition: ETag.h:20
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:858
void rawAppendFinish(const char *start, size_type actualSize)
Definition: SBuf.cc:144
bool needUpdate(const HttpHeader *fresh) const
Definition: HttpHeader.cc:252
int type
Definition: errorpage.cc:152
const SQUIDCEXTERN char * xint64toa(int64_t num)
Definition: util.c:88
static HttpHeaderMask ReplyHeadersMask
Definition: HttpHeader.cc:73
void httpHdrCcUpdateStats(const HttpHdrCc *cc, StatHist *hist)
Definition: HttpHdrCc.cc:320
static bool Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
Definition: HttpHeader.cc:333
#define DBG_CRITICAL
Definition: Debug.h:45
String getStrOrList(Http::HdrType id) const
Definition: HttpHeader.cc:892
time_t time
Definition: TimeOrTag.h:21
int hasByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:1733
char ThisCache[RFC2181_MAXHOSTNAMELEN<< 1]
#define PROF_stop(probename)
Definition: Profiler.h:63
#define DBG_IMPORTANT
Definition: Debug.h:46
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
ProtocolType protocol
which protocol this version is for
@ PROTO_UNKNOWN
Definition: ProtocolType.h:43
@ CONTENT_LENGTH
number
Definition: testStatHist.cc:16
StatHist fieldTypeDistr
char * rawAppendStart(size_type anticipatedSize)
Definition: SBuf.cc:136
static void httpHeaderFldsPerHdrDumper(StoreEntry *sentry, int idx, double val, double, int count)
Definition: HttpHeader.cc:1629
bool sawBad
whether a malformed Content-Length value was present
bool skipUpdateHeader(const Http::HdrType id) const
Definition: HttpHeader.cc:286
bool any_HdrType_enum_value(const Http::HdrType id)
match any known header type, including OTHER and BAD
static int HeaderEntryParsedCount
Definition: HttpHeader.cc:91
@ WWW_AUTHENTICATE
@ hoNone
Definition: HttpHeader.h:32
Definition: Debug.h:188
TimeOrTag getTimeOrTag(Http::HdrType id) const
Definition: HttpHeader.cc:1378
void strListAdd(String &str, const char *item, const size_t itemSize, const char delimiter)
Appends the given item of a given size to a delimiter-separated list in str.
Definition: StrList.cc:18
static size_type SizeMaxXXX()
Definition: SquidString.h:70
void putContRange(const HttpHdrContRange *cr)
Definition: HttpHeader.cc:1100
bool hasNamed(const SBuf &s, String *value=0) const
Definition: HttpHeader.cc:935
int base64_decode_final(struct base64_decode_ctx *ctx)
Definition: base64.c:159
int64_t getInt64() const
Definition: HttpHeader.cc:1582
#define NULL
Definition: types.h:166
const char * rawContent() const
Definition: SBuf.cc:519
StatHist scTypeDistr
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
time_t parse_rfc1123(const char *str)
Definition: rfc1123.c:159
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
bool parse(const String &s)
parse a header-string and fill in appropriate values.
Definition: HttpHdrCc.cc:94
const char * getLastStr(Http::HdrType id) const
Definition: HttpHeader.cc:1224
void dump(StoreEntry *sentry, StatHistBinDumper *bd) const
Definition: StatHist.cc:173
std::vector< HttpHeaderFieldStat > headerStatsTable(Http::HdrType::enumEnd_)
const char * text
Definition: errorpage.cc:153
char HttpHeaderMask[12]
void putAuth(const char *auth_scheme, const char *realm)
Definition: HttpHeader.cc:1077
std::vector< HttpHeaderEntry *, PoolingAllocator< HttpHeaderEntry * > > entries
Definition: HttpHeader.h:171
void putWarning(const int code, const char *const text)
add a Warning header
Definition: HttpHeader.cc:1148
Definition: MemBuf.h:23
SBuf StringToSBuf(const String &s)
create a new SBuf from a String by copying contents
Definition: StringConvert.h:17
static const CharacterSet TCHAR
Definition: CharacterSet.h:105
HttpHdrSc * getSc() const
Definition: HttpHeader.cc:1287
void insertEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:807
void clean()
Definition: MemBuf.cc:113
unsigned char code
Definition: html_quote.c:20
bool httpHeaderParseOffset(const char *start, int64_t *value, char **endPtr)
int delById(Http::HdrType id)
Definition: HttpHeader.cc:711
Http::HdrType id
Definition: HttpHeader.h:63
HttpHdrSc * httpHdrScParseCreate(const String &str)
Definition: HttpHdrSc.cc:60
void httpHeaderInitModule(void)
Definition: HttpHeader.cc:119
void putExt(const char *name, const char *value)
Definition: HttpHeader.cc:1157
int64_t getInt64(Http::HdrType id) const
Definition: HttpHeader.cc:1178
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:781
time_t getTime(Http::HdrType id) const
Definition: HttpHeader.cc:1191
const char * visible_appname_string
SBuf getAuthToken(Http::HdrType id, const char *auth_scheme) const
Definition: HttpHeader.cc:1323
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1059
void count(double val)
Definition: StatHist.cc:57
#define assert(EX)
Definition: assert.h:19
int base64_decode_update(struct base64_decode_ctx *ctx, size_t *dst_length, uint8_t *dst, size_t src_length, const char *src)
Definition: base64.c:129
void httpHdrScInitModule(void)
Definition: HttpHdrSc.cc:49
#define CBIT_TEST(mask, bit)
Definition: defines.h:114
bool any_valid_header(const Http::HdrType id)
match any valid header type, including OTHER but not BAD
int getInt(Http::HdrType id) const
Definition: HttpHeader.cc:1165
@ hoEnd
Definition: HttpHeader.h:41
@ TRANSFER_ENCODING
int valid
Definition: TimeOrTag.h:22
const HeaderTableRecord & lookup(const char *buf, const std::size_t len) const
look record type up by name (C-string and length)
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:584
HttpHeaderMask * owner_mask
ETag tag
Definition: TimeOrTag.h:20
static int sc[16]
Definition: smbdes.c:121
const char * c_str()
Definition: SBuf.cc:526
HttpHdrContRange * httpHdrContRangeParseCreate(const char *str)
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:404
static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error)
Definition: HttpHeader.cc:1591
SBuf & append(const SBuf &S)
Definition: SBuf.cc:195
struct SquidConfig::@112 onoff
const char * mkrfc1123(time_t)
Definition: rfc1123.c:202
void putCc(const HttpHdrCc *cc)
Definition: HttpHeader.cc:1084
void removeConnectionHeaderEntries()
Definition: HttpHeader.cc:1774
ETag getETag(Http::HdrType id) const
Definition: HttpHeader.cc:1365
const HeaderLookupTable_t HeaderLookupTable
const char * getStringPrefix(const char *str, size_t sz)
String getByName(const SBuf &name) const
Definition: HttpHeader.cc:918
@ PROTO_HTTPS
Definition: ProtocolType.h:27
void putInt(Http::HdrType id, int number)
Definition: HttpHeader.cc:1041
HttpHeaderEntry * findEntry(Http::HdrType id) const
Definition: HttpHeader.cc:647
static HttpHdrRange * ParseCreate(const String *range_spec)
@ PROTO_HTTP
Definition: ProtocolType.h:25
static int HttpHeaderStatCount
Definition: HttpHeader.cc:89
static void httpHeaderRegisterWithCacheManager(void)
Definition: HttpHeader.cc:111
const char * termedBuf() const
Definition: SquidString.h:91
void httpHdrCcStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrCc.cc:330
SBuf getByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:987
int has(Http::HdrType id) const
Definition: HttpHeader.cc:1007
bool getByIdIfPresent(Http::HdrType id, String *result) const
Definition: HttpHeader.cc:941
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1068
size_type size() const
Definition: SquidString.h:72
int getInt() const
Definition: HttpHeader.cc:1570
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
void assign(const char *str, int len)
Definition: String.cc:94
int httpHeaderParseInt(const char *start, int *value)
void updateWarnings()
Definition: HttpHeader.cc:273
void Controller::create() STUB void Controller Controller nil
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
HttpHeaderEntry(Http::HdrType id, const SBuf &name, const char *value)
Definition: HttpHeader.cc:1408
const char * label
SQUIDCEXTERN double xdiv(double nom, double denom)
Definition: util.c:72
static void httpHeaderStoreReport(StoreEntry *e)
Definition: HttpHeader.cc:1666
SBuf getListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:998
static HttpHeaderMask RequestHeadersMask
Definition: HttpHeader.cc:70
void compact()
Definition: HttpHeader.cc:756
void update(const HttpHeader *fresh)
Definition: HttpHeader.cc:297
void putSc(HttpHdrSc *sc)
Definition: HttpHeader.cc:1132
#define CBIT_SET(mask, bit)
Definition: defines.h:112
int caseCmp(const SBuf &S, const size_type n) const
shorthand version for case-insensitive compare()
Definition: SBuf.h:272
void delAt(HttpHeaderPos pos, int &headers_deleted)
Definition: HttpHeader.cc:739
const char * headerWideProblem
worst header-wide problem found (or nil)
#define xisspace(x)
Definition: xis.h:17
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:628
StatHist ccTypeDistr
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
void httpHdrContRangePackInto(const HttpHdrContRange *range, Packable *p)
void httpHeaderMaskInit(HttpHeaderMask *mask, int value)
#define false
Definition: GnuRegex.c:233
void packInto(Packable *p) const
Definition: HttpHdrCc.cc:246
HttpHdrContRange * getContRange() const
Definition: HttpHeader.cc:1309
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=0)
Definition: HttpHeader.cc:1015
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:16
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
const A & min(A const &lhs, A const &rhs)
@ hoRequest
Definition: HttpHeader.h:36
void putRange(const HttpHdrRange *range)
Definition: HttpHeader.cc:1116
void clean()
Definition: HttpHeader.cc:190
void const char * buf
Definition: stub_helper.cc:16
static void httpHeaderStatDump(const HttpHeaderStat *hs, StoreEntry *e)
Definition: HttpHeader.cc:1638
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1239
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1050
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1554
#define BASE64_DECODE_LENGTH(length)
Definition: base64.h:120
#define PROF_start(probename)
Definition: Profiler.h:62
class SquidConfig Config
Definition: SquidConfig.cc:12
http_hdr_owner_type
Definition: HttpHeader.h:31
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:86
@ PROXY_AUTHORIZATION
void packInto(Packable *p) const

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors