HttpHeader.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 55 HTTP Header */
10 
11 #include "squid.h"
12 #include "base/CharacterSet.h"
13 #include "base/EnumIterator.h"
14 #include "base64.h"
15 #include "globals.h"
17 #include "HttpHdrCc.h"
18 #include "HttpHdrContRange.h"
19 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
20 #include "HttpHeader.h"
21 #include "HttpHeaderFieldInfo.h"
22 #include "HttpHeaderStat.h"
23 #include "HttpHeaderTools.h"
24 #include "MemBuf.h"
25 #include "mgr/Registration.h"
26 #include "mime_header.h"
27 #include "rfc1123.h"
28 #include "sbuf/StringConvert.h"
29 #include "SquidConfig.h"
30 #include "StatHist.h"
31 #include "Store.h"
32 #include "StrList.h"
33 #include "TimeOrTag.h"
34 #include "util.h"
35 
36 #include <algorithm>
37 #include <array>
38 
39 /* XXX: the whole set of API managing the entries vector should be rethought
40  * after the parse4r-ng effort is complete.
41  */
42 
43 /*
44  * On naming conventions:
45  *
46  * HTTP/1.1 defines message-header as
47  *
48  * message-header = field-name ":" [ field-value ] CRLF
49  * field-name = token
50  * field-value = *( field-content | LWS )
51  *
52  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
53  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
54  *
55  * HttpHeader is an object that represents all message-headers in a message.
56  * HttpHeader does not manage start-line.
57  *
58  * HttpHeader is implemented as a collection of header "entries".
59  * An entry is a (field_id, field_name, field_value) triplet.
60  */
61 
62 /*
63  * local constants and vars
64  */
65 
66 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
67 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
68 
69 /* request-only headers. Used for cachemgr */
70 static HttpHeaderMask RequestHeadersMask; /* set run-time using RequestHeaders */
71 
72 /* reply-only headers. Used for cachemgr */
73 static HttpHeaderMask ReplyHeadersMask; /* set run-time using ReplyHeaders */
74 
75 /* header accounting */
76 // NP: keep in sync with enum http_hdr_owner_type
77 static std::array<HttpHeaderStat, hoEnd> HttpHeaderStats = {
78  HttpHeaderStat(/*hoNone*/ "all", NULL),
79 #if USE_HTCP
80  HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
81 #endif
82  HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
83  HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
84 #if USE_OPENSSL
85  , HttpHeaderStat(/*hoErrorDetail*/ "error detail templates", nullptr)
86 #endif
87  /* hoEnd */
88 };
89 
90 static int HeaderEntryParsedCount = 0;
91 
92 /*
93  * forward declarations and local routines
94  */
95 
96 class StoreEntry;
97 
98 // update parse statistics for header id; if error is true also account
99 // for errors and write to debug log what happened
100 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
101 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
103 static void httpHeaderStoreReport(StoreEntry * e);
104 
105 /*
106  * Module initialization routines
107  */
108 
109 static void
111 {
112  Mgr::RegisterAction("http_headers",
113  "HTTP Header Statistics",
114  httpHeaderStoreReport, 0, 1);
115 }
116 
117 void
119 {
120  /* check that we have enough space for masks */
122 
123  // masks are needed for stats page still
124  for (auto h : WholeEnum<Http::HdrType>()) {
125  if (Http::HeaderLookupTable.lookup(h).request)
127  if (Http::HeaderLookupTable.lookup(h).reply)
129  }
130 
131  assert(HttpHeaderStats[0].label && "httpHeaderInitModule() called via main()");
132  assert(HttpHeaderStats[hoEnd-1].label && "HttpHeaderStats created with all elements");
133 
134  /* init dependent modules */
137 
139 }
140 
141 /*
142  * HttpHeader Implementation
143  */
144 
145 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
146 {
147  entries.reserve(32);
149 }
150 
151 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
152 {
153  assert(anOwner > hoNone && anOwner < hoEnd);
154  debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
155  entries.reserve(32);
157 }
158 
159 // XXX: Delete as unused, expensive, and violating copy semantics by skipping Warnings
160 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
161 {
162  entries.reserve(other.entries.capacity());
164  update(&other); // will update the mask as well
165 }
166 
168 {
169  clean();
170 }
171 
172 // XXX: Delete as unused, expensive, and violating assignment semantics by skipping Warnings
173 HttpHeader &
175 {
176  if (this != &other) {
177  // we do not really care, but the caller probably does
178  assert(owner == other.owner);
179  clean();
180  update(&other); // will update the mask as well
181  len = other.len;
184  }
185  return *this;
186 }
187 
188 void
190 {
191 
192  assert(owner > hoNone && owner < hoEnd);
193  debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
194 
195  if (owner <= hoReply) {
196  /*
197  * An unfortunate bug. The entries array is initialized
198  * such that count is set to zero. httpHeaderClean() seems to
199  * be called both when 'hdr' is created, and destroyed. Thus,
200  * we accumulate a large number of zero counts for 'hdr' before
201  * it is ever used. Can't think of a good way to fix it, except
202  * adding a state variable that indicates whether or not 'hdr'
203  * has been used. As a hack, just never count zero-sized header
204  * arrays.
205  */
206  if (!entries.empty())
207  HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
208 
209  ++ HttpHeaderStats[owner].destroyedCount;
210 
211  HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
212  } // if (owner <= hoReply)
213 
214  for (HttpHeaderEntry *e : entries) {
215  if (e == nullptr)
216  continue;
217  if (!Http::any_valid_header(e->id)) {
218  debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
219  } else {
220  if (owner <= hoReply)
221  HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
222  delete e;
223  }
224  }
225 
226  entries.clear();
228  len = 0;
230  teUnsupported_ = false;
231 }
232 
233 /* append entries (also see httpHeaderUpdate) */
234 void
236 {
237  assert(src);
238  assert(src != this);
239  debugs(55, 7, "appending hdr: " << this << " += " << src);
240 
241  for (auto e : src->entries) {
242  if (e)
243  addEntry(e->clone());
244  }
245 }
246 
247 bool
249 {
250  // our 1xx Warnings must be removed
251  for (const auto e: entries) {
252  // TODO: Move into HttpHeaderEntry::is1xxWarning() before official commit.
253  if (e && e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1))
254  return true;
255  }
256 
257  for (const auto e: fresh->entries) {
258  if (!e || skipUpdateHeader(e->id))
259  continue;
260  String value;
261  if (!hasNamed(e->name, &value) ||
262  (value != fresh->getByName(e->name)))
263  return true;
264  }
265  return false;
266 }
267 
268 void
270 {
271  int count = 0;
273 
274  // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
275  while (HttpHeaderEntry *e = getEntry(&pos)) {
276  if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
277  delAt(pos, count);
278  }
279 }
280 
281 bool
283 {
284  return
285  // RFC 7234, section 4.3.4: use header fields other than Warning
286  (id == Http::HdrType::WARNING) ||
287  // TODO: Consider updating Vary headers after comparing the magnitude of
288  // the required changes (and/or cache losses) with compliance gains.
289  (id == Http::HdrType::VARY);
290 }
291 
292 void
294 {
295  assert(fresh);
296  assert(this != fresh);
297 
298  updateWarnings();
299 
300  const HttpHeaderEntry *e;
302 
303  while ((e = fresh->getEntry(&pos))) {
304  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
305 
306  if (skipUpdateHeader(e->id))
307  continue;
308 
309  if (e->id != Http::HdrType::OTHER)
310  delById(e->id);
311  else
312  delByName(e->name);
313  }
314 
315  pos = HttpHeaderInitPos;
316  while ((e = fresh->getEntry(&pos))) {
317  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
318 
319  if (skipUpdateHeader(e->id))
320  continue;
321 
322  debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
323 
324  addEntry(e->clone());
325  }
326 }
327 
328 bool
329 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
330 {
331  /*
332  * parse_start points to the first line of HTTP message *headers*,
333  * not including the request or status lines
334  */
335  const size_t end = headersEnd(*parse_start, l);
336 
337  if (end) {
338  *blk_start = *parse_start;
339  *blk_end = *parse_start + end - 1;
340  assert(**blk_end == '\n');
341  // Point blk_end to the first character after the last header field.
342  // In other words, blk_end should point to the CR?LF header terminator.
343  if (end > 1 && *(*blk_end - 1) == '\r')
344  --(*blk_end);
345  *parse_start += end;
346  }
347  return end;
348 }
349 
350 int
351 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz, Http::ContentLengthInterpreter &clen)
352 {
353  const char *parse_start = buf;
354  const char *blk_start, *blk_end;
355  hdr_sz = 0;
356 
357  if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
358  // XXX: do not parse non-isolated headers even if the connection is closed.
359  // Treat unterminated headers as "partial headers" framing errors.
360  if (!atEnd)
361  return 0;
362  blk_start = parse_start;
363  blk_end = blk_start + strlen(blk_start);
364  }
365 
366  if (parse(blk_start, blk_end - blk_start, clen)) {
367  hdr_sz = parse_start - buf;
368  return 1;
369  }
370  return -1;
371 }
372 
373 // XXX: callers treat this return as boolean.
374 // XXX: A better mechanism is needed to signal different types of error.
375 // lexicon, syntax, semantics, validation, access policy - are all (ab)using 'return 0'
376 int
377 HttpHeader::parse(const char *header_start, size_t hdrLen, Http::ContentLengthInterpreter &clen)
378 {
379  const char *field_ptr = header_start;
380  const char *header_end = header_start + hdrLen; // XXX: remove
381  int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
382 
383  assert(header_start && header_end);
384  debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
385  ++ HttpHeaderStats[owner].parsedCount;
386 
387  char *nulpos;
388  if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
389  debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
390  getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
391  clean();
392  return 0;
393  }
394 
395  /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
396  * continuation lines start with a (single) space or tab */
397  while (field_ptr < header_end) {
398  const char *field_start = field_ptr;
399  const char *field_end;
400 
401  const char *hasBareCr = nullptr;
402  size_t lines = 0;
403  do {
404  const char *this_line = field_ptr;
405  field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
406  ++lines;
407 
408  if (!field_ptr) {
409  // missing <LF>
410  clean();
411  return 0;
412  }
413 
414  field_end = field_ptr;
415 
416  ++field_ptr; /* Move to next line */
417 
418  if (field_end > this_line && field_end[-1] == '\r') {
419  --field_end; /* Ignore CR LF */
420 
421  if (owner == hoRequest && field_end > this_line) {
422  bool cr_only = true;
423  for (const char *p = this_line; p < field_end && cr_only; ++p) {
424  if (*p != '\r')
425  cr_only = false;
426  }
427  if (cr_only) {
428  debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
429  "header field to prevent request smuggling attacks: {" <<
430  getStringPrefix(header_start, hdrLen) << "}");
431  clean();
432  return 0;
433  }
434  }
435  }
436 
437  /* Barf on stray CR characters */
438  if (memchr(this_line, '\r', field_end - this_line)) {
439  hasBareCr = "bare CR";
440  debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
441  getStringPrefix(field_start, field_end-field_start) << "}");
442 
444  char *p = (char *) this_line; /* XXX Warning! This destroys original header content and violates specifications somewhat */
445 
446  while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
447  *p = ' ';
448  ++p;
449  }
450  } else {
451  clean();
452  return 0;
453  }
454  }
455 
456  if (this_line + 1 == field_end && this_line > field_start) {
457  debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
458  getStringPrefix(header_start, hdrLen) << "}");
459  clean();
460  return 0;
461  }
462  } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
463 
464  if (field_start == field_end) {
465  if (field_ptr < header_end) {
466  debugs(55, warnOnError, "WARNING: unparsable HTTP header field near {" <<
467  getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
468  clean();
469  return 0;
470  }
471 
472  break; /* terminating blank line */
473  }
474 
475  const auto e = HttpHeaderEntry::parse(field_start, field_end, owner);
476  if (!e) {
477  debugs(55, warnOnError, "WARNING: unparsable HTTP header field {" <<
478  getStringPrefix(field_start, field_end-field_start) << "}");
479  debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
480 
481  clean();
482  return 0;
483  }
484 
485  if (lines > 1 || hasBareCr) {
486  const auto framingHeader = (e->id == Http::HdrType::CONTENT_LENGTH || e->id == Http::HdrType::TRANSFER_ENCODING);
487  if (framingHeader) {
488  if (!hasBareCr) // already warned about bare CRs
489  debugs(55, warnOnError, "WARNING: obs-fold in framing-sensitive " << e->name << ": " << e->value);
490  delete e;
491  clean();
492  return 0;
493  }
494  }
495 
496  if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
497  delete e;
498 
500  continue; // clen has printed any necessary warnings
501 
502  clean();
503  return 0;
504  }
505 
506  addEntry(e);
507  }
508 
509  if (clen.headerWideProblem) {
510  debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
511  " Content-Length field values in" <<
512  Raw("header", header_start, hdrLen));
513  }
514 
515  String rawTe;
516  if (clen.prohibitedAndIgnored()) {
517  // prohibitedAndIgnored() includes trailer header blocks
518  // being parsed as a case to forbid/ignore these headers.
519 
520  // RFC 7230 section 3.3.2: A server MUST NOT send a Content-Length
521  // header field in any response with a status code of 1xx (Informational)
522  // or 204 (No Content). And RFC 7230 3.3.3#1 tells recipients to ignore
523  // such Content-Lengths.
525  debugs(55, 3, "Content-Length is " << clen.prohibitedAndIgnored());
526 
527  // The same RFC 7230 3.3.3#1-based logic applies to Transfer-Encoding
528  // banned by RFC 7230 section 3.3.1.
530  debugs(55, 3, "Transfer-Encoding is " << clen.prohibitedAndIgnored());
531 
533  // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
534  // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
536  // and clen state becomes irrelevant
537 
538  if (rawTe.caseCmp("chunked") == 0) {
539  ; // leave header present for chunked() method
540  } else if (rawTe.caseCmp("identity") == 0) { // deprecated. no coding
542  } else {
543  // This also rejects multiple encodings until we support them properly.
544  debugs(55, warnOnError, "WARNING: unsupported Transfer-Encoding used by client: " << rawTe);
545  teUnsupported_ = true;
546  }
547 
548  } else if (clen.sawBad) {
549  // ensure our callers do not accidentally see bad Content-Length values
551  conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
552  } else if (clen.needsSanitizing) {
553  // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
554  // ensure our callers see a clean Content-Length value or none at all
556  if (clen.sawGood) {
558  debugs(55, 5, "sanitized Content-Length to be " << clen.value);
559  }
560  }
561 
562  return 1; /* even if no fields where found, it is a valid header */
563 }
564 
565 /* packs all the entries using supplied packer */
566 void
567 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
568 {
570  const HttpHeaderEntry *e;
571  assert(p);
572  debugs(55, 7, this << " into " << p <<
573  (mask_sensitive_info ? " while masking" : ""));
574  /* pack all entries one by one */
575  while ((e = getEntry(&pos))) {
576  if (!mask_sensitive_info) {
577  e->packInto(p);
578  continue;
579  }
580 
581  bool maskThisEntry = false;
582  switch (e->id) {
585  maskThisEntry = true;
586  break;
587 
590  maskThisEntry = (cmd->value == "PASS");
591  break;
592 
593  default:
594  break;
595  }
596  if (maskThisEntry) {
597  p->append(e->name.rawContent(), e->name.length());
598  p->append(": ** NOT DISPLAYED **\r\n", 23);
599  } else {
600  e->packInto(p);
601  }
602 
603  }
604  /* Pack in the "special" entries */
605 
606  /* Cache-Control */
607 }
608 
609 /* returns next valid entry */
612 {
613  assert(pos);
614  assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
615 
616  for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
617  if (entries[*pos])
618  return static_cast<HttpHeaderEntry*>(entries[*pos]);
619  }
620 
621  return NULL;
622 }
623 
624 /*
625  * returns a pointer to a specified entry if any
626  * note that we return one entry so it does not make much sense to ask for
627  * "list" headers
628  */
631 {
633  assert(!Http::HeaderLookupTable.lookup(id).list);
634 
635  /* check mask first */
636 
637  if (!CBIT_TEST(mask, id))
638  return NULL;
639 
640  /* looks like we must have it, do linear search */
641  for (auto e : entries) {
642  if (e && e->id == id)
643  return e;
644  }
645 
646  /* hm.. we thought it was there, but it was not found */
647  assert(false);
648  return nullptr; /* not reached */
649 }
650 
651 /*
652  * same as httpHeaderFindEntry
653  */
656 {
658  assert(!Http::HeaderLookupTable.lookup(id).list);
659 
660  /* check mask first */
661  if (!CBIT_TEST(mask, id))
662  return NULL;
663 
664  for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
665  if (*e && (*e)->id == id)
666  return *e;
667  }
668 
669  /* hm.. we thought it was there, but it was not found */
670  assert(false);
671  return nullptr; /* not reached */
672 }
673 
674 int
676 {
677  int count = 0;
679  httpHeaderMaskInit(&mask, 0); /* temporal inconsistency */
680  debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
681 
682  while (const HttpHeaderEntry *e = getEntry(&pos)) {
683  if (!e->name.caseCmp(name))
684  delAt(pos, count);
685  else
686  CBIT_SET(mask, e->id);
687  }
688 
689  return count;
690 }
691 
692 /* deletes all entries with a given id, returns the #entries deleted */
693 int
695 {
696  debugs(55, 8, this << " del-by-id " << id);
698 
699  if (!CBIT_TEST(mask, id))
700  return 0;
701 
702  int count = 0;
703 
705  while (HttpHeaderEntry *e = getEntry(&pos)) {
706  if (e->id == id)
707  delAt(pos, count); // deletes e
708  }
709 
710  CBIT_CLR(mask, id);
711  assert(count);
712  return count;
713 }
714 
715 /*
716  * deletes an entry at pos and leaves a gap; leaving a gap makes it
717  * possible to iterate(search) and delete fields at the same time
718  * NOTE: Does not update the header mask. Caller must follow up with
719  * a call to refreshMask() if headers_deleted was incremented.
720  */
721 void
722 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
723 {
724  HttpHeaderEntry *e;
725  assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
726  e = static_cast<HttpHeaderEntry*>(entries[pos]);
727  entries[pos] = NULL;
728  /* decrement header length, allow for ": " and crlf */
729  len -= e->name.length() + 2 + e->value.size() + 2;
730  assert(len >= 0);
731  delete e;
732  ++headers_deleted;
733 }
734 
735 /*
736  * Compacts the header storage
737  */
738 void
740 {
741  // TODO: optimize removal, or possibly make it so that's not needed.
742  entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
743  entries.end());
744 }
745 
746 /*
747  * Refreshes the header mask. Required after delAt() calls.
748  */
749 void
751 {
753  debugs(55, 7, "refreshing the mask in hdr " << this);
754  for (auto e : entries) {
755  if (e)
756  CBIT_SET(mask, e->id);
757  }
758 }
759 
760 /* appends an entry;
761  * does not call e->clone() so one should not reuse "*e"
762  */
763 void
765 {
766  assert(e);
768  assert(e->name.length());
769 
770  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
771 
772  if (e->id != Http::HdrType::BAD_HDR) {
773  if (CBIT_TEST(mask, e->id)) {
774  ++ headerStatsTable[e->id].repCount;
775  } else {
776  CBIT_SET(mask, e->id);
777  }
778  }
779 
780  entries.push_back(e);
781 
782  /* increment header length, allow for ": " and crlf */
783  len += e->name.length() + 2 + e->value.size() + 2;
784 }
785 
786 /* inserts an entry;
787  * does not call e->clone() so one should not reuse "*e"
788  */
789 void
791 {
792  assert(e);
794 
795  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
796 
797  // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
798  if (CBIT_TEST(mask, e->id)) {
799  ++ headerStatsTable[e->id].repCount;
800  } else {
801  CBIT_SET(mask, e->id);
802  }
803 
804  entries.insert(entries.begin(),e);
805 
806  /* increment header length, allow for ": " and crlf */
807  len += e->name.length() + 2 + e->value.size() + 2;
808 }
809 
810 bool
812 {
813  debugs(55, 9, this << " joining for id " << id);
814  /* only fields from ListHeaders array can be "listed" */
815  assert(Http::HeaderLookupTable.lookup(id).list);
816 
817  if (!CBIT_TEST(mask, id))
818  return false;
819 
820  for (auto e: entries) {
821  if (e && e->id == id)
822  strListAdd(s, e->value.termedBuf(), ',');
823  }
824 
825  /*
826  * note: we might get an empty (size==0) string if there was an "empty"
827  * header. This results in an empty length String, which may have a NULL
828  * buffer.
829  */
830  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
831  if (!s->size())
832  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
833  else
834  debugs(55, 6, this << ": joined for id " << id << ": " << s);
835 
836  return true;
837 }
838 
839 /* return a list of entries with the same id separated by ',' and ws */
840 String
842 {
843  HttpHeaderEntry *e;
845  debugs(55, 9, this << "joining for id " << id);
846  /* only fields from ListHeaders array can be "listed" */
847  assert(Http::HeaderLookupTable.lookup(id).list);
848 
849  if (!CBIT_TEST(mask, id))
850  return String();
851 
852  String s;
853 
854  while ((e = getEntry(&pos))) {
855  if (e->id == id)
856  strListAdd(&s, e->value.termedBuf(), ',');
857  }
858 
859  /*
860  * note: we might get an empty (size==0) string if there was an "empty"
861  * header. This results in an empty length String, which may have a NULL
862  * buffer.
863  */
864  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
865  if (!s.size())
866  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
867  else
868  debugs(55, 6, this << ": joined for id " << id << ": " << s);
869 
870  return s;
871 }
872 
873 /* return a string or list of entries with the same id separated by ',' and ws */
874 String
876 {
877  HttpHeaderEntry *e;
878 
879  if (Http::HeaderLookupTable.lookup(id).list)
880  return getList(id);
881 
882  if ((e = findEntry(id)))
883  return e->value;
884 
885  return String();
886 }
887 
888 /*
889  * Returns the value of the specified header and/or an undefined String.
890  */
891 String
892 HttpHeader::getByName(const char *name) const
893 {
894  String result;
895  // ignore presence: return undefined string if an empty header is present
896  (void)hasNamed(name, strlen(name), &result);
897  return result;
898 }
899 
900 String
901 HttpHeader::getByName(const SBuf &name) const
902 {
903  String result;
904  // ignore presence: return undefined string if an empty header is present
905  (void)hasNamed(name, &result);
906  return result;
907 }
908 
909 String
911 {
912  String result;
913  (void)getByIdIfPresent(id, &result);
914  return result;
915 }
916 
917 bool
918 HttpHeader::hasNamed(const SBuf &s, String *result) const
919 {
920  return hasNamed(s.rawContent(), s.length(), result);
921 }
922 
923 bool
925 {
926  if (id == Http::HdrType::BAD_HDR)
927  return false;
928  if (!has(id))
929  return false;
930  if (result)
931  *result = getStrOrList(id);
932  return true;
933 }
934 
935 bool
936 HttpHeader::hasNamed(const char *name, unsigned int namelen, String *result) const
937 {
938  Http::HdrType id;
940  HttpHeaderEntry *e;
941 
942  assert(name);
943 
944  /* First try the quick path */
945  id = Http::HeaderLookupTable.lookup(name,namelen).id;
946 
947  if (id != Http::HdrType::BAD_HDR) {
948  if (getByIdIfPresent(id, result))
949  return true;
950  }
951 
952  /* Sorry, an unknown header name. Do linear search */
953  bool found = false;
954  while ((e = getEntry(&pos))) {
955  if (e->id == Http::HdrType::OTHER && e->name.length() == namelen && e->name.caseCmp(name, namelen) == 0) {
956  found = true;
957  if (!result)
958  break;
959  strListAdd(result, e->value.termedBuf(), ',');
960  }
961  }
962 
963  return found;
964 }
965 
966 /*
967  * Returns a the value of the specified list member, if any.
968  */
969 SBuf
970 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
971 {
972  assert(name);
973  const auto header = getByName(name);
974  return ::getListMember(header, member, separator);
975 }
976 
977 /*
978  * returns a the value of the specified list member, if any.
979  */
980 SBuf
981 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
982 {
984  const auto header = getStrOrList(id);
985  return ::getListMember(header, member, separator);
986 }
987 
988 /* test if a field is present */
989 int
991 {
993  debugs(55, 9, this << " lookup for " << id);
994  return CBIT_TEST(mask, id);
995 }
996 
997 void
999 {
1000  // TODO: do not add Via header for messages where Squid itself
1001  // generated the message (i.e., Downloader or ESI) there should be no Via header added at all.
1002 
1003  if (Config.onoff.via) {
1004  SBuf buf;
1005  // RFC 7230 section 5.7.1.: protocol-name is omitted when
1006  // the received protocol is HTTP.
1009  buf.appendf("%s/", AnyP::ProtocolType_str[ver.protocol]);
1010  buf.appendf("%d.%d %s", ver.major, ver.minor, ThisCache);
1011  const HttpHeader *hdr = from ? from : this;
1012  SBuf strVia = StringToSBuf(hdr->getList(Http::HdrType::VIA));
1013  if (!strVia.isEmpty())
1014  strVia.append(", ", 2);
1015  strVia.append(buf);
1016  // XXX: putStr() still suffers from String size limits
1017  Must(strVia.length() < String::SizeMaxXXX());
1019  putStr(Http::HdrType::VIA, strVia.c_str());
1020  }
1021 }
1022 
1023 void
1025 {
1027  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1028  assert(number >= 0);
1029  addEntry(new HttpHeaderEntry(id, SBuf(), xitoa(number)));
1030 }
1031 
1032 void
1034 {
1036  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1037  assert(number >= 0);
1039 }
1040 
1041 void
1043 {
1045  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1046  assert(htime >= 0);
1047  addEntry(new HttpHeaderEntry(id, SBuf(), mkrfc1123(htime)));
1048 }
1049 
1050 void
1051 HttpHeader::putStr(Http::HdrType id, const char *str)
1052 {
1054  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1055  assert(str);
1056  addEntry(new HttpHeaderEntry(id, SBuf(), str));
1057 }
1058 
1059 void
1060 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1061 {
1062  assert(auth_scheme && realm);
1063  httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1064 }
1065 
1066 void
1068 {
1069  assert(cc);
1070  /* remove old directives if any */
1072  /* pack into mb */
1073  MemBuf mb;
1074  mb.init();
1075  cc->packInto(&mb);
1076  /* put */
1078  /* cleanup */
1079  mb.clean();
1080 }
1081 
1082 void
1084 {
1085  assert(cr);
1086  /* remove old directives if any */
1088  /* pack into mb */
1089  MemBuf mb;
1090  mb.init();
1091  httpHdrContRangePackInto(cr, &mb);
1092  /* put */
1094  /* cleanup */
1095  mb.clean();
1096 }
1097 
1098 void
1100 {
1101  assert(range);
1102  /* remove old directives if any */
1104  /* pack into mb */
1105  MemBuf mb;
1106  mb.init();
1107  range->packInto(&mb);
1108  /* put */
1110  /* cleanup */
1111  mb.clean();
1112 }
1113 
1114 void
1116 {
1117  assert(sc);
1118  /* remove old directives if any */
1120  /* pack into mb */
1121  MemBuf mb;
1122  mb.init();
1123  sc->packInto(&mb);
1124  /* put */
1126  /* cleanup */
1127  mb.clean();
1128 }
1129 
1130 void
1131 HttpHeader::putWarning(const int code, const char *const text)
1132 {
1133  char buf[512];
1134  snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1136 }
1137 
1138 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1139 void
1140 HttpHeader::putExt(const char *name, const char *value)
1141 {
1142  assert(name && value);
1143  debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1144  addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, SBuf(name), value));
1145 }
1146 
1147 int
1149 {
1151  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1152  HttpHeaderEntry *e;
1153 
1154  if ((e = findEntry(id)))
1155  return e->getInt();
1156 
1157  return -1;
1158 }
1159 
1160 int64_t
1162 {
1164  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1165  HttpHeaderEntry *e;
1166 
1167  if ((e = findEntry(id)))
1168  return e->getInt64();
1169 
1170  return -1;
1171 }
1172 
1173 time_t
1175 {
1176  HttpHeaderEntry *e;
1177  time_t value = -1;
1179  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1180 
1181  if ((e = findEntry(id))) {
1182  value = parse_rfc1123(e->value.termedBuf());
1183  httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1184  }
1185 
1186  return value;
1187 }
1188 
1189 /* sync with httpHeaderGetLastStr */
1190 const char *
1192 {
1193  HttpHeaderEntry *e;
1195  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1196 
1197  if ((e = findEntry(id))) {
1198  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1199  return e->value.termedBuf();
1200  }
1201 
1202  return NULL;
1203 }
1204 
1205 /* unusual */
1206 const char *
1208 {
1209  HttpHeaderEntry *e;
1211  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1212 
1213  if ((e = findLastEntry(id))) {
1214  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1215  return e->value.termedBuf();
1216  }
1217 
1218  return NULL;
1219 }
1220 
1221 HttpHdrCc *
1223 {
1225  return NULL;
1226 
1227  String s;
1229 
1230  HttpHdrCc *cc=new HttpHdrCc();
1231 
1232  if (!cc->parse(s)) {
1233  delete cc;
1234  cc = NULL;
1235  }
1236 
1237  ++ HttpHeaderStats[owner].ccParsedCount;
1238 
1239  if (cc)
1240  httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1241 
1243 
1244  return cc;
1245 }
1246 
1247 HttpHdrRange *
1249 {
1250  HttpHdrRange *r = NULL;
1251  HttpHeaderEntry *e;
1252  /* some clients will send "Request-Range" _and_ *matching* "Range"
1253  * who knows, some clients might send Request-Range only;
1254  * this "if" should work correctly in both cases;
1255  * hopefully no clients send mismatched headers! */
1256 
1257  if ((e = findEntry(Http::HdrType::RANGE)) ||
1260  httpHeaderNoteParsedEntry(e->id, e->value, !r);
1261  }
1262 
1263  return r;
1264 }
1265 
1266 HttpHdrSc *
1268 {
1270  return NULL;
1271 
1272  String s;
1273 
1275 
1277 
1278  ++ HttpHeaderStats[owner].ccParsedCount;
1279 
1280  if (sc)
1281  sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1282 
1284 
1285  return sc;
1286 }
1287 
1290 {
1291  HttpHdrContRange *cr = NULL;
1292  HttpHeaderEntry *e;
1293 
1296  httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1297  }
1298 
1299  return cr;
1300 }
1301 
1302 SBuf
1303 HttpHeader::getAuthToken(Http::HdrType id, const char *auth_scheme) const
1304 {
1305  const char *field;
1306  int l;
1307  assert(auth_scheme);
1308  field = getStr(id);
1309 
1310  static const SBuf nil;
1311  if (!field) /* no authorization field */
1312  return nil;
1313 
1314  l = strlen(auth_scheme);
1315 
1316  if (!l || strncasecmp(field, auth_scheme, l)) /* wrong scheme */
1317  return nil;
1318 
1319  field += l;
1320 
1321  if (!xisspace(*field)) /* wrong scheme */
1322  return nil;
1323 
1324  /* skip white space */
1325  for (; field && xisspace(*field); ++field);
1326 
1327  if (!*field) /* no authorization cookie */
1328  return nil;
1329 
1330  const auto fieldLen = strlen(field);
1331  SBuf result;
1332  char *decodedAuthToken = result.rawAppendStart(BASE64_DECODE_LENGTH(fieldLen));
1333  struct base64_decode_ctx ctx;
1334  base64_decode_init(&ctx);
1335  size_t decodedLen = 0;
1336  if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), fieldLen, field) ||
1337  !base64_decode_final(&ctx)) {
1338  return nil;
1339  }
1340  result.rawAppendFinish(decodedAuthToken, decodedLen);
1341  return result;
1342 }
1343 
1344 ETag
1346 {
1347  ETag etag = {NULL, -1};
1348  HttpHeaderEntry *e;
1349  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag); /* must be of an appropriate type */
1350 
1351  if ((e = findEntry(id)))
1352  etagParseInit(&etag, e->value.termedBuf());
1353 
1354  return etag;
1355 }
1356 
1357 TimeOrTag
1359 {
1360  TimeOrTag tot;
1361  HttpHeaderEntry *e;
1362  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag); /* must be of an appropriate type */
1363  memset(&tot, 0, sizeof(tot));
1364 
1365  if ((e = findEntry(id))) {
1366  const char *str = e->value.termedBuf();
1367  /* try as an ETag */
1368 
1369  if (etagParseInit(&tot.tag, str)) {
1370  tot.valid = tot.tag.str != NULL;
1371  tot.time = -1;
1372  } else {
1373  /* or maybe it is time? */
1374  tot.time = parse_rfc1123(str);
1375  tot.valid = tot.time >= 0;
1376  tot.tag.str = NULL;
1377  }
1378  }
1379 
1380  assert(tot.time < 0 || !tot.tag.str); /* paranoid */
1381  return tot;
1382 }
1383 
1384 /*
1385  * HttpHeaderEntry
1386  */
1387 
1388 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const SBuf &aName, const char *aValue)
1389 {
1391  id = anId;
1392 
1393  if (id != Http::HdrType::OTHER)
1395  else
1396  name = aName;
1397 
1398  value = aValue;
1399 
1400  if (id != Http::HdrType::BAD_HDR)
1401  ++ headerStatsTable[id].aliveCount;
1402 
1403  debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1404 }
1405 
1407 {
1408  debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1409 
1410  if (id != Http::HdrType::BAD_HDR) {
1411  assert(headerStatsTable[id].aliveCount);
1412  -- headerStatsTable[id].aliveCount;
1413  id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1414  }
1415 
1416 }
1417 
1418 /* parses and inits header entry, returns true/false */
1420 HttpHeaderEntry::parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
1421 {
1422  /* note: name_start == field_start */
1423  const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1424  int name_len = name_end ? name_end - field_start :0;
1425  const char *value_start = field_start + name_len + 1; /* skip ':' */
1426  /* note: value_end == field_end */
1427 
1429 
1430  /* do we have a valid field name within this field? */
1431 
1432  if (!name_len || name_end > field_end)
1433  return NULL;
1434 
1435  if (name_len > 65534) {
1436  /* String must be LESS THAN 64K and it adds a terminating NULL */
1437  // TODO: update this to show proper name_len in Raw markup, but not print all that
1438  debugs(55, 2, "ignoring huge header field (" << Raw("field_start", field_start, 100) << "...)");
1439  return NULL;
1440  }
1441 
1442  /*
1443  * RFC 7230 section 3.2.4:
1444  * "No whitespace is allowed between the header field-name and colon.
1445  * ...
1446  * A server MUST reject any received request message that contains
1447  * whitespace between a header field-name and colon with a response code
1448  * of 400 (Bad Request). A proxy MUST remove any such whitespace from a
1449  * response message before forwarding the message downstream."
1450  */
1451  if (xisspace(field_start[name_len - 1])) {
1452 
1453  if (msgType == hoRequest)
1454  return nullptr;
1455 
1456  // for now, also let relaxed parser remove this BWS from any non-HTTP messages
1457  const bool stripWhitespace = (msgType == hoReply) ||
1459  if (!stripWhitespace)
1460  return nullptr; // reject if we cannot strip
1461 
1462  debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1463  "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1464 
1465  while (name_len > 0 && xisspace(field_start[name_len - 1]))
1466  --name_len;
1467 
1468  if (!name_len) {
1469  debugs(55, 2, "found header with only whitespace for name");
1470  return NULL;
1471  }
1472  }
1473 
1474  /* RFC 7230 section 3.2:
1475  *
1476  * header-field = field-name ":" OWS field-value OWS
1477  * field-name = token
1478  * token = 1*TCHAR
1479  */
1480  for (const char *pos = field_start; pos < (field_start+name_len); ++pos) {
1481  if (!CharacterSet::TCHAR[*pos]) {
1482  debugs(55, 2, "found header with invalid characters in " <<
1483  Raw("field-name", field_start, min(name_len,100)) << "...");
1484  return nullptr;
1485  }
1486  }
1487 
1488  /* now we know we can parse it */
1489 
1490  debugs(55, 9, "parsing HttpHeaderEntry: near '" << getStringPrefix(field_start, field_end-field_start) << "'");
1491 
1492  /* is it a "known" field? */
1493  Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1494  debugs(55, 9, "got hdr-id=" << id);
1495 
1496  SBuf theName;
1497 
1498  String value;
1499 
1500  if (id == Http::HdrType::BAD_HDR)
1501  id = Http::HdrType::OTHER;
1502 
1503  /* set field name */
1504  if (id == Http::HdrType::OTHER)
1505  theName.append(field_start, name_len);
1506  else
1507  theName = Http::HeaderLookupTable.lookup(id).name;
1508 
1509  /* trim field value */
1510  while (value_start < field_end && xisspace(*value_start))
1511  ++value_start;
1512 
1513  while (value_start < field_end && xisspace(field_end[-1]))
1514  --field_end;
1515 
1516  if (field_end - value_start > 65534) {
1517  /* String must be LESS THAN 64K and it adds a terminating NULL */
1518  debugs(55, 2, "WARNING: found '" << theName << "' header of " << (field_end - value_start) << " bytes");
1519  return NULL;
1520  }
1521 
1522  /* set field value */
1523  value.assign(value_start, field_end - value_start);
1524 
1525  if (id != Http::HdrType::BAD_HDR)
1526  ++ headerStatsTable[id].seenCount;
1527 
1528  debugs(55, 9, "parsed HttpHeaderEntry: '" << theName << ": " << value << "'");
1529 
1530  return new HttpHeaderEntry(id, theName, value.termedBuf());
1531 }
1532 
1535 {
1536  return new HttpHeaderEntry(id, name, value.termedBuf());
1537 }
1538 
1539 void
1541 {
1542  assert(p);
1543  p->append(name.rawContent(), name.length());
1544  p->append(": ", 2);
1545  p->append(value.rawBuf(), value.size());
1546  p->append("\r\n", 2);
1547 }
1548 
1549 int
1551 {
1552  int val = -1;
1553  int ok = httpHeaderParseInt(value.termedBuf(), &val);
1554  httpHeaderNoteParsedEntry(id, value, ok == 0);
1555  /* XXX: Should we check ok - ie
1556  * return ok ? -1 : value;
1557  */
1558  return val;
1559 }
1560 
1561 int64_t
1563 {
1564  int64_t val = -1;
1565  const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1566  httpHeaderNoteParsedEntry(id, value, !ok);
1567  return val; // remains -1 if !ok (XXX: bad method API)
1568 }
1569 
1570 static void
1572 {
1573  if (id != Http::HdrType::BAD_HDR)
1574  ++ headerStatsTable[id].parsCount;
1575 
1576  if (error) {
1577  if (id != Http::HdrType::BAD_HDR)
1578  ++ headerStatsTable[id].errCount;
1579  debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1580  }
1581 }
1582 
1583 /*
1584  * Reports
1585  */
1586 
1587 /* tmp variable used to pass stat info to dumpers */
1588 extern const HttpHeaderStat *dump_stat; /* argh! */
1590 
1591 static void
1592 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1593 {
1594  const int id = static_cast<int>(val);
1595  const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1596  const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1597  int visible = count > 0;
1598  /* for entries with zero count, list only those that belong to current type of message */
1599 
1600  if (!visible && valid_id && dump_stat->owner_mask)
1601  visible = CBIT_TEST(*dump_stat->owner_mask, id);
1602 
1603  if (visible)
1604  storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1605  id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1606 }
1607 
1608 static void
1609 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1610 {
1611  if (count)
1612  storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1613  idx, (int) val, count,
1614  xpercent(count, dump_stat->destroyedCount));
1615 }
1616 
1617 static void
1619 {
1620  assert(hs);
1621  assert(e);
1622 
1623  if (!hs->owner_mask)
1624  return; // these HttpHeaderStat objects were not meant to be dumped here
1625 
1626  dump_stat = hs;
1627  storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1628  storeAppendPrintf(e, "\nField type distribution\n");
1629  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1630  "id", "name", "count", "#/header");
1632  storeAppendPrintf(e, "\nCache-control directives distribution\n");
1633  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1634  "id", "name", "count", "#/cc_field");
1636  storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1637  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1638  "id", "name", "count", "#/sc_field");
1640  storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1641  storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1642  "id", "#flds", "count", "%total");
1644  storeAppendPrintf(e, "\n");
1645  dump_stat = NULL;
1646 }
1647 
1648 void
1650 {
1651  assert(e);
1652 
1653  HttpHeaderStats[0].parsedCount =
1654  HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1655  HttpHeaderStats[0].ccParsedCount =
1656  HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1657  HttpHeaderStats[0].destroyedCount =
1658  HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1659  HttpHeaderStats[0].busyDestroyedCount =
1660  HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1661 
1662  for (const auto &stats: HttpHeaderStats)
1664 
1665  /* field stats for all messages */
1666  storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1667 
1668  storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1669  "id", "name", "#alive", "%err", "%repeat");
1670 
1671  // scan heaaderTable and output
1672  for (auto h : WholeEnum<Http::HdrType>()) {
1673  auto stats = headerStatsTable[h];
1674  storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1675  Http::HeaderLookupTable.lookup(h).id,
1676  Http::HeaderLookupTable.lookup(h).name,
1677  stats.aliveCount,
1678  xpercent(stats.errCount, stats.parsCount),
1679  xpercent(stats.repCount, stats.seenCount));
1680  }
1681 
1682  storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1683  HttpHeaderStats[hoRequest].parsedCount,
1684  HttpHeaderStats[hoReply].parsedCount,
1685  HttpHeaderStats[0].parsedCount);
1686  storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1687 }
1688 
1689 int
1690 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1691 {
1692  int result = 0;
1693  const char *pos = NULL;
1694  const char *item;
1695  int ilen;
1696  int mlen = strlen(member);
1697 
1699 
1700  String header (getStrOrList(id));
1701 
1702  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1703  if (strncasecmp(item, member, mlen) == 0
1704  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1705  result = 1;
1706  break;
1707  }
1708  }
1709 
1710  return result;
1711 }
1712 
1713 int
1714 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1715 {
1716  int result = 0;
1717  const char *pos = NULL;
1718  const char *item;
1719  int ilen;
1720  int mlen = strlen(member);
1721 
1722  assert(name);
1723 
1724  String header (getByName(name));
1725 
1726  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1727  if (strncasecmp(item, member, mlen) == 0
1728  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1729  result = 1;
1730  break;
1731  }
1732  }
1733 
1734  return result;
1735 }
1736 
1737 void
1739 {
1741 
1742  const HttpHeaderEntry *e;
1744  int headers_deleted = 0;
1745  while ((e = getEntry(&pos))) {
1746  Http::HdrType id = e->id;
1747  if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1748  delAt(pos, headers_deleted);
1749  CBIT_CLR(mask, id);
1750  }
1751  }
1752 }
1753 
1754 void
1756 {
1758  /* anything that matches Connection list member will be deleted */
1759  String strConnection;
1760 
1761  (void) getList(Http::HdrType::CONNECTION, &strConnection);
1762  const HttpHeaderEntry *e;
1764  /*
1765  * think: on-average-best nesting of the two loops (hdrEntry
1766  * and strListItem) @?@
1767  */
1768  /*
1769  * maybe we should delete standard stuff ("keep-alive","close")
1770  * from strConnection first?
1771  */
1772 
1773  int headers_deleted = 0;
1774  while ((e = getEntry(&pos))) {
1775  if (strListIsMember(&strConnection, e->name, ','))
1776  delAt(pos, headers_deleted);
1777  }
1778  if (headers_deleted)
1779  refreshMask();
1780  }
1781 }
1782 
int caseCmp(char const *) const
Definition: String.cc:287
void append(const HttpHeader *src)
Definition: HttpHeader.cc:235
SQUIDCEXTERN double xpercent(double part, double whole)
Definition: util.c:54
bool any_registered_header(const Http::HdrType id)
void refreshMask()
Definition: HttpHeader.cc:750
char * buf
Definition: MemBuf.h:134
@ SURROGATE_CONTROL
class Ping::pingStats_ stats
@ hoReply
Definition: HttpHeader.h:37
unsigned int major
major version number
static HttpHeaderEntry * parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
Definition: HttpHeader.cc:1420
HttpHdrRange * getRange() const
Definition: HttpHeader.cc:1248
int delByName(const SBuf &name)
Definition: HttpHeader.cc:675
int relaxed_header_parser
Definition: SquidConfig.h:322
Definition: ETag.h:18
http_hdr_owner_type owner
Definition: HttpHeader.h:173
SBuf getListMember(const String &list, const char *key, const char delimiter)
Definition: StrList.cc:144
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1191
void httpHdrScStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrSc.cc:272
void removeHopByHopEntries()
Definition: HttpHeader.cc:1738
bool teUnsupported_
Definition: HttpHeader.h:194
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1690
void packInto(Packable *p) const
Definition: HttpHeader.cc:1540
bool isEmpty() const
Definition: SBuf.h:424
HttpHeader & operator=(const HttpHeader &other)
Definition: HttpHeader.cc:174
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
HttpHeaderMask mask
Definition: HttpHeader.h:172
#define CBIT_CLR(mask, bit)
Definition: defines.h:75
HTTP per header statistics.
String getById(Http::HdrType id) const
Definition: HttpHeader.cc:910
HttpHeaderEntry * findLastEntry(Http::HdrType id) const
Definition: HttpHeader.cc:655
@ PROTO_NONE
Definition: ProtocolType.h:24
void httpHdrCcInitModule(void)
Module initialization hook.
Definition: HttpHdrCc.cc:61
void base64_decode_init(struct base64_decode_ctx *ctx)
Definition: base64.c:54
int etagParseInit(ETag *etag, const char *str)
Definition: ETag.cc:29
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:869
void error(char *format,...)
unsigned int minor
minor version number
bool conflictingContentLength_
Definition: HttpHeader.h:191
int parse(const char *header_start, size_t len, Http::ContentLengthInterpreter &interpreter)
Definition: HttpHeader.cc:377
const HttpHeaderStat * dump_stat
Definition: HttpHeader.cc:1589
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:95
const char * ProtocolType_str[]
Definition: SBuf.h:87
StatHist hdrUCountDistr
virtual void append(const char *buf, int size)=0
Appends a c-string to existing packed data.
const char * str
quoted-string
Definition: ETag.h:20
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:841
void rawAppendFinish(const char *start, size_type actualSize)
Definition: SBuf.cc:144
bool needUpdate(const HttpHeader *fresh) const
Definition: HttpHeader.cc:248
int type
Definition: errorpage.cc:153
static HttpHeaderMask ReplyHeadersMask
Definition: HttpHeader.cc:73
void httpHdrCcUpdateStats(const HttpHdrCc *cc, StatHist *hist)
Definition: HttpHdrCc.cc:321
static bool Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
Definition: HttpHeader.cc:329
#define DBG_CRITICAL
Definition: Debug.h:40
String getStrOrList(Http::HdrType id) const
Definition: HttpHeader.cc:875
time_t time
Definition: TimeOrTag.h:21
int hasByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:1714
char ThisCache[RFC2181_MAXHOSTNAMELEN<< 1]
#define DBG_IMPORTANT
Definition: Debug.h:41
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
ProtocolType protocol
which protocol this version is for
@ PROTO_UNKNOWN
Definition: ProtocolType.h:43
@ CONTENT_LENGTH
number
Definition: testStatHist.cc:16
StatHist fieldTypeDistr
char * rawAppendStart(size_type anticipatedSize)
Definition: SBuf.cc:136
static void httpHeaderFldsPerHdrDumper(StoreEntry *sentry, int idx, double val, double, int count)
Definition: HttpHeader.cc:1609
bool sawBad
whether a malformed Content-Length value was present
bool skipUpdateHeader(const Http::HdrType id) const
Definition: HttpHeader.cc:282
bool any_HdrType_enum_value(const Http::HdrType id)
match any known header type, including OTHER and BAD
static int HeaderEntryParsedCount
Definition: HttpHeader.cc:90
@ WWW_AUTHENTICATE
static std::array< HttpHeaderStat, hoEnd > HttpHeaderStats
Definition: HttpHeader.cc:77
@ hoNone
Definition: HttpHeader.h:32
Definition: Debug.h:184
TimeOrTag getTimeOrTag(Http::HdrType id) const
Definition: HttpHeader.cc:1358
void strListAdd(String &str, const char *item, const size_t itemSize, const char delimiter)
Appends the given item of a given size to a delimiter-separated list in str.
Definition: StrList.cc:18
static size_type SizeMaxXXX()
Definition: SquidString.h:71
void putContRange(const HttpHdrContRange *cr)
Definition: HttpHeader.cc:1083
bool hasNamed(const SBuf &s, String *value=0) const
Definition: HttpHeader.cc:918
int base64_decode_final(struct base64_decode_ctx *ctx)
Definition: base64.c:159
int64_t getInt64() const
Definition: HttpHeader.cc:1562
#define NULL
Definition: types.h:166
const char * rawContent() const
Definition: SBuf.cc:509
StatHist scTypeDistr
SQUIDCEXTERN const char * xint64toa(int64_t num)
Definition: util.c:88
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
time_t parse_rfc1123(const char *str)
Definition: rfc1123.c:159
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
bool parse(const String &s)
parse a header-string and fill in appropriate values.
Definition: HttpHdrCc.cc:95
const char * getLastStr(Http::HdrType id) const
Definition: HttpHeader.cc:1207
char const * visible_appname_string
void dump(StoreEntry *sentry, StatHistBinDumper *bd) const
Definition: StatHist.cc:173
std::vector< HttpHeaderFieldStat > headerStatsTable(Http::HdrType::enumEnd_)
const char * text
Definition: errorpage.cc:154
char HttpHeaderMask[12]
void putAuth(const char *auth_scheme, const char *realm)
Definition: HttpHeader.cc:1060
std::vector< HttpHeaderEntry *, PoolingAllocator< HttpHeaderEntry * > > entries
Definition: HttpHeader.h:171
void putWarning(const int code, const char *const text)
add a Warning header
Definition: HttpHeader.cc:1131
Definition: MemBuf.h:24
SBuf StringToSBuf(const String &s)
create a new SBuf from a String by copying contents
Definition: StringConvert.h:17
static const CharacterSet TCHAR
Definition: CharacterSet.h:105
HttpHdrSc * getSc() const
Definition: HttpHeader.cc:1267
void insertEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:790
void clean()
Definition: MemBuf.cc:112
unsigned char code
Definition: html_quote.c:20
bool httpHeaderParseOffset(const char *start, int64_t *value, char **endPtr)
int delById(Http::HdrType id)
Definition: HttpHeader.cc:694
Http::HdrType id
Definition: HttpHeader.h:63
HttpHdrSc * httpHdrScParseCreate(const String &str)
Definition: HttpHdrSc.cc:60
void httpHeaderInitModule(void)
Definition: HttpHeader.cc:118
void putExt(const char *name, const char *value)
Definition: HttpHeader.cc:1140
int64_t getInt64(Http::HdrType id) const
Definition: HttpHeader.cc:1161
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:764
time_t getTime(Http::HdrType id) const
Definition: HttpHeader.cc:1174
SBuf getAuthToken(Http::HdrType id, const char *auth_scheme) const
Definition: HttpHeader.cc:1303
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1042
#define assert(EX)
Definition: assert.h:19
int base64_decode_update(struct base64_decode_ctx *ctx, size_t *dst_length, uint8_t *dst, size_t src_length, const char *src)
Definition: base64.c:129
char const * termedBuf() const
Definition: SquidString.h:92
void httpHdrScInitModule(void)
Definition: HttpHdrSc.cc:49
#define CBIT_TEST(mask, bit)
Definition: defines.h:76
bool any_valid_header(const Http::HdrType id)
match any valid header type, including OTHER but not BAD
int getInt(Http::HdrType id) const
Definition: HttpHeader.cc:1148
@ hoEnd
Definition: HttpHeader.h:41
@ TRANSFER_ENCODING
int valid
Definition: TimeOrTag.h:22
const HeaderTableRecord & lookup(const char *buf, const std::size_t len) const
look record type up by name (C-string and length)
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:567
HttpHeaderMask * owner_mask
ETag tag
Definition: TimeOrTag.h:20
static int sc[16]
Definition: smbdes.c:121
const char * c_str()
Definition: SBuf.cc:516
HttpHdrContRange * httpHdrContRangeParseCreate(const char *str)
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:408
static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error)
Definition: HttpHeader.cc:1571
SBuf & append(const SBuf &S)
Definition: SBuf.cc:185
const char * mkrfc1123(time_t)
Definition: rfc1123.c:202
SQUIDCEXTERN const char * xitoa(int num)
Definition: util.c:79
void putCc(const HttpHdrCc *cc)
Definition: HttpHeader.cc:1067
void removeConnectionHeaderEntries()
Definition: HttpHeader.cc:1755
ETag getETag(Http::HdrType id) const
Definition: HttpHeader.cc:1345
const HeaderLookupTable_t HeaderLookupTable
const char * getStringPrefix(const char *str, size_t sz)
String getByName(const SBuf &name) const
Definition: HttpHeader.cc:901
@ PROTO_HTTPS
Definition: ProtocolType.h:27
void putInt(Http::HdrType id, int number)
Definition: HttpHeader.cc:1024
HttpHeaderEntry * findEntry(Http::HdrType id) const
Definition: HttpHeader.cc:630
static HttpHdrRange * ParseCreate(const String *range_spec)
@ PROTO_HTTP
Definition: ProtocolType.h:25
static void httpHeaderRegisterWithCacheManager(void)
Definition: HttpHeader.cc:110
void httpHdrCcStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrCc.cc:331
struct SquidConfig::@111 onoff
SBuf getByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:970
int has(Http::HdrType id) const
Definition: HttpHeader.cc:990
bool getByIdIfPresent(Http::HdrType id, String *result) const
Definition: HttpHeader.cc:924
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1051
size_type size() const
Definition: SquidString.h:73
static void httpHeaderFieldStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHeader.cc:1592
int getInt() const
Definition: HttpHeader.cc:1550
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:73
void assign(const char *str, int len)
Definition: String.cc:91
int httpHeaderParseInt(const char *start, int *value)
void updateWarnings()
Definition: HttpHeader.cc:269
void Controller::create() STUB void Controller Controller nil
HttpHeaderEntry(Http::HdrType id, const SBuf &name, const char *value)
Definition: HttpHeader.cc:1388
const char * label
SQUIDCEXTERN double xdiv(double nom, double denom)
Definition: util.c:72
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:229
static void httpHeaderStoreReport(StoreEntry *e)
Definition: HttpHeader.cc:1649
SBuf getListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:981
static HttpHeaderMask RequestHeadersMask
Definition: HttpHeader.cc:70
void compact()
Definition: HttpHeader.cc:739
void update(const HttpHeader *fresh)
Definition: HttpHeader.cc:293
void putSc(HttpHdrSc *sc)
Definition: HttpHeader.cc:1115
#define CBIT_SET(mask, bit)
Definition: defines.h:74
int caseCmp(const SBuf &S, const size_type n) const
shorthand version for case-insensitive compare()
Definition: SBuf.h:276
void delAt(HttpHeaderPos pos, int &headers_deleted)
Definition: HttpHeader.cc:722
const char * headerWideProblem
worst header-wide problem found (or nil)
#define xisspace(x)
Definition: xis.h:17
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:611
StatHist ccTypeDistr
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
void httpHdrContRangePackInto(const HttpHdrContRange *range, Packable *p)
void httpHeaderMaskInit(HttpHeaderMask *mask, int value)
static StatHist s
#define false
Definition: GnuRegex.c:233
void packInto(Packable *p) const
Definition: HttpHdrCc.cc:247
HttpHdrContRange * getContRange() const
Definition: HttpHeader.cc:1289
A const & min(A const &lhs, A const &rhs)
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=0)
Definition: HttpHeader.cc:998
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:17
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
@ hoRequest
Definition: HttpHeader.h:36
char const * rawBuf() const
Definition: SquidString.h:86
void putRange(const HttpHdrRange *range)
Definition: HttpHeader.cc:1099
void clean()
Definition: HttpHeader.cc:189
static void httpHeaderStatDump(const HttpHeaderStat *hs, StoreEntry *e)
Definition: HttpHeader.cc:1618
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1222
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1033
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1534
#define BASE64_DECODE_LENGTH(length)
Definition: base64.h:120
class SquidConfig Config
Definition: SquidConfig.cc:12
http_hdr_owner_type
Definition: HttpHeader.h:31
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:86
@ PROXY_AUTHORIZATION
void packInto(Packable *p) const

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors