HttpHeader.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 55 HTTP Header */
10 
11 #include "squid.h"
12 #include "base/CharacterSet.h"
13 #include "base/EnumIterator.h"
14 #include "base64.h"
15 #include "globals.h"
17 #include "HttpHdrCc.h"
18 #include "HttpHdrContRange.h"
19 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
20 #include "HttpHeader.h"
21 #include "HttpHeaderFieldInfo.h"
22 #include "HttpHeaderStat.h"
23 #include "HttpHeaderTools.h"
24 #include "MemBuf.h"
25 #include "mgr/Registration.h"
26 #include "mime_header.h"
27 #include "profiler/Profiler.h"
28 #include "rfc1123.h"
29 #include "sbuf/StringConvert.h"
30 #include "SquidConfig.h"
31 #include "StatHist.h"
32 #include "Store.h"
33 #include "StrList.h"
34 #include "TimeOrTag.h"
35 #include "util.h"
36 
37 #include <algorithm>
38 
39 /* XXX: the whole set of API managing the entries vector should be rethought
40  * after the parse4r-ng effort is complete.
41  */
42 
43 /*
44  * On naming conventions:
45  *
46  * HTTP/1.1 defines message-header as
47  *
48  * message-header = field-name ":" [ field-value ] CRLF
49  * field-name = token
50  * field-value = *( field-content | LWS )
51  *
52  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
53  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
54  *
55  * HttpHeader is an object that represents all message-headers in a message.
56  * HttpHeader does not manage start-line.
57  *
58  * HttpHeader is implemented as a collection of header "entries".
59  * An entry is a (field_id, field_name, field_value) triplet.
60  */
61 
62 /*
63  * local constants and vars
64  */
65 
66 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
67 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
68 
69 /* request-only headers. Used for cachemgr */
70 static HttpHeaderMask RequestHeadersMask; /* set run-time using RequestHeaders */
71 
72 /* reply-only headers. Used for cachemgr */
73 static HttpHeaderMask ReplyHeadersMask; /* set run-time using ReplyHeaders */
74 
75 /* header accounting */
76 // NP: keep in sync with enum http_hdr_owner_type
77 static HttpHeaderStat HttpHeaderStats[] = {
78  HttpHeaderStat(/*hoNone*/ "all", NULL),
79 #if USE_HTCP
80  HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
81 #endif
82  HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
83  HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
84 #if USE_OPENSSL
85  /* hoErrorDetail */
86 #endif
87  /* hoEnd */
88 };
89 static int HttpHeaderStatCount = countof(HttpHeaderStats);
90 
91 static int HeaderEntryParsedCount = 0;
92 
93 /*
94  * forward declarations and local routines
95  */
96 
97 class StoreEntry;
98 
99 // update parse statistics for header id; if error is true also account
100 // for errors and write to debug log what happened
101 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
102 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
104 static void httpHeaderStoreReport(StoreEntry * e);
105 
106 /*
107  * Module initialization routines
108  */
109 
110 static void
112 {
113  Mgr::RegisterAction("http_headers",
114  "HTTP Header Statistics",
115  httpHeaderStoreReport, 0, 1);
116 }
117 
118 void
120 {
121  /* check that we have enough space for masks */
123 
124  // masks are needed for stats page still
125  for (auto h : WholeEnum<Http::HdrType>()) {
126  if (Http::HeaderLookupTable.lookup(h).request)
128  if (Http::HeaderLookupTable.lookup(h).reply)
130  }
131 
132  /* header stats initialized by class constructor */
134 
135  /* init dependent modules */
138 
140 }
141 
142 /*
143  * HttpHeader Implementation
144  */
145 
146 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
147 {
148  entries.reserve(32);
150 }
151 
153 {
154  assert(anOwner > hoNone && anOwner < hoEnd);
155  debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
156  entries.reserve(32);
158 }
159 
160 // XXX: Delete as unused, expensive, and violating copy semantics by skipping Warnings
162 {
163  entries.reserve(other.entries.capacity());
165  update(&other); // will update the mask as well
166 }
167 
169 {
170  clean();
171 }
172 
173 // XXX: Delete as unused, expensive, and violating assignment semantics by skipping Warnings
174 HttpHeader &
176 {
177  if (this != &other) {
178  // we do not really care, but the caller probably does
179  assert(owner == other.owner);
180  clean();
181  update(&other); // will update the mask as well
182  len = other.len;
184  }
185  return *this;
186 }
187 
188 void
190 {
191 
192  assert(owner > hoNone && owner < hoEnd);
193  debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
194 
195  PROF_start(HttpHeaderClean);
196 
197  if (owner <= hoReply) {
198  /*
199  * An unfortunate bug. The entries array is initialized
200  * such that count is set to zero. httpHeaderClean() seems to
201  * be called both when 'hdr' is created, and destroyed. Thus,
202  * we accumulate a large number of zero counts for 'hdr' before
203  * it is ever used. Can't think of a good way to fix it, except
204  * adding a state variable that indicates whether or not 'hdr'
205  * has been used. As a hack, just never count zero-sized header
206  * arrays.
207  */
208  if (!entries.empty())
209  HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
210 
211  ++ HttpHeaderStats[owner].destroyedCount;
212 
213  HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
214  } // if (owner <= hoReply)
215 
216  for (HttpHeaderEntry *e : entries) {
217  if (e == nullptr)
218  continue;
219  if (!Http::any_valid_header(e->id)) {
220  debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
221  } else {
222  if (owner <= hoReply)
223  HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
224  delete e;
225  }
226  }
227 
228  entries.clear();
230  len = 0;
232  PROF_stop(HttpHeaderClean);
233 }
234 
235 /* append entries (also see httpHeaderUpdate) */
236 void
238 {
239  assert(src);
240  assert(src != this);
241  debugs(55, 7, "appending hdr: " << this << " += " << src);
242 
243  for (auto e : src->entries) {
244  if (e)
245  addEntry(e->clone());
246  }
247 }
248 
249 bool
251 {
252  // our 1xx Warnings must be removed
253  for (const auto e: entries) {
254  // TODO: Move into HttpHeaderEntry::is1xxWarning() before official commit.
255  if (e && e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1))
256  return true;
257  }
258 
259  for (const auto e: fresh->entries) {
260  if (!e || skipUpdateHeader(e->id))
261  continue;
262  String value;
263  if (!hasNamed(e->name, &value) ||
264  (value != fresh->getByName(e->name)))
265  return true;
266  }
267  return false;
268 }
269 
270 void
272 {
273  int count = 0;
275 
276  // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
277  while (HttpHeaderEntry *e = getEntry(&pos)) {
278  if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
279  delAt(pos, count);
280  }
281 }
282 
283 bool
285 {
286  return
287  // RFC 7234, section 4.3.4: use header fields other than Warning
288  (id == Http::HdrType::WARNING) ||
289  // TODO: Consider updating Vary headers after comparing the magnitude of
290  // the required changes (and/or cache losses) with compliance gains.
291  (id == Http::HdrType::VARY);
292 }
293 
294 void
296 {
297  assert(fresh);
298  assert(this != fresh);
299 
300  updateWarnings();
301 
302  const HttpHeaderEntry *e;
304 
305  while ((e = fresh->getEntry(&pos))) {
306  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
307 
308  if (skipUpdateHeader(e->id))
309  continue;
310 
311  if (e->id != Http::HdrType::OTHER)
312  delById(e->id);
313  else
314  delByName(e->name);
315  }
316 
317  pos = HttpHeaderInitPos;
318  while ((e = fresh->getEntry(&pos))) {
319  /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
320 
321  if (skipUpdateHeader(e->id))
322  continue;
323 
324  debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
325 
326  addEntry(e->clone());
327  }
328 }
329 
330 bool
331 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
332 {
333  /*
334  * parse_start points to the first line of HTTP message *headers*,
335  * not including the request or status lines
336  */
337  const size_t end = headersEnd(*parse_start, l);
338 
339  if (end) {
340  *blk_start = *parse_start;
341  *blk_end = *parse_start + end - 1;
342  assert(**blk_end == '\n');
343  // Point blk_end to the first character after the last header field.
344  // In other words, blk_end should point to the CR?LF header terminator.
345  if (end > 1 && *(*blk_end - 1) == '\r')
346  --(*blk_end);
347  *parse_start += end;
348  }
349  return end;
350 }
351 
352 int
353 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz, Http::ContentLengthInterpreter &clen)
354 {
355  const char *parse_start = buf;
356  const char *blk_start, *blk_end;
357  hdr_sz = 0;
358 
359  if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
360  // XXX: do not parse non-isolated headers even if the connection is closed.
361  // Treat unterminated headers as "partial headers" framing errors.
362  if (!atEnd)
363  return 0;
364  blk_start = parse_start;
365  blk_end = blk_start + strlen(blk_start);
366  }
367 
368  if (parse(blk_start, blk_end - blk_start, clen)) {
369  hdr_sz = parse_start - buf;
370  return 1;
371  }
372  return -1;
373 }
374 
375 int
376 HttpHeader::parse(const char *header_start, size_t hdrLen, Http::ContentLengthInterpreter &clen)
377 {
378  const char *field_ptr = header_start;
379  const char *header_end = header_start + hdrLen; // XXX: remove
380  int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
381 
382  PROF_start(HttpHeaderParse);
383 
384  assert(header_start && header_end);
385  debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
386  ++ HttpHeaderStats[owner].parsedCount;
387 
388  char *nulpos;
389  if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
390  debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
391  getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
392  PROF_stop(HttpHeaderParse);
393  clean();
394  return 0;
395  }
396 
397  /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
398  * continuation lines start with a (single) space or tab */
399  while (field_ptr < header_end) {
400  const char *field_start = field_ptr;
401  const char *field_end;
402 
403  do {
404  const char *this_line = field_ptr;
405  field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
406 
407  if (!field_ptr) {
408  // missing <LF>
409  PROF_stop(HttpHeaderParse);
410  clean();
411  return 0;
412  }
413 
414  field_end = field_ptr;
415 
416  ++field_ptr; /* Move to next line */
417 
418  if (field_end > this_line && field_end[-1] == '\r') {
419  --field_end; /* Ignore CR LF */
420 
421  if (owner == hoRequest && field_end > this_line) {
422  bool cr_only = true;
423  for (const char *p = this_line; p < field_end && cr_only; ++p) {
424  if (*p != '\r')
425  cr_only = false;
426  }
427  if (cr_only) {
428  debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
429  "header field to prevent request smuggling attacks: {" <<
430  getStringPrefix(header_start, hdrLen) << "}");
431  PROF_stop(HttpHeaderParse);
432  clean();
433  return 0;
434  }
435  }
436  }
437 
438  /* Barf on stray CR characters */
439  if (memchr(this_line, '\r', field_end - this_line)) {
440  debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
441  getStringPrefix(field_start, field_end-field_start) << "}");
442 
444  char *p = (char *) this_line; /* XXX Warning! This destroys original header content and violates specifications somewhat */
445 
446  while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
447  *p = ' ';
448  ++p;
449  }
450  } else {
451  PROF_stop(HttpHeaderParse);
452  clean();
453  return 0;
454  }
455  }
456 
457  if (this_line + 1 == field_end && this_line > field_start) {
458  debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
459  getStringPrefix(header_start, hdrLen) << "}");
460  PROF_stop(HttpHeaderParse);
461  clean();
462  return 0;
463  }
464  } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
465 
466  if (field_start == field_end) {
467  if (field_ptr < header_end) {
468  debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
469  getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
470  PROF_stop(HttpHeaderParse);
471  clean();
472  return 0;
473  }
474 
475  break; /* terminating blank line */
476  }
477 
478  const auto e = HttpHeaderEntry::parse(field_start, field_end, owner);
479  if (!e) {
480  debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
481  getStringPrefix(field_start, field_end-field_start) << "}");
482  debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
483 
484  PROF_stop(HttpHeaderParse);
485  clean();
486  return 0;
487  }
488 
489  if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
490  delete e;
491 
493  continue; // clen has printed any necessary warnings
494 
495  PROF_stop(HttpHeaderParse);
496  clean();
497  return 0;
498  }
499 
500  /* AYJ 2017-05-23: I suspect we need to change this whitespace check to conform to the
501  updated WSP character set in RFC 7230/7231. For now I left it as the
502  characters in w_space which the previous code was using. */
504  if (e->id == Http::HdrType::OTHER && e->name.findFirstOf(wsp) != SBuf::npos) {
505  debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
506  getStringPrefix(field_start, field_end-field_start) << "}");
507 
509  delete e;
510  PROF_stop(HttpHeaderParse);
511  clean();
512  return 0;
513  }
514  }
515 
516  addEntry(e);
517  }
518 
519  if (clen.headerWideProblem) {
520  debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
521  " Content-Length field values in" <<
522  Raw("header", header_start, hdrLen));
523  }
524 
525  if (clen.prohibitedAndIgnored()) {
526  // RFC 7230 section 3.3.2: A server MUST NOT send a Content-Length
527  // header field in any response with a status code of 1xx (Informational)
528  // or 204 (No Content). And RFC 7230 3.3.3#1 tells recipients to ignore
529  // such Content-Lengths.
531  debugs(55, 3, "Content-Length is " << clen.prohibitedAndIgnored());
532  } else if (chunked()) {
533  // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
534  // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
536  // and clen state becomes irrelevant
537  } else if (clen.sawBad) {
538  // ensure our callers do not accidentally see bad Content-Length values
540  conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
541  } else if (clen.needsSanitizing) {
542  // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
543  // ensure our callers see a clean Content-Length value or none at all
545  if (clen.sawGood) {
547  debugs(55, 5, "sanitized Content-Length to be " << clen.value);
548  }
549  }
550 
551  PROF_stop(HttpHeaderParse);
552  return 1; /* even if no fields where found, it is a valid header */
553 }
554 
555 /* packs all the entries using supplied packer */
556 void
557 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
558 {
560  const HttpHeaderEntry *e;
561  assert(p);
562  debugs(55, 7, this << " into " << p <<
563  (mask_sensitive_info ? " while masking" : ""));
564  /* pack all entries one by one */
565  while ((e = getEntry(&pos))) {
566  if (!mask_sensitive_info) {
567  e->packInto(p);
568  continue;
569  }
570 
571  bool maskThisEntry = false;
572  switch (e->id) {
575  maskThisEntry = true;
576  break;
577 
580  maskThisEntry = (cmd->value == "PASS");
581  break;
582 
583  default:
584  break;
585  }
586  if (maskThisEntry) {
587  p->append(e->name.rawContent(), e->name.length());
588  p->append(": ** NOT DISPLAYED **\r\n", 23);
589  } else {
590  e->packInto(p);
591  }
592 
593  }
594  /* Pack in the "special" entries */
595 
596  /* Cache-Control */
597 }
598 
599 /* returns next valid entry */
602 {
603  assert(pos);
604  assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
605 
606  for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
607  if (entries[*pos])
608  return static_cast<HttpHeaderEntry*>(entries[*pos]);
609  }
610 
611  return NULL;
612 }
613 
614 /*
615  * returns a pointer to a specified entry if any
616  * note that we return one entry so it does not make much sense to ask for
617  * "list" headers
618  */
621 {
623  assert(!Http::HeaderLookupTable.lookup(id).list);
624 
625  /* check mask first */
626 
627  if (!CBIT_TEST(mask, id))
628  return NULL;
629 
630  /* looks like we must have it, do linear search */
631  for (auto e : entries) {
632  if (e && e->id == id)
633  return e;
634  }
635 
636  /* hm.. we thought it was there, but it was not found */
637  assert(false);
638  return nullptr; /* not reached */
639 }
640 
641 /*
642  * same as httpHeaderFindEntry
643  */
646 {
648  assert(!Http::HeaderLookupTable.lookup(id).list);
649 
650  /* check mask first */
651  if (!CBIT_TEST(mask, id))
652  return NULL;
653 
654  for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
655  if (*e && (*e)->id == id)
656  return *e;
657  }
658 
659  /* hm.. we thought it was there, but it was not found */
660  assert(false);
661  return nullptr; /* not reached */
662 }
663 
664 int
666 {
667  int count = 0;
669  httpHeaderMaskInit(&mask, 0); /* temporal inconsistency */
670  debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
671 
672  while (const HttpHeaderEntry *e = getEntry(&pos)) {
673  if (!e->name.caseCmp(name))
674  delAt(pos, count);
675  else
676  CBIT_SET(mask, e->id);
677  }
678 
679  return count;
680 }
681 
682 /* deletes all entries with a given id, returns the #entries deleted */
683 int
685 {
686  debugs(55, 8, this << " del-by-id " << id);
688 
689  if (!CBIT_TEST(mask, id))
690  return 0;
691 
692  int count = 0;
693 
695  while (HttpHeaderEntry *e = getEntry(&pos)) {
696  if (e->id == id)
697  delAt(pos, count); // deletes e
698  }
699 
700  CBIT_CLR(mask, id);
701  assert(count);
702  return count;
703 }
704 
705 /*
706  * deletes an entry at pos and leaves a gap; leaving a gap makes it
707  * possible to iterate(search) and delete fields at the same time
708  * NOTE: Does not update the header mask. Caller must follow up with
709  * a call to refreshMask() if headers_deleted was incremented.
710  */
711 void
712 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
713 {
714  HttpHeaderEntry *e;
715  assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
716  e = static_cast<HttpHeaderEntry*>(entries[pos]);
717  entries[pos] = NULL;
718  /* decrement header length, allow for ": " and crlf */
719  len -= e->name.length() + 2 + e->value.size() + 2;
720  assert(len >= 0);
721  delete e;
722  ++headers_deleted;
723 }
724 
725 /*
726  * Compacts the header storage
727  */
728 void
730 {
731  // TODO: optimize removal, or possibly make it so that's not needed.
732  entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
733  entries.end());
734 }
735 
736 /*
737  * Refreshes the header mask. Required after delAt() calls.
738  */
739 void
741 {
743  debugs(55, 7, "refreshing the mask in hdr " << this);
744  for (auto e : entries) {
745  if (e)
746  CBIT_SET(mask, e->id);
747  }
748 }
749 
750 /* appends an entry;
751  * does not call e->clone() so one should not reuse "*e"
752  */
753 void
755 {
756  assert(e);
758  assert(e->name.length());
759 
760  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
761 
762  if (e->id != Http::HdrType::BAD_HDR) {
763  if (CBIT_TEST(mask, e->id)) {
764  ++ headerStatsTable[e->id].repCount;
765  } else {
766  CBIT_SET(mask, e->id);
767  }
768  }
769 
770  entries.push_back(e);
771 
772  /* increment header length, allow for ": " and crlf */
773  len += e->name.length() + 2 + e->value.size() + 2;
774 }
775 
776 /* inserts an entry;
777  * does not call e->clone() so one should not reuse "*e"
778  */
779 void
781 {
782  assert(e);
784 
785  debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
786 
787  // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
788  if (CBIT_TEST(mask, e->id)) {
789  ++ headerStatsTable[e->id].repCount;
790  } else {
791  CBIT_SET(mask, e->id);
792  }
793 
794  entries.insert(entries.begin(),e);
795 
796  /* increment header length, allow for ": " and crlf */
797  len += e->name.length() + 2 + e->value.size() + 2;
798 }
799 
800 bool
802 {
803  debugs(55, 9, this << " joining for id " << id);
804  /* only fields from ListHeaders array can be "listed" */
805  assert(Http::HeaderLookupTable.lookup(id).list);
806 
807  if (!CBIT_TEST(mask, id))
808  return false;
809 
810  for (auto e: entries) {
811  if (e && e->id == id)
812  strListAdd(s, e->value.termedBuf(), ',');
813  }
814 
815  /*
816  * note: we might get an empty (size==0) string if there was an "empty"
817  * header. This results in an empty length String, which may have a NULL
818  * buffer.
819  */
820  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
821  if (!s->size())
822  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
823  else
824  debugs(55, 6, this << ": joined for id " << id << ": " << s);
825 
826  return true;
827 }
828 
829 /* return a list of entries with the same id separated by ',' and ws */
830 String
832 {
833  HttpHeaderEntry *e;
835  debugs(55, 9, this << "joining for id " << id);
836  /* only fields from ListHeaders array can be "listed" */
837  assert(Http::HeaderLookupTable.lookup(id).list);
838 
839  if (!CBIT_TEST(mask, id))
840  return String();
841 
842  String s;
843 
844  while ((e = getEntry(&pos))) {
845  if (e->id == id)
846  strListAdd(&s, e->value.termedBuf(), ',');
847  }
848 
849  /*
850  * note: we might get an empty (size==0) string if there was an "empty"
851  * header. This results in an empty length String, which may have a NULL
852  * buffer.
853  */
854  /* temporary warning: remove it? (Is it useful for diagnostics ?) */
855  if (!s.size())
856  debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
857  else
858  debugs(55, 6, this << ": joined for id " << id << ": " << s);
859 
860  return s;
861 }
862 
863 /* return a string or list of entries with the same id separated by ',' and ws */
864 String
866 {
867  HttpHeaderEntry *e;
868 
869  if (Http::HeaderLookupTable.lookup(id).list)
870  return getList(id);
871 
872  if ((e = findEntry(id)))
873  return e->value;
874 
875  return String();
876 }
877 
878 /*
879  * Returns the value of the specified header and/or an undefined String.
880  */
881 String
882 HttpHeader::getByName(const char *name) const
883 {
884  String result;
885  // ignore presence: return undefined string if an empty header is present
886  (void)hasNamed(name, strlen(name), &result);
887  return result;
888 }
889 
890 String
891 HttpHeader::getByName(const SBuf &name) const
892 {
893  String result;
894  // ignore presence: return undefined string if an empty header is present
895  (void)hasNamed(name, &result);
896  return result;
897 }
898 
899 String
901 {
902  String result;
903  (void)getByIdIfPresent(id, &result);
904  return result;
905 }
906 
907 bool
908 HttpHeader::hasNamed(const SBuf &s, String *result) const
909 {
910  return hasNamed(s.rawContent(), s.length(), result);
911 }
912 
913 bool
915 {
916  if (id == Http::HdrType::BAD_HDR)
917  return false;
918  if (!has(id))
919  return false;
920  if (result)
921  *result = getStrOrList(id);
922  return true;
923 }
924 
925 bool
926 HttpHeader::hasNamed(const char *name, unsigned int namelen, String *result) const
927 {
928  Http::HdrType id;
930  HttpHeaderEntry *e;
931 
932  assert(name);
933 
934  /* First try the quick path */
935  id = Http::HeaderLookupTable.lookup(name,namelen).id;
936 
937  if (id != Http::HdrType::BAD_HDR) {
938  if (getByIdIfPresent(id, result))
939  return true;
940  }
941 
942  /* Sorry, an unknown header name. Do linear search */
943  bool found = false;
944  while ((e = getEntry(&pos))) {
945  if (e->id == Http::HdrType::OTHER && e->name.length() == namelen && e->name.caseCmp(name, namelen) == 0) {
946  found = true;
947  if (!result)
948  break;
949  strListAdd(result, e->value.termedBuf(), ',');
950  }
951  }
952 
953  return found;
954 }
955 
956 /*
957  * Returns a the value of the specified list member, if any.
958  */
959 SBuf
960 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
961 {
962  assert(name);
963  const auto header = getByName(name);
964  return ::getListMember(header, member, separator);
965 }
966 
967 /*
968  * returns a the value of the specified list member, if any.
969  */
970 SBuf
971 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
972 {
974  const auto header = getStrOrList(id);
975  return ::getListMember(header, member, separator);
976 }
977 
978 /* test if a field is present */
979 int
981 {
983  debugs(55, 9, this << " lookup for " << id);
984  return CBIT_TEST(mask, id);
985 }
986 
987 void
989 {
990  // TODO: do not add Via header for messages where Squid itself
991  // generated the message (i.e., Downloader or ESI) there should be no Via header added at all.
992 
993  if (Config.onoff.via) {
994  SBuf buf;
995  // RFC 7230 section 5.7.1.: protocol-name is omitted when
996  // the received protocol is HTTP.
999  buf.appendf("%s/", AnyP::ProtocolType_str[ver.protocol]);
1000  buf.appendf("%d.%d %s", ver.major, ver.minor, ThisCache);
1001  const HttpHeader *hdr = from ? from : this;
1002  SBuf strVia = StringToSBuf(hdr->getList(Http::HdrType::VIA));
1003  if (!strVia.isEmpty())
1004  strVia.append(", ", 2);
1005  strVia.append(buf);
1006  // XXX: putStr() still suffers from String size limits
1007  Must(strVia.length() < String::SizeMaxXXX());
1009  putStr(Http::HdrType::VIA, strVia.c_str());
1010  }
1011 }
1012 
1013 void
1015 {
1017  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1018  assert(number >= 0);
1019  addEntry(new HttpHeaderEntry(id, SBuf(), xitoa(number)));
1020 }
1021 
1022 void
1024 {
1026  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1027  assert(number >= 0);
1028  addEntry(new HttpHeaderEntry(id, SBuf(), xint64toa(number)));
1029 }
1030 
1031 void
1033 {
1035  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1036  assert(htime >= 0);
1037  addEntry(new HttpHeaderEntry(id, SBuf(), mkrfc1123(htime)));
1038 }
1039 
1040 void
1041 HttpHeader::putStr(Http::HdrType id, const char *str)
1042 {
1044  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1045  assert(str);
1046  addEntry(new HttpHeaderEntry(id, SBuf(), str));
1047 }
1048 
1049 void
1050 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1051 {
1052  assert(auth_scheme && realm);
1053  httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1054 }
1055 
1056 void
1058 {
1059  assert(cc);
1060  /* remove old directives if any */
1062  /* pack into mb */
1063  MemBuf mb;
1064  mb.init();
1065  cc->packInto(&mb);
1066  /* put */
1068  /* cleanup */
1069  mb.clean();
1070 }
1071 
1072 void
1074 {
1075  assert(cr);
1076  /* remove old directives if any */
1078  /* pack into mb */
1079  MemBuf mb;
1080  mb.init();
1081  httpHdrContRangePackInto(cr, &mb);
1082  /* put */
1084  /* cleanup */
1085  mb.clean();
1086 }
1087 
1088 void
1090 {
1091  assert(range);
1092  /* remove old directives if any */
1094  /* pack into mb */
1095  MemBuf mb;
1096  mb.init();
1097  range->packInto(&mb);
1098  /* put */
1100  /* cleanup */
1101  mb.clean();
1102 }
1103 
1104 void
1106 {
1107  assert(sc);
1108  /* remove old directives if any */
1110  /* pack into mb */
1111  MemBuf mb;
1112  mb.init();
1113  sc->packInto(&mb);
1114  /* put */
1116  /* cleanup */
1117  mb.clean();
1118 }
1119 
1120 void
1121 HttpHeader::putWarning(const int code, const char *const text)
1122 {
1123  char buf[512];
1124  snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1126 }
1127 
1128 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1129 void
1130 HttpHeader::putExt(const char *name, const char *value)
1131 {
1132  assert(name && value);
1133  debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1134  addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, SBuf(name), value));
1135 }
1136 
1137 int
1139 {
1141  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt); /* must be of an appropriate type */
1142  HttpHeaderEntry *e;
1143 
1144  if ((e = findEntry(id)))
1145  return e->getInt();
1146 
1147  return -1;
1148 }
1149 
1150 int64_t
1152 {
1154  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64); /* must be of an appropriate type */
1155  HttpHeaderEntry *e;
1156 
1157  if ((e = findEntry(id)))
1158  return e->getInt64();
1159 
1160  return -1;
1161 }
1162 
1163 time_t
1165 {
1166  HttpHeaderEntry *e;
1167  time_t value = -1;
1169  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123); /* must be of an appropriate type */
1170 
1171  if ((e = findEntry(id))) {
1172  value = parse_rfc1123(e->value.termedBuf());
1173  httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1174  }
1175 
1176  return value;
1177 }
1178 
1179 /* sync with httpHeaderGetLastStr */
1180 const char *
1182 {
1183  HttpHeaderEntry *e;
1185  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1186 
1187  if ((e = findEntry(id))) {
1188  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1189  return e->value.termedBuf();
1190  }
1191 
1192  return NULL;
1193 }
1194 
1195 /* unusual */
1196 const char *
1198 {
1199  HttpHeaderEntry *e;
1201  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr); /* must be of an appropriate type */
1202 
1203  if ((e = findLastEntry(id))) {
1204  httpHeaderNoteParsedEntry(e->id, e->value, false); /* no errors are possible */
1205  return e->value.termedBuf();
1206  }
1207 
1208  return NULL;
1209 }
1210 
1211 HttpHdrCc *
1213 {
1215  return NULL;
1216  PROF_start(HttpHeader_getCc);
1217 
1218  String s;
1220 
1221  HttpHdrCc *cc=new HttpHdrCc();
1222 
1223  if (!cc->parse(s)) {
1224  delete cc;
1225  cc = NULL;
1226  }
1227 
1228  ++ HttpHeaderStats[owner].ccParsedCount;
1229 
1230  if (cc)
1231  httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1232 
1234 
1235  PROF_stop(HttpHeader_getCc);
1236 
1237  return cc;
1238 }
1239 
1240 HttpHdrRange *
1242 {
1243  HttpHdrRange *r = NULL;
1244  HttpHeaderEntry *e;
1245  /* some clients will send "Request-Range" _and_ *matching* "Range"
1246  * who knows, some clients might send Request-Range only;
1247  * this "if" should work correctly in both cases;
1248  * hopefully no clients send mismatched headers! */
1249 
1250  if ((e = findEntry(Http::HdrType::RANGE)) ||
1253  httpHeaderNoteParsedEntry(e->id, e->value, !r);
1254  }
1255 
1256  return r;
1257 }
1258 
1259 HttpHdrSc *
1261 {
1263  return NULL;
1264 
1265  String s;
1266 
1268 
1270 
1271  ++ HttpHeaderStats[owner].ccParsedCount;
1272 
1273  if (sc)
1274  sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1275 
1277 
1278  return sc;
1279 }
1280 
1283 {
1284  HttpHdrContRange *cr = NULL;
1285  HttpHeaderEntry *e;
1286 
1289  httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1290  }
1291 
1292  return cr;
1293 }
1294 
1295 SBuf
1296 HttpHeader::getAuthToken(Http::HdrType id, const char *auth_scheme) const
1297 {
1298  const char *field;
1299  int l;
1300  assert(auth_scheme);
1301  field = getStr(id);
1302 
1303  static const SBuf nil;
1304  if (!field) /* no authorization field */
1305  return nil;
1306 
1307  l = strlen(auth_scheme);
1308 
1309  if (!l || strncasecmp(field, auth_scheme, l)) /* wrong scheme */
1310  return nil;
1311 
1312  field += l;
1313 
1314  if (!xisspace(*field)) /* wrong scheme */
1315  return nil;
1316 
1317  /* skip white space */
1318  for (; field && xisspace(*field); ++field);
1319 
1320  if (!*field) /* no authorization cookie */
1321  return nil;
1322 
1323  const auto fieldLen = strlen(field);
1324  SBuf result;
1325  char *decodedAuthToken = result.rawAppendStart(BASE64_DECODE_LENGTH(fieldLen));
1326  struct base64_decode_ctx ctx;
1327  base64_decode_init(&ctx);
1328  size_t decodedLen = 0;
1329  if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), fieldLen, field) ||
1330  !base64_decode_final(&ctx)) {
1331  return nil;
1332  }
1333  result.rawAppendFinish(decodedAuthToken, decodedLen);
1334  return result;
1335 }
1336 
1337 ETag
1339 {
1340  ETag etag = {NULL, -1};
1341  HttpHeaderEntry *e;
1342  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag); /* must be of an appropriate type */
1343 
1344  if ((e = findEntry(id)))
1345  etagParseInit(&etag, e->value.termedBuf());
1346 
1347  return etag;
1348 }
1349 
1350 TimeOrTag
1352 {
1353  TimeOrTag tot;
1354  HttpHeaderEntry *e;
1355  assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag); /* must be of an appropriate type */
1356  memset(&tot, 0, sizeof(tot));
1357 
1358  if ((e = findEntry(id))) {
1359  const char *str = e->value.termedBuf();
1360  /* try as an ETag */
1361 
1362  if (etagParseInit(&tot.tag, str)) {
1363  tot.valid = tot.tag.str != NULL;
1364  tot.time = -1;
1365  } else {
1366  /* or maybe it is time? */
1367  tot.time = parse_rfc1123(str);
1368  tot.valid = tot.time >= 0;
1369  tot.tag.str = NULL;
1370  }
1371  }
1372 
1373  assert(tot.time < 0 || !tot.tag.str); /* paranoid */
1374  return tot;
1375 }
1376 
1377 /*
1378  * HttpHeaderEntry
1379  */
1380 
1381 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const SBuf &aName, const char *aValue)
1382 {
1384  id = anId;
1385 
1386  if (id != Http::HdrType::OTHER)
1387  name = Http::HeaderLookupTable.lookup(id).name;
1388  else
1389  name = aName;
1390 
1391  value = aValue;
1392 
1393  if (id != Http::HdrType::BAD_HDR)
1394  ++ headerStatsTable[id].aliveCount;
1395 
1396  debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1397 }
1398 
1400 {
1401  debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1402 
1403  if (id != Http::HdrType::BAD_HDR) {
1404  assert(headerStatsTable[id].aliveCount);
1405  -- headerStatsTable[id].aliveCount;
1406  id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1407  }
1408 
1409 }
1410 
1411 /* parses and inits header entry, returns true/false */
1413 HttpHeaderEntry::parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
1414 {
1415  /* note: name_start == field_start */
1416  const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1417  int name_len = name_end ? name_end - field_start :0;
1418  const char *value_start = field_start + name_len + 1; /* skip ':' */
1419  /* note: value_end == field_end */
1420 
1422 
1423  /* do we have a valid field name within this field? */
1424 
1425  if (!name_len || name_end > field_end)
1426  return NULL;
1427 
1428  if (name_len > 65534) {
1429  /* String must be LESS THAN 64K and it adds a terminating NULL */
1430  // TODO: update this to show proper name_len in Raw markup, but not print all that
1431  debugs(55, 2, "ignoring huge header field (" << Raw("field_start", field_start, 100) << "...)");
1432  return NULL;
1433  }
1434 
1435  /*
1436  * RFC 7230 section 3.2.4:
1437  * "No whitespace is allowed between the header field-name and colon.
1438  * ...
1439  * A server MUST reject any received request message that contains
1440  * whitespace between a header field-name and colon with a response code
1441  * of 400 (Bad Request). A proxy MUST remove any such whitespace from a
1442  * response message before forwarding the message downstream."
1443  */
1444  if (xisspace(field_start[name_len - 1])) {
1445 
1446  if (msgType == hoRequest)
1447  return nullptr;
1448 
1449  // for now, also let relaxed parser remove this BWS from any non-HTTP messages
1450  const bool stripWhitespace = (msgType == hoReply) ||
1452  if (!stripWhitespace)
1453  return nullptr; // reject if we cannot strip
1454 
1455  debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1456  "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1457 
1458  while (name_len > 0 && xisspace(field_start[name_len - 1]))
1459  --name_len;
1460 
1461  if (!name_len) {
1462  debugs(55, 2, "found header with only whitespace for name");
1463  return NULL;
1464  }
1465  }
1466 
1467  /* now we know we can parse it */
1468 
1469  debugs(55, 9, "parsing HttpHeaderEntry: near '" << getStringPrefix(field_start, field_end-field_start) << "'");
1470 
1471  /* is it a "known" field? */
1472  Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1473  debugs(55, 9, "got hdr-id=" << id);
1474 
1475  SBuf theName;
1476 
1477  String value;
1478 
1479  if (id == Http::HdrType::BAD_HDR)
1480  id = Http::HdrType::OTHER;
1481 
1482  /* set field name */
1483  if (id == Http::HdrType::OTHER)
1484  theName.append(field_start, name_len);
1485  else
1486  theName = Http::HeaderLookupTable.lookup(id).name;
1487 
1488  /* trim field value */
1489  while (value_start < field_end && xisspace(*value_start))
1490  ++value_start;
1491 
1492  while (value_start < field_end && xisspace(field_end[-1]))
1493  --field_end;
1494 
1495  if (field_end - value_start > 65534) {
1496  /* String must be LESS THAN 64K and it adds a terminating NULL */
1497  debugs(55, 2, "WARNING: found '" << theName << "' header of " << (field_end - value_start) << " bytes");
1498  return NULL;
1499  }
1500 
1501  /* set field value */
1502  value.assign(value_start, field_end - value_start);
1503 
1504  if (id != Http::HdrType::BAD_HDR)
1505  ++ headerStatsTable[id].seenCount;
1506 
1507  debugs(55, 9, "parsed HttpHeaderEntry: '" << theName << ": " << value << "'");
1508 
1509  return new HttpHeaderEntry(id, theName, value.termedBuf());
1510 }
1511 
1514 {
1515  return new HttpHeaderEntry(id, name, value.termedBuf());
1516 }
1517 
1518 void
1520 {
1521  assert(p);
1522  p->append(name.rawContent(), name.length());
1523  p->append(": ", 2);
1524  p->append(value.rawBuf(), value.size());
1525  p->append("\r\n", 2);
1526 }
1527 
1528 int
1530 {
1531  int val = -1;
1532  int ok = httpHeaderParseInt(value.termedBuf(), &val);
1533  httpHeaderNoteParsedEntry(id, value, ok == 0);
1534  /* XXX: Should we check ok - ie
1535  * return ok ? -1 : value;
1536  */
1537  return val;
1538 }
1539 
1540 int64_t
1542 {
1543  int64_t val = -1;
1544  const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1545  httpHeaderNoteParsedEntry(id, value, !ok);
1546  return val; // remains -1 if !ok (XXX: bad method API)
1547 }
1548 
1549 static void
1551 {
1552  if (id != Http::HdrType::BAD_HDR)
1553  ++ headerStatsTable[id].parsCount;
1554 
1555  if (error) {
1556  if (id != Http::HdrType::BAD_HDR)
1557  ++ headerStatsTable[id].errCount;
1558  debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1559  }
1560 }
1561 
1562 /*
1563  * Reports
1564  */
1565 
1566 /* tmp variable used to pass stat info to dumpers */
1567 extern const HttpHeaderStat *dump_stat; /* argh! */
1568 const HttpHeaderStat *dump_stat = NULL;
1569 
1570 void
1571 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1572 {
1573  const int id = static_cast<int>(val);
1574  const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1575  const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1576  int visible = count > 0;
1577  /* for entries with zero count, list only those that belong to current type of message */
1578 
1579  if (!visible && valid_id && dump_stat->owner_mask)
1580  visible = CBIT_TEST(*dump_stat->owner_mask, id);
1581 
1582  if (visible)
1583  storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1584  id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1585 }
1586 
1587 static void
1588 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1589 {
1590  if (count)
1591  storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1592  idx, (int) val, count,
1593  xpercent(count, dump_stat->destroyedCount));
1594 }
1595 
1596 static void
1598 {
1599  assert(hs);
1600  assert(e);
1601 
1602  dump_stat = hs;
1603  storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1604  storeAppendPrintf(e, "\nField type distribution\n");
1605  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1606  "id", "name", "count", "#/header");
1608  storeAppendPrintf(e, "\nCache-control directives distribution\n");
1609  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1610  "id", "name", "count", "#/cc_field");
1612  storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1613  storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1614  "id", "name", "count", "#/sc_field");
1616  storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1617  storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1618  "id", "#flds", "count", "%total");
1620  storeAppendPrintf(e, "\n");
1621  dump_stat = NULL;
1622 }
1623 
1624 void
1626 {
1627  int i;
1628  assert(e);
1629 
1630  HttpHeaderStats[0].parsedCount =
1631  HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1632  HttpHeaderStats[0].ccParsedCount =
1633  HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1634  HttpHeaderStats[0].destroyedCount =
1635  HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1636  HttpHeaderStats[0].busyDestroyedCount =
1637  HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1638 
1639  for (i = 1; i < HttpHeaderStatCount; ++i) {
1640  httpHeaderStatDump(HttpHeaderStats + i, e);
1641  }
1642 
1643  /* field stats for all messages */
1644  storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1645 
1646  storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1647  "id", "name", "#alive", "%err", "%repeat");
1648 
1649  // scan heaaderTable and output
1650  for (auto h : WholeEnum<Http::HdrType>()) {
1651  auto stats = headerStatsTable[h];
1652  storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1653  Http::HeaderLookupTable.lookup(h).id,
1654  Http::HeaderLookupTable.lookup(h).name,
1655  stats.aliveCount,
1656  xpercent(stats.errCount, stats.parsCount),
1657  xpercent(stats.repCount, stats.seenCount));
1658  }
1659 
1660  storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1661  HttpHeaderStats[hoRequest].parsedCount,
1662  HttpHeaderStats[hoReply].parsedCount,
1663  HttpHeaderStats[0].parsedCount);
1664  storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1665 }
1666 
1667 int
1668 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1669 {
1670  int result = 0;
1671  const char *pos = NULL;
1672  const char *item;
1673  int ilen;
1674  int mlen = strlen(member);
1675 
1677 
1678  String header (getStrOrList(id));
1679 
1680  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1681  if (strncasecmp(item, member, mlen) == 0
1682  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1683  result = 1;
1684  break;
1685  }
1686  }
1687 
1688  return result;
1689 }
1690 
1691 int
1692 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1693 {
1694  int result = 0;
1695  const char *pos = NULL;
1696  const char *item;
1697  int ilen;
1698  int mlen = strlen(member);
1699 
1700  assert(name);
1701 
1702  String header (getByName(name));
1703 
1704  while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1705  if (strncasecmp(item, member, mlen) == 0
1706  && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1707  result = 1;
1708  break;
1709  }
1710  }
1711 
1712  return result;
1713 }
1714 
1715 void
1717 {
1719 
1720  const HttpHeaderEntry *e;
1722  int headers_deleted = 0;
1723  while ((e = getEntry(&pos))) {
1724  Http::HdrType id = e->id;
1725  if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1726  delAt(pos, headers_deleted);
1727  CBIT_CLR(mask, id);
1728  }
1729  }
1730 }
1731 
1732 void
1734 {
1736  /* anything that matches Connection list member will be deleted */
1737  String strConnection;
1738 
1739  (void) getList(Http::HdrType::CONNECTION, &strConnection);
1740  const HttpHeaderEntry *e;
1742  /*
1743  * think: on-average-best nesting of the two loops (hdrEntry
1744  * and strListItem) @?@
1745  */
1746  /*
1747  * maybe we should delete standard stuff ("keep-alive","close")
1748  * from strConnection first?
1749  */
1750 
1751  int headers_deleted = 0;
1752  while ((e = getEntry(&pos))) {
1753  if (strListIsMember(&strConnection, e->name, ','))
1754  delAt(pos, headers_deleted);
1755  }
1756  if (headers_deleted)
1757  refreshMask();
1758  }
1759 }
1760 
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1668
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:404
int delById(Http::HdrType id)
Definition: HttpHeader.cc:684
HttpHdrSc * httpHdrScParseCreate(const String &str)
Definition: HttpHdrSc.cc:60
void removeConnectionHeaderEntries()
Definition: HttpHeader.cc:1733
bool httpHeaderParseOffset(const char *start, int64_t *value, char **endPtr)
void count(double val)
Definition: StatHist.cc:57
void httpHeaderFieldStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHeader.cc:1571
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:77
void httpHdrScInitModule(void)
Definition: HttpHdrSc.cc:49
#define assert(EX)
Definition: assert.h:17
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:239
time_t parse_rfc1123(const char *str)
Definition: rfc1123.c:159
static int sc[16]
Definition: smbdes.c:121
const HeaderTableRecord & lookup(const char *buf, const std::size_t len) const
look record type up by name (C-string and length)
const char * getLastStr(Http::HdrType id) const
Definition: HttpHeader.cc:1197
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
const char * mkrfc1123(time_t)
Definition: rfc1123.c:202
int caseCmp(const SBuf &S, const size_type n) const
shorthand version for case-insensitive compare()
Definition: SBuf.h:272
http_hdr_owner_type
Definition: HttpHeader.h:31
SQUIDCEXTERN double xpercent(double part, double whole)
Definition: util.c:54
#define CBIT_TEST(mask, bit)
Definition: defines.h:114
static const CharacterSet LF
Definition: CharacterSet.h:89
SQUIDCEXTERN const char * xitoa(int num)
Definition: util.c:79
Definition: ETag.h:17
int getInt(Http::HdrType id) const
Definition: HttpHeader.cc:1138
std::vector< HttpHeaderEntry *, PoolingAllocator< HttpHeaderEntry * > > entries
Definition: HttpHeader.h:165
int type
Definition: errorpage.cc:152
void packInto(Packable *p) const
Definition: HttpHdrCc.cc:246
Definition: SBuf.h:86
void putContRange(const HttpHdrContRange *cr)
Definition: HttpHeader.cc:1073
String getByName(const SBuf &name) const
Definition: HttpHeader.cc:891
int etagParseInit(ETag *etag, const char *str)
Definition: ETag.cc:29
SBuf getListMember(const String &list, const char *key, const char delimiter)
Definition: StrList.cc:135
HttpHeaderEntry * findEntry(Http::HdrType id) const
Definition: HttpHeader.cc:620
bool any_HdrType_enum_value(const Http::HdrType id)
match any known header type, including OTHER and BAD
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:831
time_t time
Definition: TimeOrTag.h:21
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
void error(char *format,...)
int delByName(const SBuf &name)
Definition: HttpHeader.cc:665
int i
Definition: membanger.c:49
void removeHopByHopEntries()
Definition: HttpHeader.cc:1716
SBuf & append(const SBuf &S)
Definition: SBuf.cc:195
void updateStats(StatHist *) const
Definition: HttpHdrSc.cc:277
static bool Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
Definition: HttpHeader.cc:331
HttpHeaderMask mask
Definition: HttpHeader.h:166
void putExt(const char *name, const char *value)
Definition: HttpHeader.cc:1130
class Ping::pingStats_ stats
void base64_decode_init(struct base64_decode_ctx *ctx)
Definition: base64.c:54
String getStrOrList(Http::HdrType id) const
Definition: HttpHeader.cc:865
void httpHdrCcStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrCc.cc:330
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1513
static size_type SizeMaxXXX()
Definition: SquidString.h:70
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
bool isEmpty() const
Definition: SBuf.h:420
void strListAdd(String *str, const char *item, char del)
Definition: StrList.cc:19
bool conflictingContentLength_
found different Content-Length fields
Definition: HttpHeader.h:185
#define xisspace(x)
Definition: xis.h:17
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:601
#define DBG_CRITICAL
Definition: Debug.h:45
char * p
Definition: membanger.c:43
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:37
#define BASE64_DECODE_LENGTH(length)
Definition: base64.h:120
SBuf getListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:971
bool needUpdate(const HttpHeader *fresh) const
Definition: HttpHeader.cc:250
virtual void append(const char *buf, int size)=0
Appends a c-string to existing packed data.
#define CBIT_SET(mask, bit)
Definition: defines.h:112
static void httpHeaderFldsPerHdrDumper(StoreEntry *sentry, int idx, double val, double, int count)
Definition: HttpHeader.cc:1588
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
ETag tag
Definition: TimeOrTag.h:20
static const CharacterSet WSP
Definition: CharacterSet.h:95
int hasByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:1692
void rawAppendFinish(const char *start, size_type actualSize)
Definition: SBuf.cc:144
Http::HdrType id
Definition: HttpHeader.h:63
const char * headerWideProblem
worst header-wide problem found (or nil)
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
void delAt(HttpHeaderPos pos, int &headers_deleted)
Definition: HttpHeader.cc:712
void append(const HttpHeader *src)
Definition: HttpHeader.cc:237
unsigned int major
major version number
int getInt() const
Definition: HttpHeader.cc:1529
String getById(Http::HdrType id) const
Definition: HttpHeader.cc:900
int relaxed_header_parser
Definition: SquidConfig.h:318
ETag getETag(Http::HdrType id) const
Definition: HttpHeader.cc:1338
char * rawAppendStart(size_type anticipatedSize)
Definition: SBuf.cc:136
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
HttpHdrContRange * getContRange() const
Definition: HttpHeader.cc:1282
void httpHdrScStatDumper(StoreEntry *sentry, int, double val, double, int count)
Definition: HttpHdrSc.cc:301
#define DBG_IMPORTANT
Definition: Debug.h:46
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
int parse(const char *header_start, size_t len, Http::ContentLengthInterpreter &interpreter)
Definition: HttpHeader.cc:376
int64_t getInt64() const
Definition: HttpHeader.cc:1541
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1032
bool chunked() const
whether message uses chunked Transfer-Encoding
Definition: HttpHeader.h:196
static HttpHeaderMask ReplyHeadersMask
Definition: HttpHeader.cc:73
HttpHeaderEntry(Http::HdrType id, const SBuf &name, const char *value)
Definition: HttpHeader.cc:1381
SBuf getByNameListMember(const char *name, const char *member, const char separator) const
Definition: HttpHeader.cc:960
static void httpHeaderRegisterWithCacheManager(void)
Definition: HttpHeader.cc:111
HttpHeaderEntry * findLastEntry(Http::HdrType id) const
Definition: HttpHeader.cc:645
#define countof(arr)
Definition: defines.h:186
const char * label
static const CharacterSet CR
Definition: CharacterSet.h:77
http_hdr_owner_type owner
Definition: HttpHeader.h:167
char HttpHeaderMask[12]
bool parse(const String &s)
parse a header-string and fill in appropriate values.
Definition: HttpHdrCc.cc:94
static HttpHeaderEntry * parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType)
Definition: HttpHeader.cc:1413
const char * c_str()
Definition: SBuf.cc:526
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:17
void putSc(HttpHdrSc *sc)
Definition: HttpHeader.cc:1105
void putWarning(const int code, const char *const text)
add a Warning header
Definition: HttpHeader.cc:1121
static HttpHdrRange * ParseCreate(const String *range_spec)
void refreshMask()
Definition: HttpHeader.cc:740
const char * getStringPrefix(const char *str, size_t sz)
void putInt(Http::HdrType id, int number)
Definition: HttpHeader.cc:1014
StatHist ccTypeDistr
HttpHdrContRange * httpHdrContRangeParseCreate(const char *str)
char const * termedBuf() const
Definition: SquidString.h:91
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
time_t getTime(Http::HdrType id) const
Definition: HttpHeader.cc:1164
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=0)
Definition: HttpHeader.cc:988
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1181
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1212
unsigned char code
Definition: html_quote.c:20
void const char * buf
Definition: stub_helper.cc:16
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
struct SquidConfig::@112 onoff
static int HeaderEntryParsedCount
Definition: HttpHeader.cc:91
void httpHeaderInitModule(void)
Definition: HttpHeader.cc:119
HTTP per header statistics.
void clean()
Definition: MemBuf.cc:113
void httpHdrCcInitModule(void)
Module initialization hook.
Definition: HttpHdrCc.cc:60
StatHist fieldTypeDistr
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1023
char * buf
Definition: MemBuf.h:134
SQUIDCEXTERN const char * xint64toa(int64_t num)
Definition: util.c:88
int has(Http::HdrType id) const
Definition: HttpHeader.cc:980
#define CBIT_CLR(mask, bit)
Definition: defines.h:113
number
Definition: testStatHist.cc:16
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1041
void clean()
Definition: HttpHeader.cc:189
int valid
Definition: TimeOrTag.h:22
const HeaderLookupTable_t HeaderLookupTable
bool any_valid_header(const Http::HdrType id)
match any valid header type, including OTHER but not BAD
void packInto(Packable *p) const
Definition: HttpHeader.cc:1519
#define PROF_start(probename)
Definition: Profiler.h:62
void putCc(const HttpHdrCc *cc)
Definition: HttpHeader.cc:1057
SQUIDCEXTERN double xdiv(double nom, double denom)
Definition: util.c:72
void update(const HttpHeader *fresh)
Definition: HttpHeader.cc:295
void httpHdrContRangePackInto(const HttpHdrContRange *range, Packable *p)
void updateWarnings()
Definition: HttpHeader.cc:271
static void httpHeaderStatDump(const HttpHeaderStat *hs, StoreEntry *e)
Definition: HttpHeader.cc:1597
char ThisCache[RFC2181_MAXHOSTNAMELEN<< 1]
HttpHeader & operator=(const HttpHeader &other)
Definition: HttpHeader.cc:175
void compact()
Definition: HttpHeader.cc:729
static const size_type npos
Definition: SBuf.h:92
std::vector< HttpHeaderFieldStat > headerStatsTable(Http::HdrType::enumEnd_)
void httpHdrCcUpdateStats(const HttpHdrCc *cc, StatHist *hist)
Definition: HttpHdrCc.cc:320
Definition: MemBuf.h:23
bool any_registered_header(const Http::HdrType id)
HttpHdrSc * getSc() const
Definition: HttpHeader.cc:1260
const char * ProtocolType_str[]
bool skipUpdateHeader(const Http::HdrType id) const
Definition: HttpHeader.cc:284
const char * str
quoted-string
Definition: ETag.h:20
bool sawBad
whether a malformed Content-Length value was present
bool getByIdIfPresent(Http::HdrType id, String *result) const
Definition: HttpHeader.cc:914
static int HttpHeaderStatCount
Definition: HttpHeader.cc:89
Definition: Debug.h:188
unsigned int minor
minor version number
void packInto(Packable *p) const
#define PROF_stop(probename)
Definition: Profiler.h:63
SBuf StringToSBuf(const String &s)
create a new SBuf from a String by copying contents
Definition: StringConvert.h:17
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:754
void assign(const char *str, int len)
Definition: String.cc:94
static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error)
Definition: HttpHeader.cc:1550
SBuf getAuthToken(Http::HdrType id, const char *auth_scheme) const
Definition: HttpHeader.cc:1296
void insertEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:780
void httpHeaderMaskInit(HttpHeaderMask *mask, int value)
int base64_decode_update(struct base64_decode_ctx *ctx, size_t *dst_length, uint8_t *dst, size_t src_length, const char *src)
Definition: base64.c:129
const HttpHeaderStat * dump_stat
Definition: HttpHeader.cc:1568
TimeOrTag getTimeOrTag(Http::HdrType id) const
Definition: HttpHeader.cc:1351
ProtocolType protocol
which protocol this version is for
StatHist hdrUCountDistr
size_t headersEnd(const char *mime, size_t l, bool &containsObsFold)
Definition: mime_header.cc:16
void putRange(const HttpHdrRange *range)
Definition: HttpHeader.cc:1089
void packInto(Packable *p) const
Definition: HttpHdrSc.cc:250
bool hasNamed(const SBuf &s, String *value=0) const
Definition: HttpHeader.cc:908
int base64_decode_final(struct base64_decode_ctx *ctx)
Definition: base64.c:159
HttpHeaderMask * owner_mask
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:881
char const * visible_appname_string
size_type size() const
Definition: SquidString.h:72
StatHist scTypeDistr
static HttpHeaderMask RequestHeadersMask
Definition: HttpHeader.cc:70
void putAuth(const char *auth_scheme, const char *realm)
Definition: HttpHeader.cc:1050
class SquidConfig Config
Definition: SquidConfig.cc:12
void Controller::create() STUB void Controller Controller nil
#define NULL
Definition: types.h:166
const char * text
Definition: errorpage.cc:153
static void httpHeaderStoreReport(StoreEntry *e)
Definition: HttpHeader.cc:1625
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:557
void dump(StoreEntry *sentry, StatHistBinDumper *bd) const
Definition: StatHist.cc:173
int httpHeaderParseInt(const char *start, int *value)
const char * rawContent() const
Definition: SBuf.cc:519
#define false
Definition: GnuRegex.c:233
int64_t getInt64(Http::HdrType id) const
Definition: HttpHeader.cc:1151
HttpHdrRange * getRange() const
Definition: HttpHeader.cc:1241

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors