Tokenizer.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #ifndef SQUID_PARSER_TOKENIZER_H_
10 #define SQUID_PARSER_TOKENIZER_H_
11 
12 #include "base/CharacterSet.h"
13 #include "sbuf/SBuf.h"
14 
16 namespace Parser
17 {
18 
29 class Tokenizer
30 {
31 public:
32  explicit Tokenizer(const SBuf &inBuf) : buf_(inBuf), parsed_(0) {}
33 
35  SBuf buf() const { return buf_; }
36 
38  SBuf::size_type parsedSize() const { return parsed_; }
39 
41  bool atEnd() const { return buf_.isEmpty(); }
42 
44  const SBuf& remaining() const { return buf_; }
45 
47  void reset(const SBuf &newBuf) { undoParse(newBuf, 0); }
48 
61  bool token(SBuf &returnedToken, const CharacterSet &delimiters);
62 
71  bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
72 
82  bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
83 
92  bool skipSuffix(const SBuf &tokenToSkip);
93 
98  bool skip(const SBuf &tokenToSkip);
99 
104  bool skip(const char tokenChar);
105 
110  bool skipOne(const CharacterSet &discardables);
111 
116  SBuf::size_type skipAll(const CharacterSet &discardables);
117 
122  bool skipOneTrailing(const CharacterSet &discardables);
123 
128  SBuf::size_type skipAllTrailing(const CharacterSet &discardables);
129 
144  bool int64(int64_t &result, int base = 0, bool allowSign = true, SBuf::size_type limit = SBuf::npos);
145 
146  /*
147  * The methods below mimic their counterparts documented above, but they
148  * throw on errors, including InsufficientInput. The field description
149  * parameter is used for error reporting and debugging.
150  */
151 
154  SBuf prefix(const char *description, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
155 
157  int64_t udec64(const char *description, SBuf::size_type limit = SBuf::npos);
158 
159 protected:
160  SBuf consume(const SBuf::size_type n);
164 
166  void undoParse(const SBuf &newBuf, SBuf::size_type cParsed) { buf_ = newBuf; parsed_ = cParsed; }
167 
168 private:
171 };
172 
173 } /* namespace Parser */
174 
175 #endif /* SQUID_PARSER_TOKENIZER_H_ */
176 
int64_t udec64(const char *description, SBuf::size_type limit=SBuf::npos)
int64() wrapper but limited to unsigned decimal integers (for now)
Definition: Tokenizer.cc:306
SBuf consume(const SBuf::size_type n)
convenience method: consumes up to n bytes, counts, and returns them
Definition: Tokenizer.cc:24
Definition: SBuf.h:86
bool isEmpty() const
Definition: SBuf.h:420
Generic protocol-agnostic parsing tools.
Definition: RequestParser.h:15
SBuf::size_type successTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns their count
Definition: Tokenizer.cc:57
bool skipSuffix(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:162
SBuf consumeTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns them
Definition: Tokenizer.cc:42
SBuf::size_type parsed_
bytes successfully parsed, including skipped
Definition: Tokenizer.h:170
SBuf::size_type skipAllTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:212
SBuf buf() const
yet unparsed data
Definition: Tokenizer.h:35
SBuf buf_
yet unparsed input
Definition: Tokenizer.h:169
bool skipOneTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:201
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Tokenizer.h:44
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:17
bool skipOne(const CharacterSet &discardables)
Definition: Tokenizer.cc:151
SBuf::size_type success(const SBuf::size_type n)
convenience method: consume()s up to n bytes and returns their count
Definition: Tokenizer.cc:35
void reset(const SBuf &newBuf)
reinitialize processing for a new buffer
Definition: Tokenizer.h:47
SBuf::size_type parsedSize() const
number of parsed bytes, including skipped ones
Definition: Tokenizer.h:38
bool skip(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:179
Tokenizer(const SBuf &inBuf)
Definition: Tokenizer.h:32
bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:81
bool atEnd() const
whether the end of the buffer has been reached
Definition: Tokenizer.h:41
bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:119
void undoParse(const SBuf &newBuf, SBuf::size_type cParsed)
reset the buffer and parsed stats to a saved checkpoint
Definition: Tokenizer.h:166
static const size_type npos
Definition: SBuf.h:92
SBuf::size_type skipAll(const CharacterSet &discardables)
Definition: Tokenizer.cc:139
MemBlob::size_type size_type
Definition: SBuf.h:89
bool token(SBuf &returnedToken, const CharacterSet &delimiters)
Definition: Tokenizer.cc:63
bool int64(int64_t &result, int base=0, bool allowSign=true, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:228

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors