Tokenizer.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #ifndef SQUID_PARSER_TOKENIZER_H_
10 #define SQUID_PARSER_TOKENIZER_H_
11 
12 #include "base/CharacterSet.h"
13 #include "sbuf/SBuf.h"
14 
16 namespace Parser
17 {
18 
29 class Tokenizer
30 {
31 public:
32  explicit Tokenizer(const SBuf &inBuf) : buf_(inBuf), parsed_(0) {}
33 
35  SBuf buf() const { return buf_; }
36 
38  SBuf::size_type parsedSize() const { return parsed_; }
39 
41  bool atEnd() const { return buf_.isEmpty(); }
42 
44  const SBuf& remaining() const { return buf_; }
45 
47  void reset(const SBuf &newBuf) { undoParse(newBuf, 0); }
48 
61  bool token(SBuf &returnedToken, const CharacterSet &delimiters);
62 
71  bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
72 
82  bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
83 
92  bool skipSuffix(const SBuf &tokenToSkip);
93 
98  bool skip(const SBuf &tokenToSkip);
99 
104  bool skip(const char tokenChar);
105 
110  bool skipOne(const CharacterSet &discardables);
111 
116  SBuf::size_type skipAll(const CharacterSet &discardables);
117 
122  bool skipOneTrailing(const CharacterSet &discardables);
123 
128  SBuf::size_type skipAllTrailing(const CharacterSet &discardables);
129 
144  bool int64(int64_t &result, int base = 0, bool allowSign = true, SBuf::size_type limit = SBuf::npos);
145 
146 protected:
147  SBuf consume(const SBuf::size_type n);
151 
153  void undoParse(const SBuf &newBuf, SBuf::size_type cParsed) { buf_ = newBuf; parsed_ = cParsed; }
154 
155 private:
158 };
159 
160 } /* namespace Parser */
161 
162 #endif /* SQUID_PARSER_TOKENIZER_H_ */
163 
SBuf consume(const SBuf::size_type n)
convenience method: consumes up to n bytes, counts, and returns them
Definition: Tokenizer.cc:22
Definition: SBuf.h:87
bool isEmpty() const
Definition: SBuf.h:422
SBuf::size_type successTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns their count
Definition: Tokenizer.cc:55
bool skipSuffix(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:143
SBuf consumeTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns them
Definition: Tokenizer.cc:40
SBuf::size_type parsed_
bytes successfully parsed, including skipped
Definition: Tokenizer.h:157
SBuf::size_type skipAllTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:193
SBuf::size_type parsedSize() const
number of parsed bytes, including skipped ones
Definition: Tokenizer.h:38
bool atEnd() const
whether the end of the buffer has been reached
Definition: Tokenizer.h:41
SBuf buf_
yet unparsed input
Definition: Tokenizer.h:156
bool skipOneTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:182
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:17
bool skipOne(const CharacterSet &discardables)
Definition: Tokenizer.cc:132
SBuf::size_type success(const SBuf::size_type n)
convenience method: consume()s up to n bytes and returns their count
Definition: Tokenizer.cc:33
void reset(const SBuf &newBuf)
reinitialize processing for a new buffer
Definition: Tokenizer.h:47
bool skip(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:160
Tokenizer(const SBuf &inBuf)
Definition: Tokenizer.h:32
bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:79
bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:100
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Tokenizer.h:44
void undoParse(const SBuf &newBuf, SBuf::size_type cParsed)
reset the buffer and parsed stats to a saved checkpoint
Definition: Tokenizer.h:153
static const size_type npos
Definition: SBuf.h:93
SBuf::size_type skipAll(const CharacterSet &discardables)
Definition: Tokenizer.cc:120
MemBlob::size_type size_type
Definition: SBuf.h:90
SBuf buf() const
yet unparsed data
Definition: Tokenizer.h:35
bool token(SBuf &returnedToken, const CharacterSet &delimiters)
Definition: Tokenizer.cc:61
bool int64(int64_t &result, int base=0, bool allowSign=true, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:209

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors