Tokenizer.h
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9#ifndef SQUID_PARSER_TOKENIZER_H_
10#define SQUID_PARSER_TOKENIZER_H_
11
12#include "base/CharacterSet.h"
13#include "sbuf/SBuf.h"
14
16namespace Parser
17{
18
30{
31public:
32 explicit Tokenizer(const SBuf &inBuf) : buf_(inBuf), parsed_(0) {}
33
35 SBuf buf() const { return buf_; }
36
39
41 bool atEnd() const { return buf_.isEmpty(); }
42
44 const SBuf& remaining() const { return buf_; }
45
47 void reset(const SBuf &newBuf) { undoParse(newBuf, 0); }
48
61 bool token(SBuf &returnedToken, const CharacterSet &delimiters);
62
71 bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
72
82 bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
83
92 bool skipSuffix(const SBuf &tokenToSkip);
93
98 bool skip(const SBuf &tokenToSkip);
99
104 bool skip(const char tokenChar);
105
110 bool skipOne(const CharacterSet &discardables);
111
116 SBuf::size_type skipAll(const CharacterSet &discardables);
117
123 void skipRequired(const char *description, const SBuf &tokenToSkip);
124
129 bool skipOneTrailing(const CharacterSet &discardables);
130
135 SBuf::size_type skipAllTrailing(const CharacterSet &discardables);
136
151 bool int64(int64_t &result, int base = 0, bool allowSign = true, SBuf::size_type limit = SBuf::npos);
152
153 /*
154 * The methods below mimic their counterparts documented above, but they
155 * throw on errors, including InsufficientInput. The field description
156 * parameter is used for error reporting and debugging.
157 */
158
161 SBuf prefix(const char *description, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
162
164 int64_t udec64(const char *description, SBuf::size_type limit = SBuf::npos);
165
166protected:
171
173 void undoParse(const SBuf &newBuf, SBuf::size_type cParsed) { buf_ = newBuf; parsed_ = cParsed; }
174
175private:
178};
179
180} /* namespace Parser */
181
182#endif /* SQUID_PARSER_TOKENIZER_H_ */
183
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:18
SBuf::size_type skipAllTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:222
bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:79
SBuf buf_
yet unparsed input
Definition: Tokenizer.h:176
bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:117
SBuf::size_type successTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns their count
Definition: Tokenizer.cc:55
bool skipOne(const CharacterSet &discardables)
Definition: Tokenizer.cc:161
bool token(SBuf &returnedToken, const CharacterSet &delimiters)
Definition: Tokenizer.cc:61
SBuf consumeTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns them
Definition: Tokenizer.cc:40
SBuf::size_type success(const SBuf::size_type n)
convenience method: consume()s up to n bytes and returns their count
Definition: Tokenizer.cc:33
void reset(const SBuf &newBuf)
reinitialize processing for a new buffer
Definition: Tokenizer.h:47
void undoParse(const SBuf &newBuf, SBuf::size_type cParsed)
reset the buffer and parsed stats to a saved checkpoint
Definition: Tokenizer.h:173
bool skipSuffix(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:172
SBuf::size_type skipAll(const CharacterSet &discardables)
Definition: Tokenizer.cc:137
bool int64(int64_t &result, int base=0, bool allowSign=true, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:238
SBuf::size_type parsedSize() const
number of parsed bytes, including skipped ones
Definition: Tokenizer.h:38
SBuf consume(const SBuf::size_type n)
convenience method: consumes up to n bytes, counts, and returns them
Definition: Tokenizer.cc:22
SBuf buf() const
yet unparsed data
Definition: Tokenizer.h:35
Tokenizer(const SBuf &inBuf)
Definition: Tokenizer.h:32
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Tokenizer.h:44
bool skipOneTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:211
void skipRequired(const char *description, const SBuf &tokenToSkip)
Definition: Tokenizer.cc:149
bool atEnd() const
whether the end of the buffer has been reached
Definition: Tokenizer.h:41
SBuf::size_type parsed_
bytes successfully parsed, including skipped
Definition: Tokenizer.h:177
int64_t udec64(const char *description, SBuf::size_type limit=SBuf::npos)
int64() wrapper but limited to unsigned decimal integers (for now)
Definition: Tokenizer.cc:316
bool skip(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:189
Definition: SBuf.h:94
static const size_type npos
Definition: SBuf.h:99
bool isEmpty() const
Definition: SBuf.h:431
MemBlob::size_type size_type
Definition: SBuf.h:96
Generic protocol-agnostic parsing tools.
Definition: RequestParser.h:15

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors