Tokenizer.h
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9#ifndef SQUID_PARSER_TOKENIZER_H_
10#define SQUID_PARSER_TOKENIZER_H_
11
12#include "base/CharacterSet.h"
13#include "sbuf/SBuf.h"
14
16namespace Parser
17{
18
30{
31public:
32 explicit Tokenizer(const SBuf &inBuf) : buf_(inBuf), parsed_(0) {}
33
35 SBuf buf() const { return buf_; }
36
39
41 bool atEnd() const { return buf_.isEmpty(); }
42
44 const SBuf& remaining() const { return buf_; }
45
47 void reset(const SBuf &newBuf) { undoParse(newBuf, 0); }
48
61 bool token(SBuf &returnedToken, const CharacterSet &delimiters);
62
71 bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
72
82 bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
83
92 bool skipSuffix(const SBuf &tokenToSkip);
93
98 bool skip(const SBuf &tokenToSkip);
99
104 bool skip(const char tokenChar);
105
110 bool skipOne(const CharacterSet &discardables);
111
116 SBuf::size_type skipAll(const CharacterSet &discardables);
117
122 bool skipOneTrailing(const CharacterSet &discardables);
123
128 SBuf::size_type skipAllTrailing(const CharacterSet &discardables);
129
144 bool int64(int64_t &result, int base = 0, bool allowSign = true, SBuf::size_type limit = SBuf::npos);
145
146 /*
147 * The methods below mimic their counterparts documented above, but they
148 * throw on errors, including InsufficientInput. The field description
149 * parameter is used for error reporting and debugging.
150 */
151
154 SBuf prefix(const char *description, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
155
157 int64_t udec64(const char *description, SBuf::size_type limit = SBuf::npos);
158
159protected:
164
166 void undoParse(const SBuf &newBuf, SBuf::size_type cParsed) { buf_ = newBuf; parsed_ = cParsed; }
167
168private:
171};
172
173} /* namespace Parser */
174
175#endif /* SQUID_PARSER_TOKENIZER_H_ */
176
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:18
SBuf::size_type skipAllTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:212
bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:81
SBuf buf_
yet unparsed input
Definition: Tokenizer.h:169
bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:119
SBuf::size_type successTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns their count
Definition: Tokenizer.cc:57
bool skipOne(const CharacterSet &discardables)
Definition: Tokenizer.cc:151
bool token(SBuf &returnedToken, const CharacterSet &delimiters)
Definition: Tokenizer.cc:63
SBuf consumeTrailing(const SBuf::size_type n)
convenience method: consumes up to n last bytes and returns them
Definition: Tokenizer.cc:42
SBuf::size_type success(const SBuf::size_type n)
convenience method: consume()s up to n bytes and returns their count
Definition: Tokenizer.cc:35
void reset(const SBuf &newBuf)
reinitialize processing for a new buffer
Definition: Tokenizer.h:47
void undoParse(const SBuf &newBuf, SBuf::size_type cParsed)
reset the buffer and parsed stats to a saved checkpoint
Definition: Tokenizer.h:166
bool skipSuffix(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:162
SBuf::size_type skipAll(const CharacterSet &discardables)
Definition: Tokenizer.cc:139
bool int64(int64_t &result, int base=0, bool allowSign=true, SBuf::size_type limit=SBuf::npos)
Definition: Tokenizer.cc:228
SBuf::size_type parsedSize() const
number of parsed bytes, including skipped ones
Definition: Tokenizer.h:38
SBuf consume(const SBuf::size_type n)
convenience method: consumes up to n bytes, counts, and returns them
Definition: Tokenizer.cc:24
SBuf buf() const
yet unparsed data
Definition: Tokenizer.h:35
Tokenizer(const SBuf &inBuf)
Definition: Tokenizer.h:32
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Tokenizer.h:44
bool skipOneTrailing(const CharacterSet &discardables)
Definition: Tokenizer.cc:201
bool atEnd() const
whether the end of the buffer has been reached
Definition: Tokenizer.h:41
SBuf::size_type parsed_
bytes successfully parsed, including skipped
Definition: Tokenizer.h:170
int64_t udec64(const char *description, SBuf::size_type limit=SBuf::npos)
int64() wrapper but limited to unsigned decimal integers (for now)
Definition: Tokenizer.cc:306
bool skip(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:179
Definition: SBuf.h:94
static const size_type npos
Definition: SBuf.h:99
bool isEmpty() const
Definition: SBuf.h:431
MemBlob::size_type size_type
Definition: SBuf.h:96
Generic protocol-agnostic parsing tools.
Definition: RequestParser.h:15

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors