17SBuf text(
"GET http://resource.com/path HTTP/1.1\r\n"
18 "Host: resource.com\r\n"
19 "Cookie: laijkpk3422r j1noin \r\n"
30 const SBuf canary(
"This text should not be changed.");
44 CPPUNIT_ASSERT(!t.
prefix(s, all, 0));
46 CPPUNIT_ASSERT_EQUAL(canary, s);
48 CPPUNIT_ASSERT_EQUAL(before, t.
remaining());
52 CPPUNIT_ASSERT_EQUAL(
SBuf(
"GET"),s);
54 CPPUNIT_ASSERT_EQUAL(
SBuf(
" "),s);
58 CPPUNIT_ASSERT_EQUAL(
SBuf(
" "),s);
62 CPPUNIT_ASSERT_EQUAL(
SBuf(
"http"),s);
66 CPPUNIT_ASSERT_EQUAL(
SBuf(
"http"),s);
69 CPPUNIT_ASSERT(t.
prefix(s,all));
74 CPPUNIT_ASSERT(!t.
prefix(s, all));
76 CPPUNIT_ASSERT_EQUAL(canary, s);
88 CPPUNIT_ASSERT_EQUAL(
SBuf(
"GET"),s);
94 CPPUNIT_ASSERT_EQUAL(
SBuf(
"http"),s);
100 CPPUNIT_ASSERT_EQUAL(
SBuf(
"resource"),s);
104 CPPUNIT_ASSERT(!t.
skip(
SBuf(
"://")));
105 CPPUNIT_ASSERT(!t.
skip(
'a'));
108 CPPUNIT_ASSERT(t.
skip(
'.'));
121 CPPUNIT_ASSERT_EQUAL(
SBuf(
"GET"),s);
123 CPPUNIT_ASSERT_EQUAL(
SBuf(
"http://resource.com/path"),s);
125 CPPUNIT_ASSERT_EQUAL(
SBuf(
"HTTP/1.1"),s);
127 CPPUNIT_ASSERT_EQUAL(
SBuf(
"Host:"),s);
134 const SBuf canary(
"This text should not be changed.");
148 CPPUNIT_ASSERT(!t.
suffix(s, all, 0));
150 CPPUNIT_ASSERT_EQUAL(canary, s);
152 CPPUNIT_ASSERT_EQUAL(before, t.
remaining());
170 CPPUNIT_ASSERT_EQUAL(
SBuf(
"1"), s);
174 CPPUNIT_ASSERT_EQUAL(
SBuf(
"HTTP"), s);
176 CPPUNIT_ASSERT_EQUAL(
SBuf(
" "), s);
179 CPPUNIT_ASSERT(t.
suffix(s, all));
184 CPPUNIT_ASSERT(!t.
suffix(s, all));
185 CPPUNIT_ASSERT_EQUAL(canary, s);
204 const int64_t benchmark = 1234;
205 CPPUNIT_ASSERT(t.
int64(rv, 10));
206 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
214 const int64_t benchmark = 1234;
215 CPPUNIT_ASSERT(t.
int64(rv));
216 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
224 const int64_t benchmark = 01234;
225 CPPUNIT_ASSERT(t.
int64(rv));
226 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
234 const int64_t benchmark = 0x12f4;
235 CPPUNIT_ASSERT(t.
int64(rv));
236 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
244 CPPUNIT_ASSERT(!t.
int64(rv));
245 CPPUNIT_ASSERT_EQUAL(
SBuf(
" 1234"), t.
buf());
252 CPPUNIT_ASSERT(!t.
int64(rv));
253 CPPUNIT_ASSERT_EQUAL(
SBuf(
" 1234"), t.
buf());
260 const int64_t benchmark = 1234;
261 CPPUNIT_ASSERT(t.
int64(rv));
262 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
263 CPPUNIT_ASSERT_EQUAL(
SBuf(
" foo"), t.
buf());
270 const int64_t benchmark = 1234;
271 CPPUNIT_ASSERT(t.
int64(rv));
272 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
273 CPPUNIT_ASSERT_EQUAL(
SBuf(
"foo"), t.
buf());
280 const int64_t benchmark = 0x1234f;
281 CPPUNIT_ASSERT(t.
int64(rv));
282 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
283 CPPUNIT_ASSERT_EQUAL(
SBuf(
"oo"), t.
buf());
290 CPPUNIT_ASSERT(!t.
int64(rv));
291 CPPUNIT_ASSERT_EQUAL(
SBuf(
"1029397752385698678762234"), t.
buf());
297 SBuf base(
"1029397752385698678762234");
298 const int64_t benchmark = 22;
300 CPPUNIT_ASSERT_EQUAL(
SBuf(
"22"),t.
buf());
301 CPPUNIT_ASSERT(t.
int64(rv));
302 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
309 SBuf base(
"deadbeefrow");
310 const int64_t benchmark=0xdeadbeef;
312 CPPUNIT_ASSERT(t.
int64(rv,16));
313 CPPUNIT_ASSERT_EQUAL(benchmark,rv);
314 CPPUNIT_ASSERT_EQUAL(
SBuf(
"row"),t.
buf());
optimized set of C chars, with quick membership test and merge support
CharacterSet & add(const unsigned char c)
add a given character to the character set
bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit=SBuf::npos)
bool skipOne(const CharacterSet &discardables)
bool token(SBuf &returnedToken, const CharacterSet &delimiters)
bool skipSuffix(const SBuf &tokenToSkip)
SBuf::size_type skipAll(const CharacterSet &discardables)
bool int64(int64_t &result, int base=0, bool allowSign=true, SBuf::size_type limit=SBuf::npos)
SBuf buf() const
yet unparsed data
const SBuf & remaining() const
the remaining unprocessed section of buffer
bool skipOneTrailing(const CharacterSet &discardables)
bool skip(const SBuf &tokenToSkip)
static const size_type npos
size_type length() const
Returns the number of bytes stored in SBuf.
size_type findLastOf(const CharacterSet &set, size_type endPos=npos) const
bool startsWith(const SBuf &S, const SBufCaseSensitive isCaseSensitive=caseSensitive) const
SBuf substr(size_type pos, size_type n=npos) const
MemBlob::size_type size_type
void testTokenizerToken()
void testTokenizerInt64()
void testTokenizerSuffix()
void testTokenizerPrefix()
const CharacterSet tab("tab","\t")
SBuf text("GET http://resource.com/path HTTP/1.1\r\n" "Host: resource.com\r\n" "Cookie: laijkpk3422r j1noin \r\n" "\r\n")
const CharacterSet alpha("alpha","abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
const CharacterSet crlf("crlf","\r\n")
CPPUNIT_TEST_SUITE_REGISTRATION(testTokenizer)
const CharacterSet whitespace("whitespace"," \r\n")
const CharacterSet numbers("numbers","0123456789")