SwapMetaIn.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9#include "squid.h"
10#include "base/Raw.h"
11#include "base/TextException.h"
12#include "int.h"
13#include "md5.h"
14#include "MemObject.h"
15#include "sbuf/SBuf.h"
16#include "sbuf/Stream.h"
17#include "SquidMath.h"
18#include "Store.h"
19#include "store/SwapMeta.h"
20#include "store/SwapMetaIn.h"
21#include "store/SwapMetaView.h"
22
23namespace Store {
24
27{
28public:
29 /* some of the standard iterator traits */
30 using iterator_category = std::forward_iterator_tag;
31 using value_type = const SwapMetaView;
34
36 SwapMetaIterator(const void *start, const void *end);
37
38 /* some of the standard iterator methods */
39 reference operator *() const { return meta_; }
40 pointer operator ->() const { return &meta_; }
42 bool operator ==(const SwapMetaIterator &them) const { return fieldStart_ == them.fieldStart_; }
43 bool operator !=(const SwapMetaIterator &them) const { return !(*this == them); }
44
45private:
46 void sync();
47
48 const char *fieldStart_;
49 const void * const bufEnd_;
51};
52
55{
56public:
57 SwapMetaUnpacker(const char *buf, ssize_t bufferLength, size_t &swap_hdr_sz);
58
59 // for-range loop API for iterating over serialized swap metadata fields
61 Iterator cbegin() const { return Iterator(metas, metas + metasSize); }
63 Iterator begin() const { return cbegin(); }
64 Iterator end() const { return cend(); }
65
66private:
67 const char *metas;
68 size_t metasSize;
69};
70
72static void
73CheckSwapMetaKey(const SwapMetaView &meta, const StoreEntry &entry)
74{
77
78 if (!EBIT_TEST(entry.flags, KEY_PRIVATE) &&
79 memcmp(meta.rawValue, entry.key, SQUID_MD5_DIGEST_LENGTH) != 0) {
80
81 debugs(20, 2, "stored key mismatches " << entry.getMD5Text());
82
83 static unsigned int md5_mismatches = 0;
84 if (isPowTen(++md5_mismatches))
85 debugs(20, DBG_IMPORTANT, "WARNING: " << md5_mismatches << " swapin MD5 mismatches");
86
87 // TODO: Support TextException::frequent = isPowTen(++md5_mismatches)
88 // to suppress reporting, achieving the same effect as above
89 throw TextException("swap meta MD5 mismatch", Here());
90 }
91}
92
94static void
96{
99 Assure(key);
100 memcpy(key, meta.rawValue, SQUID_MD5_DIGEST_LENGTH);
101}
102
104static void
105CheckSwapMetaUrl(const SwapMetaView &meta, const StoreEntry &entry)
106{
107 Assure(meta.type == STORE_META_URL);
108
109 // PackSwapMeta() terminates; strcasecmp() and reporting below rely on that
110 if (!memrchr(meta.rawValue, '\0', meta.rawLength))
111 throw TextException("unterminated URI or bad URI length", Here());
112
113 const auto &emem = entry.mem();
114
115 if (!emem.hasUris())
116 return; // cannot validate
117
118 const auto storedUrl = static_cast<const char *>(meta.rawValue);
119 // XXX: ensure all Squid URL inputs are properly normalized then use case-sensitive compare here
120 if (strcasecmp(emem.urlXXX(), storedUrl) != 0) {
121 debugs(20, DBG_IMPORTANT, "WARNING: URL mismatch when loading a cached entry:" <<
122 Debug::Extra << "expected: " << emem.urlXXX() <<
123 Debug::Extra << "found: " << storedUrl);
124 throw TextException("URL mismatch", Here());
125 }
126}
127
129static SBuf
131{
133 SBuf rawVary(static_cast<const char *>(meta.rawValue), meta.rawLength);
134 // entries created before SBuf-based Vary may include string terminator
135 static const SBuf nul("\0", 1);
136 rawVary.trim(nul, false, true);
137
138 const auto &knownVary = entry.mem().vary_headers;
139 if (knownVary.isEmpty())
140 return rawVary; // new Vary (that we cannot validate)
141
142 if (knownVary == rawVary)
143 return SBuf(); // OK: no new Vary
144
145 throw TextException("Vary mismatch", Here());
146}
147
150static size_t
151UnpackPrefix(const char * const buf, const size_t size)
152{
153 Assure(buf);
154 auto input = buf;
155 const auto end = buf + size;
156
157 char magic = 0;
158 SwapMetaExtract(magic, input, end);
159
160 if (magic != SwapMetaMagic)
161 throw TextException("store entry metadata prefix is corrupted", Here());
162
163 RawSwapMetaPrefixLength rawMetaSize = 0; // metadata size, including the required prefix
164 SwapMetaExtract(rawMetaSize, input, end);
165
166 if (Less(rawMetaSize, SwapMetaPrefixSize))
167 throw TextException("store entry metadata length is corrupted", Here());
168
169 return rawMetaSize; // now safe to use within (buf, buf+size)
170}
171
172} // namespace Store
173
174/* Store::SwapMetaIterator */
175
176Store::SwapMetaIterator::SwapMetaIterator(const void * const start, const void * const end):
177 fieldStart_(static_cast<const char*>(start)),
178 bufEnd_(end)
179{
180 sync();
181}
182
185{
186 Assure(fieldStart_ != bufEnd_);
187 fieldStart_ += sizeof(RawSwapMetaType); // swap meta type
188 fieldStart_ += sizeof(RawSwapMetaLength); // swap meta value length
189 fieldStart_ += meta_.rawLength; // swap meta value
190
191 sync();
192 return *this;
193}
194
196void
198{
199 if (fieldStart_ == bufEnd_)
200 return; // nothing to do when we reach the end of iteration
201
202 // We cannot start beyond the end of the header: We start with valid
203 // begin/end buffer pointers, and each field checks for overreach.
204 Assure(fieldStart_ < bufEnd_);
205
206 meta_ = SwapMetaView(fieldStart_, bufEnd_);
207}
208
209/* Store::SwapMetaUnpacker */
210
211Store::SwapMetaUnpacker::SwapMetaUnpacker(const char * const buf, const ssize_t size, size_t &swap_hdr_sz)
212{
213 Assure(buf);
214 Assure(size >= 0);
215
216 const auto headerSize = UnpackPrefix(buf, size);
217
218 // We assume the caller supplied a reasonable-size buffer. If our assumption
219 // is wrong, then this is a Squid bug rather than input validation failure.
220 if (Less(size, headerSize)) {
221 throw TextException(ToSBuf("store entry metadata is too big",
222 Debug::Extra, "buffer size: ", size,
223 Debug::Extra, "metadata size: ", headerSize),
224 Here());
225 }
226
227 Assure2(headerSize >= SwapMetaPrefixSize, "UnpackPrefix() validates metadata length");
228 metasSize = headerSize - SwapMetaPrefixSize;
229
230 metas = buf + SwapMetaPrefixSize; // skip prefix
231 Assure(metas + metasSize <= buf + size); // paranoid
232
233 swap_hdr_sz = headerSize;
234}
235
236size_t
238{
239 return UnpackPrefix(buf.rawContent(), buf.length());
240}
241
242size_t
244{
245 size_t swap_hdr_sz = 0;
246
247 const SwapMetaUnpacker metaFields(buf.content(), buf.contentSize(), swap_hdr_sz);
248 for (const auto &meta: metaFields) {
249 switch (meta.type) {
250 case STORE_META_VOID:
251 // this meta.type is the unpacking code signal that it took care of this field
252 break;
253
255 // Optimization: We could postpone setting the caller's key
256 // until all fields are parsed, but that would require copying
257 // it. Instead, we treat key and tmpe.key as storage that can be
258 // safely altered even on parsing failures. This function
259 // description tells the callers that we may do that.
260 UnpackSwapMetaKey(meta, key);
261 Assure(key);
262 tmpe.key = key;
263 break;
264
265 case STORE_META_STD: {
266 // TODO: Remove. Since old_metahdr's members may have different
267 // sizes on different platforms, we cannot guarantee that serialized
268 // types in the being-loaded old cache are the same as these types.
269 meta.checkExpectedLength(STORE_HDR_METASIZE_OLD);
270 struct old_metahdr {
271 // XXX: All serialized members must have fixed-size types.
272 time_t timestamp;
273 time_t lastref;
274 time_t expires;
275 time_t lastmod;
276 size_t swap_file_sz;
277 uint16_t refcount;
278 uint16_t flags;
279 };
280 static_assert(offsetof(old_metahdr, flags) + sizeof(old_metahdr::flags) == STORE_HDR_METASIZE_OLD, "we reproduced old swap meta basics format");
281 auto basics = static_cast<const old_metahdr*>(meta.rawValue);
282 tmpe.timestamp = basics->timestamp;
283 tmpe.lastref = basics->lastref;
284 tmpe.expires = basics->expires;
285 tmpe.lastModified(basics->lastmod);
286 tmpe.swap_file_sz = basics->swap_file_sz;
287 tmpe.refcount = basics->refcount;
288 tmpe.flags = basics->flags;
289 break;
290 }
291
293 meta.checkExpectedLength(STORE_HDR_METASIZE);
294 memcpy(&tmpe.timestamp, meta.rawValue, STORE_HDR_METASIZE);
295 break;
296
297 case STORE_META_URL:
300 // We do not load this information at cache index rebuild time;
301 // UnpackHitSwapMeta() handles these MemObject fields.
302 break;
303 }
304 }
305
306 return swap_hdr_sz;
307}
308
309void
310Store::UnpackHitSwapMeta(char const * const buf, const ssize_t len, StoreEntry &entry)
311{
312 debugs(90, 7, entry << " buf len: " << len);
313 assert(len >= 0);
314
315 size_t swap_hdr_sz = 0;
316 SBuf varyHeaders;
317
318 const SwapMetaUnpacker metaFields(buf, len, swap_hdr_sz);
319 for (const auto &meta: metaFields) {
320 switch (meta.type) {
321 case STORE_META_VOID:
322 // this meta.type is the unpacking code signal that it took care of the field
323 break;
324
325 case STORE_META_URL:
326 CheckSwapMetaUrl(meta, entry);
327 break;
328
330 varyHeaders = UnpackNewSwapMetaVaryHeaders(meta, entry);
331 break;
332
334 // XXX: We swap out but never use this field; set emem.object_sz?
335 break;
336
338 // already handled by UnpackIndexSwapMeta()
339 CheckSwapMetaKey(meta, entry); // paranoid
340 break;
341
342 case STORE_META_STD:
344 // already handled by UnpackIndexSwapMeta()
345 break;
346 }
347 }
348
349 auto &emem = entry.mem();
350
351 emem.swap_hdr_sz = swap_hdr_sz;
352 if (entry.swap_file_sz > 0) { // collapsed hits may not know swap_file_sz
353 Assure(entry.swap_file_sz >= swap_hdr_sz);
354 emem.object_sz = entry.swap_file_sz - swap_hdr_sz;
355 }
356 debugs(90, 5, "swap_file_sz=" << entry.swap_file_sz <<
357 " (" << swap_hdr_sz << " + " << emem.object_sz << ")");
358
359 if (!varyHeaders.isEmpty())
360 emem.vary_headers = varyHeaders;
361}
362
#define Assure(condition)
Definition: Assure.h:35
#define Assure2(condition, description)
Definition: Assure.h:40
#define Here()
source code location of the caller
Definition: Here.h:15
int size
Definition: ModDevPoll.cc:75
constexpr bool Less(const A a, const B b)
whether integer a is less than integer b, with correct overflow handling
Definition: SquidMath.h:53
#define assert(EX)
Definition: assert.h:19
static std::ostream & Extra(std::ostream &os)
prefixes each grouped debugs() line after the first one in the group
Definition: Stream.h:117
Definition: MemBuf.h:24
char * content()
start of the added data
Definition: MemBuf.h:41
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
SBuf vary_headers
Definition: MemObject.h:204
size_t swap_hdr_sz
Definition: MemObject.h:199
Definition: SBuf.h:94
const char * rawContent() const
Definition: SBuf.cc:509
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:415
SBuf & trim(const SBuf &toRemove, bool atBeginning=true, bool atEnd=true)
Definition: SBuf.cc:551
bool isEmpty() const
Definition: SBuf.h:431
uint16_t flags
Definition: Store.h:232
MemObject & mem()
Definition: Store.h:51
time_t expires
Definition: Store.h:226
void lastModified(const time_t when)
Definition: Store.h:176
time_t timestamp
Definition: Store.h:224
const char * getMD5Text() const
Definition: store.cc:204
time_t lastref
Definition: Store.h:225
uint64_t swap_file_sz
Definition: Store.h:230
uint16_t refcount
Definition: Store.h:231
iterates serialized swap meta fields loaded into a given buffer
Definition: SwapMetaIn.cc:27
reference operator*() const
Definition: SwapMetaIn.cc:39
SwapMetaView meta_
current field; valid after sync() and before end
Definition: SwapMetaIn.cc:50
const char * fieldStart_
the start of the current field
Definition: SwapMetaIn.cc:48
void sync()
(re)set meta_
Definition: SwapMetaIn.cc:197
const void *const bufEnd_
last field must end at this boundary
Definition: SwapMetaIn.cc:49
bool operator==(const SwapMetaIterator &them) const
Definition: SwapMetaIn.cc:42
SwapMetaIterator & operator++()
Definition: SwapMetaIn.cc:184
SwapMetaIterator(const void *start, const void *end)
positions iterator at the start of a swap meta field extending up to end
Definition: SwapMetaIn.cc:176
std::forward_iterator_tag iterator_category
Definition: SwapMetaIn.cc:30
bool operator!=(const SwapMetaIterator &them) const
Definition: SwapMetaIn.cc:43
pointer operator->() const
Definition: SwapMetaIn.cc:40
Store entry metadata view providing a for-range loop meta field iterator API.
Definition: SwapMetaIn.cc:55
Iterator begin() const
Definition: SwapMetaIn.cc:63
SwapMetaIterator Iterator
Definition: SwapMetaIn.cc:60
SwapMetaUnpacker(const char *buf, ssize_t bufferLength, size_t &swap_hdr_sz)
Definition: SwapMetaIn.cc:211
Iterator cbegin() const
Definition: SwapMetaIn.cc:61
size_t metasSize
number of bytes in the metas buffer
Definition: SwapMetaIn.cc:68
Iterator cend() const
Definition: SwapMetaIn.cc:62
Iterator end() const
Definition: SwapMetaIn.cc:64
const char * metas
metadata field(s)
Definition: SwapMetaIn.cc:67
a swap metadata field inside the buffer given to SwapMetaUnpacker
Definition: SwapMetaView.h:20
void checkExpectedLength(size_t) const
ensures that our fixed-size field value has the given expected length
Definition: SwapMetaView.cc:83
SwapMetaType type
Definition: SwapMetaView.h:42
const void * rawValue
Definition: SwapMetaView.h:29
an std::runtime_error with thrower location info
Definition: TextException.h:21
#define DBG_IMPORTANT
Definition: Stream.h:41
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
@ KEY_PRIVATE
Definition: enums.h:102
int isPowTen(int count)
Definition: int.cc:17
#define SQUID_MD5_DIGEST_LENGTH
Definition: md5.h:66
void * memrchr(const void *s, int c, size_t n)
Definition: memrchr.cc:36
const auto SwapMetaPrefixSize
The size of the initial (and required) portion of any swap metadata.
Definition: SwapMeta.h:122
void SwapMetaExtract(Item &item, const char *&input, const void *end)
Definition: SwapMetaView.h:67
static void UnpackSwapMetaKey(const SwapMetaView &meta, cache_key *key)
deserializes STORE_META_KEY_MD5 swap meta field
Definition: SwapMetaIn.cc:95
int RawSwapMetaLength
Definition: SwapMeta.h:100
char RawSwapMetaType
Definition: SwapMeta.h:95
size_t UnpackSwapMetaSize(const SBuf &)
Definition: SwapMetaIn.cc:237
void UnpackHitSwapMeta(char const *, ssize_t, StoreEntry &)
deserializes entry metadata from the given buffer into the cache hit entry
Definition: SwapMetaIn.cc:310
size_t UnpackIndexSwapMeta(const MemBuf &, StoreEntry &, cache_key *)
Definition: SwapMetaIn.cc:243
@ STORE_META_URL
Definition: SwapMeta.h:65
@ STORE_META_VARY_HEADERS
Stores Vary request headers.
Definition: SwapMeta.h:83
@ STORE_META_OBJSIZE
Definition: SwapMeta.h:90
@ STORE_META_KEY_MD5
Definition: SwapMeta.h:61
@ STORE_META_STD
Definition: SwapMeta.h:80
@ STORE_META_STD_LFS
Definition: SwapMeta.h:87
@ STORE_META_VOID
Definition: SwapMeta.h:55
static void CheckSwapMetaKey(const SwapMetaView &meta, const StoreEntry &entry)
validates serialized STORE_META_KEY_MD5 swap meta field
Definition: SwapMetaIn.cc:73
static SBuf UnpackNewSwapMetaVaryHeaders(const SwapMetaView &meta, const StoreEntry &entry)
deserializes STORE_META_VARY_HEADERS swap meta field
Definition: SwapMetaIn.cc:130
const char SwapMetaMagic
the start of the swap meta section
Definition: SwapMeta.h:113
const auto STORE_HDR_METASIZE_OLD
Definition: SwapMeta.h:233
static size_t UnpackPrefix(const char *const buf, const size_t size)
Definition: SwapMetaIn.cc:151
const auto STORE_HDR_METASIZE
Definition: SwapMeta.h:228
static void CheckSwapMetaUrl(const SwapMetaView &meta, const StoreEntry &entry)
validates serialized STORE_META_URL swap meta field
Definition: SwapMetaIn.cc:105
int RawSwapMetaPrefixLength
Definition: SwapMeta.h:119
SBuf ToSBuf(Args &&... args)
slowly stream-prints all arguments into a freshly allocated SBuf
Definition: Stream.h:63
unsigned char cache_key
Store key.
Definition: forward.h:29

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors