Downloader.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9#include "squid.h"
10#include "base/Raw.h"
11#include "client_side.h"
12#include "client_side_reply.h"
13#include "client_side_request.h"
15#include "Downloader.h"
16#include "fatal.h"
18#include "http/Stream.h"
19
21
25{
27
28public:
30
32 ~DownloaderContext() override;
33 void finished();
34
38};
39
41 downloader(dl),
42 http(h)
43{
44 debugs(33, 6, "DownloaderContext constructed, this=" << (void*)this);
45}
46
48{
49 debugs(33, 6, "DownloaderContext destructed, this=" << (void*)this);
50 if (http)
51 finished();
52}
53
54void
56{
57 delete http;
58 http = nullptr;
59}
60
61std::ostream &
62operator <<(std::ostream &os, const DownloaderAnswer &answer)
63{
64 os << "outcome=" << answer.outcome;
65 if (answer.outcome == Http::scOkay)
66 os << ", resource.size=" << answer.resource.length();
67 return os;
68}
69
70Downloader::Downloader(const SBuf &url, const AsyncCallback<Answer> &cb, const MasterXactionPointer &mx, const unsigned int level):
71 AsyncJob("Downloader"),
72 url_(url),
73 callback_(cb),
74 level_(level),
75 masterXaction_(mx)
76{
77}
78
80{
81 debugs(33, 6, this);
82}
83
84void
86{
87 debugs(33, 6, this);
88
89 if (callback_) // job-ending emergencies like handleStopRequest() or callException()
91
92 if (context_) {
94 context_ = nullptr;
95 }
96}
97
98bool
100{
101 return (!callback_ || callback_->canceled()) && AsyncJob::doneAll();
102}
103
104static void
106 HttpReply * rep, StoreIOBuffer receivedData)
107{
108 debugs(33, 6, MYNAME);
109 /* Test preconditions */
110 assert(node);
111
112 /* TODO: handle this rather than asserting
113 * - it should only ever happen if we cause an abort and
114 * the callback chain loops back to here, so we can simply return.
115 * However, that itself shouldn't happen, so it stays as an assert for now.
116 */
118 assert(!node->node.next);
119 DownloaderContext::Pointer context = dynamic_cast<DownloaderContext *>(node->data.getRaw());
120 assert(context);
121
122 if (context->downloader.valid())
123 context->downloader->handleReply(node, http, rep, receivedData);
124}
125
126static void
128{
129 debugs(33, 5, MYNAME);
131}
132
134bool
136{
137 const HttpRequestMethod method = Http::METHOD_GET;
138
139 const auto request = HttpRequest::FromUrl(url_, masterXaction_, method);
140 if (!request) {
141 debugs(33, 5, "Invalid URI: " << url_);
142 return false; //earlyError(...)
143 }
144 request->http_ver = Http::ProtocolVersion();
145 request->header.putStr(Http::HdrType::HOST, request->url.host());
146 request->header.putTime(Http::HdrType::DATE, squid_curtime);
147 request->client_addr.setNoAddr();
148#if FOLLOW_X_FORWARDED_FOR
149 request->indirect_client_addr.setNoAddr();
150#endif /* FOLLOW_X_FORWARDED_FOR */
151 request->my_addr.setNoAddr(); /* undefined for internal requests */
152 request->my_addr.port(0);
153 request->downloader = this;
154
155 debugs(11, 2, "HTTP Client Downloader " << this << "/" << id);
156 debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
157 request->method << " " << url_ << " " << request->http_ver << "\n" <<
158 "\n----------");
159
160 ClientHttpRequest *const http = new ClientHttpRequest(nullptr);
161 http->initRequest(request);
162 http->req_sz = 0;
163 // XXX: performance regression. c_str() reallocates
164 http->uri = xstrdup(url_.c_str());
165
166 context_ = new DownloaderContext(this, http);
167 StoreIOBuffer tempBuffer;
168 tempBuffer.data = context_->requestBuffer;
169 tempBuffer.length = HTTP_REQBUF_SZ;
170
171 ClientStreamData newServer = new clientReplyContext(http);
172 ClientStreamData newClient = context_.getRaw();
175 downloaderDetach, newClient, tempBuffer);
176
177 // Build a ClientRequestContext to start doCallouts
178 http->calloutContext = new ClientRequestContext(http);
179 http->doCallouts();
180 return true;
181}
182
183void
185{
186 if (!buildRequest())
188}
189
190void
192{
193 DownloaderContext::Pointer callerContext = dynamic_cast<DownloaderContext *>(node->data.getRaw());
194 // TODO: remove the following check:
195 assert(callerContext == context_);
196
197 debugs(33, 4, "Received " << receivedData.length <<
198 " object data, offset: " << receivedData.offset <<
199 " error flag:" << receivedData.flags.error);
200
201 const bool failed = receivedData.flags.error;
202 if (failed) {
204 return;
205 }
206
207 const int64_t existingContent = reply ? reply->content_length : 0;
208 const size_t maxSize = MaxObjectSize > SBuf::maxSize ? SBuf::maxSize : MaxObjectSize;
209 const bool tooLarge = (existingContent > -1 && existingContent > static_cast<int64_t>(maxSize)) ||
210 (maxSize < object_.length()) ||
211 ((maxSize - object_.length()) < receivedData.length);
212
213 if (tooLarge) {
215 return;
216 }
217
218 object_.append(receivedData.data, receivedData.length);
219 http->out.size += receivedData.length;
220 http->out.offset += receivedData.length;
221
222 switch (clientStreamStatus(node, http)) {
223 case STREAM_NONE: {
224 debugs(33, 3, "Get more data");
225 StoreIOBuffer tempBuffer;
226 tempBuffer.offset = http->out.offset;
227 tempBuffer.data = context_->requestBuffer;
228 tempBuffer.length = HTTP_REQBUF_SZ;
229 clientStreamRead(node, http, tempBuffer);
230 }
231 break;
232 case STREAM_COMPLETE:
233 debugs(33, 3, "Object data transfer successfully complete");
235 break;
237 debugs(33, 3, "Object data transfer failed: STREAM_UNPLANNED_COMPLETE");
239 break;
240 case STREAM_FAILED:
241 debugs(33, 3, "Object data transfer failed: STREAM_FAILED");
243 break;
244 default:
245 fatal("unreachable code");
246 }
247}
248
249void
251{
252 debugs(33, 7, this);
253 Must(done());
254}
255
258void
260{
262 auto &answer = callback_.answer();
263 answer.outcome = statusCode;
264 if (statusCode == Http::scOkay)
265 answer.resource = object_;
266 ScheduleCallHere(callback_.release());
267
268 // We cannot deleteThis() because we may be called synchronously from
269 // doCallouts() via handleReply() (XXX), and doCallouts() may crash if we
270 // disappear. Instead, schedule an async call now so that later, when the
271 // call firing code discovers a done() job, it deletes us.
273}
274
#define ScheduleCallHere(call)
Definition: AsyncCall.h:165
#define CallJobHere(debugSection, debugLevel, job, Class, method)
Definition: AsyncJobCalls.h:58
static void downloaderDetach(clientStreamNode *node, ClientHttpRequest *http)
Definition: Downloader.cc:127
std::ostream & operator<<(std::ostream &os, const DownloaderAnswer &answer)
Definition: Downloader.cc:62
static void downloaderRecipient(clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
Definition: Downloader.cc:105
#define RefCountable
The locking interface for use on Reference-Counted classes.
Definition: Lock.h:66
time_t squid_curtime
Definition: stub_libtime.cc:20
#define Must(condition)
Definition: TextException.h:71
#define assert(EX)
Definition: assert.h:17
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:265
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:320
a smart AsyncCall pointer for delivery of future results
virtual bool doneAll() const
whether positive goal has been reached
Definition: AsyncJob.cc:97
bool done() const
the job is destroyed in callEnd() when done()
Definition: AsyncJob.cc:91
Cbc * valid() const
was set and is valid
Definition: CbcPointer.h:41
struct ClientHttpRequest::Out out
void initRequest(HttpRequest *)
size_t req_sz
raw request size on input, not current request size
ClientRequestContext * calloutContext
download result
Definition: Downloader.h:28
Http::StatusCode outcome
Definition: Downloader.h:36
MEMPROXY_CLASS(DownloaderContext)
DownloaderContext(Downloader *dl, ClientHttpRequest *h)
Definition: Downloader.cc:40
char requestBuffer[HTTP_REQBUF_SZ]
Definition: Downloader.cc:37
~DownloaderContext() override
Definition: Downloader.cc:47
ClientHttpRequest * http
Definition: Downloader.cc:36
CbcPointer< Downloader > downloader
Definition: Downloader.cc:35
RefCount< DownloaderContext > Pointer
Definition: Downloader.cc:29
void callBack(Http::StatusCode const status)
Definition: Downloader.cc:259
AsyncCallback< Answer > callback_
answer destination
Definition: Downloader.h:80
~Downloader() override
Definition: Downloader.cc:79
void downloadFinished()
delays destruction to protect doCallouts()
Definition: Downloader.cc:250
bool doneAll() const override
whether positive goal has been reached
Definition: Downloader.cc:99
void start() override
called by AsyncStart; do not call directly
Definition: Downloader.cc:184
SBuf url_
the url to download
Definition: Downloader.h:77
static const size_t MaxObjectSize
The maximum allowed object size.
Definition: Downloader.h:75
Downloader(const SBuf &url, const AsyncCallback< Answer > &, const MasterXactionPointer &, unsigned int level=0)
Definition: Downloader.cc:70
DownloaderContextPointer context_
Pointer to an object that stores the clientStream required info.
Definition: Downloader.h:87
MasterXactionPointer masterXaction_
download transaction context
Definition: Downloader.h:84
bool buildRequest()
Initializes and starts the HTTP GET request to the remote server.
Definition: Downloader.cc:135
SBuf object_
the object body data
Definition: Downloader.h:82
void swanSong() override
Definition: Downloader.cc:85
void handleReply(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
Definition: Downloader.cc:191
static HttpRequest * FromUrl(const SBuf &url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:517
int64_t content_length
Definition: Message.h:83
C * getRaw() const
Definition: RefCount.h:80
Definition: SBuf.h:94
const char * c_str()
Definition: SBuf.cc:516
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:415
SBuf & append(const SBuf &S)
Definition: SBuf.cc:185
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be < MAX(size_type)/2. Currently 256Mb.
Definition: SBuf.h:102
unsigned error
Definition: StoreIOBuffer.h:52
int64_t offset
Definition: StoreIOBuffer.h:55
struct StoreIOBuffer::@142 flags
SQUIDCEXTERN CSR clientGetMoreData
Definition: client_side.h:528
SQUIDCEXTERN CSS clientReplyStatus
Definition: client_side.h:529
SQUIDCEXTERN CSD clientReplyDetach
Definition: client_side.h:530
#define MYNAME
Definition: Stream.h:235
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:193
@ STREAM_COMPLETE
Definition: enums.h:127
@ STREAM_UNPLANNED_COMPLETE
Definition: enums.h:132
@ STREAM_NONE
Definition: enums.h:126
@ STREAM_FAILED
Definition: enums.h:137
void fatal(const char *message)
Definition: fatal.cc:28
clientStream_status_t clientStreamStatus(clientStreamNode *thisObject, ClientHttpRequest *http)
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
void clientStreamInit(dlink_list *list, CSR *func, CSD *rdetach, CSS *readstatus, ClientStreamData readdata, CSCB *callback, CSD *cdetach, ClientStreamData callbackdata, StoreIOBuffer tailBuffer)
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
#define HTTP_REQBUF_SZ
Definition: forward.h:14
StatusCode
Definition: StatusCode.h:20
@ scInternalServerError
Definition: StatusCode.h:71
@ scOkay
Definition: StatusCode.h:26
@ METHOD_GET
Definition: MethodType.h:25
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
#define xstrdup
uint64_t size
Response header and body bytes written to the client connection.
Definition: parse.c:104
struct node * next
Definition: parse.c:105

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors