Downloader.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #include "squid.h"
10 #include "client_side.h"
11 #include "client_side_reply.h"
12 #include "client_side_request.h"
13 #include "ClientRequestContext.h"
14 #include "Downloader.h"
15 #include "fatal.h"
16 #include "http/one/RequestParser.h"
17 #include "http/Stream.h"
18 
20 
24 {
26 
27 public:
29 
32  void finished();
33 
37 };
38 
40  downloader(dl),
41  http(h)
42 {
43  debugs(33, 6, "DownloaderContext constructed, this=" << (void*)this);
44 }
45 
47 {
48  debugs(33, 6, "DownloaderContext destructed, this=" << (void*)this);
49  if (http)
50  finished();
51 }
52 
53 void
55 {
56  delete http;
57  http = nullptr;
58 }
59 
60 void
61 Downloader::CbDialer::print(std::ostream &os) const
62 {
63  os << " Http Status:" << status << Raw("body data", object.rawContent(), 64).hex();
64 }
65 
66 Downloader::Downloader(SBuf &url, AsyncCall::Pointer &aCallback, const XactionInitiator initiator, unsigned int level):
67  AsyncJob("Downloader"),
68  url_(url),
69  callback_(aCallback),
70  level_(level),
71  initiator_(initiator)
72 {
73 }
74 
76 {
77  debugs(33, 6, this);
78 }
79 
80 void
82 {
83  debugs(33, 6, this);
84 
85  if (callback_) // job-ending emergencies like handleStopRequest() or callException()
87 
88  if (context_) {
89  context_->finished();
90  context_ = nullptr;
91  }
92 }
93 
94 bool
96 {
97  return (!callback_ || callback_->canceled()) && AsyncJob::doneAll();
98 }
99 
100 static void
102  HttpReply * rep, StoreIOBuffer receivedData)
103 {
104  debugs(33, 6, MYNAME);
105  /* Test preconditions */
106  assert(node);
107 
108  /* TODO: handle this rather than asserting
109  * - it should only ever happen if we cause an abort and
110  * the callback chain loops back to here, so we can simply return.
111  * However, that itself shouldn't happen, so it stays as an assert for now.
112  */
114  assert(!node->node.next);
115  DownloaderContext::Pointer context = dynamic_cast<DownloaderContext *>(node->data.getRaw());
116  assert(context);
117 
118  if (context->downloader.valid())
119  context->downloader->handleReply(node, http, rep, receivedData);
120 }
121 
122 static void
124 {
125  debugs(33, 5, MYNAME);
126  clientStreamDetach(node, http);
127 }
128 
130 bool
132 {
133  const HttpRequestMethod method = Http::METHOD_GET;
134 
136  auto * const request = HttpRequest::FromUrl(url_, mx, method);
137  if (!request) {
138  debugs(33, 5, "Invalid URI: " << url_);
139  return false; //earlyError(...)
140  }
141  request->http_ver = Http::ProtocolVersion();
142  request->header.putStr(Http::HdrType::HOST, request->url.host());
143  request->header.putTime(Http::HdrType::DATE, squid_curtime);
144  request->client_addr.setNoAddr();
145 #if FOLLOW_X_FORWARDED_FOR
146  request->indirect_client_addr.setNoAddr();
147 #endif /* FOLLOW_X_FORWARDED_FOR */
148  request->my_addr.setNoAddr(); /* undefined for internal requests */
149  request->my_addr.port(0);
150  request->downloader = this;
151 
152  debugs(11, 2, "HTTP Client Downloader " << this << "/" << id);
153  debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
154  request->method << " " << url_ << " " << request->http_ver << "\n" <<
155  "\n----------");
156 
157  ClientHttpRequest *const http = new ClientHttpRequest(nullptr);
158  http->initRequest(request);
159  http->req_sz = 0;
160  // XXX: performance regression. c_str() reallocates
161  http->uri = xstrdup(url_.c_str());
162 
163  context_ = new DownloaderContext(this, http);
164  StoreIOBuffer tempBuffer;
165  tempBuffer.data = context_->requestBuffer;
166  tempBuffer.length = HTTP_REQBUF_SZ;
167 
168  ClientStreamData newServer = new clientReplyContext(http);
169  ClientStreamData newClient = context_.getRaw();
172  downloaderDetach, newClient, tempBuffer);
173 
174  // Build a ClientRequestContext to start doCallouts
175  http->calloutContext = new ClientRequestContext(http);
176  http->doCallouts();
177  return true;
178 }
179 
180 void
182 {
183  if (!buildRequest())
185 }
186 
187 void
189 {
190  DownloaderContext::Pointer callerContext = dynamic_cast<DownloaderContext *>(node->data.getRaw());
191  // TODO: remove the following check:
192  assert(callerContext == context_);
193 
194  debugs(33, 4, "Received " << receivedData.length <<
195  " object data, offset: " << receivedData.offset <<
196  " error flag:" << receivedData.flags.error);
197 
198  const bool failed = receivedData.flags.error;
199  if (failed) {
201  return;
202  }
203 
204  const int64_t existingContent = reply ? reply->content_length : 0;
205  const size_t maxSize = MaxObjectSize > SBuf::maxSize ? SBuf::maxSize : MaxObjectSize;
206  const bool tooLarge = (existingContent > -1 && existingContent > static_cast<int64_t>(maxSize)) ||
207  (maxSize < object_.length()) ||
208  ((maxSize - object_.length()) < receivedData.length);
209 
210  if (tooLarge) {
212  return;
213  }
214 
215  object_.append(receivedData.data, receivedData.length);
216  http->out.size += receivedData.length;
217  http->out.offset += receivedData.length;
218 
219  switch (clientStreamStatus(node, http)) {
220  case STREAM_NONE: {
221  debugs(33, 3, "Get more data");
222  StoreIOBuffer tempBuffer;
223  tempBuffer.offset = http->out.offset;
224  tempBuffer.data = context_->requestBuffer;
225  tempBuffer.length = HTTP_REQBUF_SZ;
226  clientStreamRead(node, http, tempBuffer);
227  }
228  break;
229  case STREAM_COMPLETE:
230  debugs(33, 3, "Object data transfer successfully complete");
232  break;
234  debugs(33, 3, "Object data transfer failed: STREAM_UNPLANNED_COMPLETE");
236  break;
237  case STREAM_FAILED:
238  debugs(33, 3, "Object data transfer failed: STREAM_FAILED");
240  break;
241  default:
242  fatal("unreachable code");
243  }
244 }
245 
246 void
248 {
249  debugs(33, 7, this);
250  Must(done());
251 }
252 
255 void
257 {
258  assert(callback_);
259  CbDialer *dialer = dynamic_cast<CbDialer*>(callback_->getDialer());
260  Must(dialer);
261  dialer->status = statusCode;
262  if (statusCode == Http::scOkay)
263  dialer->object = object_;
265  callback_ = nullptr;
266 
267  // We cannot deleteThis() because we may be called synchronously from
268  // doCallouts() via handleReply() (XXX), and doCallouts() may crash if we
269  // disappear. Instead, schedule an async call now so that later, when the
270  // call firing code discovers a done() job, it deletes us.
272 }
273 
const unsigned int level_
Definition: Downloader.h:78
Downloader(SBuf &url, AsyncCall::Pointer &aCallback, const XactionInitiator initiator, unsigned int level=0)
Definition: Downloader.cc:66
void fatal(const char *message)
Definition: fatal.cc:28
Definition: parse.c:104
char method[16]
Definition: tcp-banger2.c:115
#define MYNAME
Definition: Debug.h:165
SQUIDCEXTERN CSR clientGetMoreData
Definition: client_side.h:528
static HttpRequest * FromUrl(const SBuf &url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:518
clientStream_status_t clientStreamStatus(clientStreamNode *thisObject, ClientHttpRequest *http)
SQUIDCEXTERN CSS clientReplyStatus
Definition: client_side.h:529
Cbc * valid() const
was set and is valid
Definition: CbcPointer.h:41
virtual bool doneAll() const
whether positive goal has been reached
Definition: Downloader.cc:95
void initRequest(HttpRequest *)
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
struct node * next
Definition: parse.c:105
@ STREAM_NONE
Definition: enums.h:126
#define ScheduleCallHere(call)
Definition: AsyncCall.h:166
int64_t offset
Definition: StoreIOBuffer.h:55
ClientHttpRequest * http
Definition: Downloader.cc:35
Definition: SBuf.h:87
#define xstrdup
virtual CallDialer * getDialer()=0
struct ClientHttpRequest::Out out
C * getRaw() const
Definition: RefCount.h:80
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:398
struct StoreIOBuffer::@147 flags
@ STREAM_COMPLETE
Definition: enums.h:127
StatusCode
Definition: StatusCode.h:20
virtual void start()
called by AsyncStart; do not call directly
Definition: Downloader.cc:181
Http::StatusCode status
Definition: Downloader.h:46
identifies a protocol agent or Squid feature initiating transactions
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be < MAX(size_type)/2. Currently 256Mb.
Definition: SBuf.h:95
@ STREAM_FAILED
Definition: enums.h:137
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
char requestBuffer[HTTP_REQBUF_SZ]
Definition: Downloader.cc:36
void downloadFinished()
delays destruction to protect doCallouts()
Definition: Downloader.cc:247
Definition: Debug.h:184
virtual void print(std::ostream &os) const
Definition: Downloader.cc:61
CbcPointer< Downloader > downloader
Definition: Downloader.cc:34
virtual bool doneAll() const
whether positive goal has been reached
Definition: AsyncJob.cc:97
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
AsyncCall::Pointer callback_
callback to call when download finishes
Definition: Downloader.h:76
unsigned error
Definition: StoreIOBuffer.h:52
@ STREAM_UNPLANNED_COMPLETE
Definition: enums.h:132
virtual void swanSong()
Definition: Downloader.cc:81
#define CallJobHere(debugSection, debugLevel, job, Class, method)
Definition: AsyncJobCalls.h:58
#define assert(EX)
Definition: assert.h:19
virtual ~Downloader()
Definition: Downloader.cc:75
SBuf url_
the url to download
Definition: Downloader.h:75
static void downloaderDetach(clientStreamNode *node, ClientHttpRequest *http)
Definition: Downloader.cc:123
@ scInternalServerError
Definition: StatusCode.h:71
const char * c_str()
Definition: SBuf.cc:516
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
void handleReply(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
Definition: Downloader.cc:188
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:408
time_t squid_curtime
Definition: stub_time.cc:17
uint64_t size
Response header and body bytes written to the client connection.
SBuf & append(const SBuf &S)
Definition: SBuf.cc:185
bool done() const
the job is destroyed in callEnd() when done()
Definition: AsyncJob.cc:91
int64_t content_length
Definition: Message.h:84
void callBack(Http::StatusCode const status)
Definition: Downloader.cc:256
static void downloaderRecipient(clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
Definition: Downloader.cc:101
#define MEMPROXY_CLASS(CLASS)
char * url
Definition: tcp-banger2.c:114
bool buildRequest()
Initializes and starts the HTTP GET request to the remote server.
Definition: Downloader.cc:131
XactionInitiator initiator_
The initiator of the download request.
Definition: Downloader.h:80
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:73
Raw & hex()
print data using two hex digits per byte (decoder: xxd -r -p)
Definition: Debug.h:193
DownloaderContextPointer context_
Pointer to an object that stores the clientStream required info.
Definition: Downloader.h:83
SQUIDCEXTERN CSD clientReplyDetach
Definition: client_side.h:530
void clientStreamInit(dlink_list *list, CSR *func, CSD *rdetach, CSS *readstatus, ClientStreamData readdata, CSCB *callback, CSD *cdetach, ClientStreamData callbackdata, StoreIOBuffer tailBuffer)
static const size_t MaxObjectSize
The maximum allowed object size.
Definition: Downloader.h:73
#define HTTP_REQBUF_SZ
Definition: forward.h:14
ClientRequestContext * calloutContext
@ scOkay
Definition: StatusCode.h:26
@ METHOD_GET
Definition: MethodType.h:25
DownloaderContext(Downloader *dl, ClientHttpRequest *h)
Definition: Downloader.cc:39
Callback data to use with Downloader callbacks.
Definition: Downloader.h:35
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
RefCount< DownloaderContext > Pointer
Definition: Downloader.cc:25
SBuf object_
the object body data
Definition: Downloader.h:77
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
bool canceled()
Definition: AsyncCall.h:54

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors