Downloader.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #include "squid.h"
10 #include "client_side.h"
11 #include "client_side_reply.h"
12 #include "client_side_request.h"
13 #include "ClientRequestContext.h"
14 #include "Downloader.h"
15 #include "fatal.h"
16 #include "http/one/RequestParser.h"
17 #include "http/Stream.h"
18 
20 
24 {
26 
27 public:
29 
32  void finished();
33 
37 };
38 
40  downloader(dl),
41  http(h)
42 {
43  debugs(33, 6, "DownloaderContext constructed, this=" << (void*)this);
44 }
45 
47 {
48  debugs(33, 6, "DownloaderContext destructed, this=" << (void*)this);
49  if (http)
50  finished();
51 }
52 
53 void
55 {
56  delete http;
57  http = nullptr;
58 }
59 
60 void
61 Downloader::CbDialer::print(std::ostream &os) const
62 {
63  os << " Http Status:" << status << Raw("body data", object.rawContent(), 64).hex();
64 }
65 
66 Downloader::Downloader(SBuf &url, AsyncCall::Pointer &aCallback, const XactionInitiator initiator, unsigned int level):
67  AsyncJob("Downloader"),
68  url_(url),
69  callback_(aCallback),
70  level_(level),
71  initiator_(initiator)
72 {
73 }
74 
76 {
77  debugs(33, 6, this);
78 }
79 
80 void
82 {
83  debugs(33, 6, this);
84  if (context_) {
85  context_->finished();
86  context_ = nullptr;
87  }
88 }
89 
90 bool
92 {
93  return (!callback_ || callback_->canceled()) && AsyncJob::doneAll();
94 }
95 
96 static void
98  HttpReply * rep, StoreIOBuffer receivedData)
99 {
100  debugs(33, 6, MYNAME);
101  /* Test preconditions */
102  assert(node);
103 
104  /* TODO: handle this rather than asserting
105  * - it should only ever happen if we cause an abort and
106  * the callback chain loops back to here, so we can simply return.
107  * However, that itself shouldn't happen, so it stays as an assert for now.
108  */
110  assert(!node->node.next);
111  DownloaderContext::Pointer context = dynamic_cast<DownloaderContext *>(node->data.getRaw());
112  assert(context);
113 
114  if (context->downloader.valid())
115  context->downloader->handleReply(node, http, rep, receivedData);
116 }
117 
118 static void
120 {
121  debugs(33, 5, MYNAME);
122  clientStreamDetach(node, http);
123 }
124 
126 bool
128 {
130 
133  if (!request) {
134  debugs(33, 5, "Invalid URI: " << url_);
135  return false; //earlyError(...)
136  }
137  request->http_ver = Http::ProtocolVersion();
138  request->header.putStr(Http::HdrType::HOST, request->url.host());
140  request->client_addr.setNoAddr();
141 #if FOLLOW_X_FORWARDED_FOR
142  request->indirect_client_addr.setNoAddr();
143 #endif /* FOLLOW_X_FORWARDED_FOR */
144  request->my_addr.setNoAddr(); /* undefined for internal requests */
145  request->my_addr.port(0);
146  request->downloader = this;
147 
148  debugs(11, 2, "HTTP Client Downloader " << this << "/" << id);
149  debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
150  request->method << " " << url_ << " " << request->http_ver << "\n" <<
151  "\n----------");
152 
153  ClientHttpRequest *const http = new ClientHttpRequest(nullptr);
154  http->request = request;
155  HTTPMSGLOCK(http->request);
156  http->req_sz = 0;
157  // XXX: performance regression. c_str() reallocates
158  http->uri = xstrdup(url_.c_str());
159  setLogUri (http, urlCanonicalClean(request));
160 
161  context_ = new DownloaderContext(this, http);
162  StoreIOBuffer tempBuffer;
163  tempBuffer.data = context_->requestBuffer;
164  tempBuffer.length = HTTP_REQBUF_SZ;
165 
166  ClientStreamData newServer = new clientReplyContext(http);
167  ClientStreamData newClient = context_.getRaw();
170  downloaderDetach, newClient, tempBuffer);
171 
172  // Build a ClientRequestContext to start doCallouts
173  http->calloutContext = new ClientRequestContext(http);
174  http->doCallouts();
175  return true;
176 }
177 
178 void
180 {
181  if (!buildRequest())
183 }
184 
185 void
187 {
188  DownloaderContext::Pointer callerContext = dynamic_cast<DownloaderContext *>(node->data.getRaw());
189  // TODO: remove the following check:
190  assert(callerContext == context_);
191 
192  debugs(33, 4, "Received " << receivedData.length <<
193  " object data, offset: " << receivedData.offset <<
194  " error flag:" << receivedData.flags.error);
195 
196  const bool failed = receivedData.flags.error;
197  if (failed) {
199  return;
200  }
201 
202  const int64_t existingContent = reply ? reply->content_length : 0;
203  const size_t maxSize = MaxObjectSize > SBuf::maxSize ? SBuf::maxSize : MaxObjectSize;
204  const bool tooLarge = (existingContent > -1 && existingContent > static_cast<int64_t>(maxSize)) ||
205  (maxSize < object_.length()) ||
206  ((maxSize - object_.length()) < receivedData.length);
207 
208  if (tooLarge) {
210  return;
211  }
212 
213  object_.append(receivedData.data, receivedData.length);
214  http->out.size += receivedData.length;
215  http->out.offset += receivedData.length;
216 
217  switch (clientStreamStatus(node, http)) {
218  case STREAM_NONE: {
219  debugs(33, 3, "Get more data");
220  StoreIOBuffer tempBuffer;
221  tempBuffer.offset = http->out.offset;
222  tempBuffer.data = context_->requestBuffer;
223  tempBuffer.length = HTTP_REQBUF_SZ;
224  clientStreamRead(node, http, tempBuffer);
225  }
226  break;
227  case STREAM_COMPLETE:
228  debugs(33, 3, "Object data transfer successfully complete");
230  break;
232  debugs(33, 3, "Object data transfer failed: STREAM_UNPLANNED_COMPLETE");
234  break;
235  case STREAM_FAILED:
236  debugs(33, 3, "Object data transfer failed: STREAM_FAILED");
238  break;
239  default:
240  fatal("unreachable code");
241  }
242 }
243 
244 void
246 {
247  debugs(33, 7, this);
248  Must(done());
249 }
250 
253 void
255 {
256  CbDialer *dialer = dynamic_cast<CbDialer*>(callback_->getDialer());
257  Must(dialer);
258  dialer->status = statusCode;
259  if (statusCode == Http::scOkay)
260  dialer->object = object_;
262  callback_ = nullptr;
263 
264  // We cannot deleteThis() because we may be called synchronously from
265  // doCallouts() via handleReply() (XXX), and doCallouts() may crash if we
266  // disappear. Instead, schedule an async call now so that later, when the
267  // call firing code discovers a done() job, it deletes us.
269 }
270 
bool done() const
the job is destroyed in callEnd() when done()
Definition: AsyncJob.cc:90
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
virtual bool doneAll() const
whether positive goal has been reached
Definition: AsyncJob.cc:96
#define HTTP_REQBUF_SZ
Definition: defines.h:216
#define assert(EX)
Definition: assert.h:17
Ip::Address my_addr
Definition: HttpRequest.h:143
void clientStreamInit(dlink_list *list, CSR *func, CSD *rdetach, CSS *readstatus, ClientStreamData readdata, CSCB *callback, CSD *cdetach, ClientStreamData callbackdata, StoreIOBuffer tailBuffer)
#define CallJobHere(debugSection, debugLevel, job, Class, method)
Definition: AsyncJobCalls.h:57
SBuf url_
the url to download
Definition: Downloader.h:75
Raw & hex()
print data using two hex digits per byte (decoder: xxd -r -p)
Definition: Debug.h:189
Downloader(SBuf &url, AsyncCall::Pointer &aCallback, const XactionInitiator initiator, unsigned int level=0)
Definition: Downloader.cc:66
Definition: SBuf.h:87
SQUIDCEXTERN CSD clientReplyDetach
Definition: client_side.h:446
HttpRequestMethod method
Definition: HttpRequest.h:102
void setLogUri(ClientHttpRequest *http, char const *uri, bool cleanUrl)
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
#define xstrdup
SBuf & append(const SBuf &S)
Definition: SBuf.cc:207
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:154
AnyP::ProtocolVersion http_ver
Definition: Message.h:72
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
MEMPROXY_CLASS(DownloaderContext)
XactionInitiator initiator_
The initiator of the download request.
Definition: Downloader.h:80
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be < MAX(size_type)/2. Currently 256Mb. ...
Definition: SBuf.h:96
time_t squid_curtime
Definition: stub_time.cc:17
void host(const char *src)
Definition: url.cc:47
static const size_t MaxObjectSize
The maximum allowed object size.
Definition: Downloader.h:73
virtual bool doneAll() const
whether positive goal has been reached
Definition: Downloader.cc:91
identifies a protocol agent or Squid feature initiating transactions
static void downloaderRecipient(clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
Definition: Downloader.cc:97
virtual void print(std::ostream &os) const
Definition: Downloader.cc:61
virtual void start()
called by AsyncStart; do not call directly
Definition: Downloader.cc:179
clientStream_status_t clientStreamStatus(clientStreamNode *thisObject, ClientHttpRequest *http)
void setNoAddr()
Definition: Address.cc:285
virtual CallDialer * getDialer()=0
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:405
int64_t offset
Definition: StoreIOBuffer.h:55
Definition: parse.c:104
StatusCode
Definition: StatusCode.h:20
int64_t content_length
Definition: Message.h:83
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
dlink_node node
Definition: clientStream.h:88
Ip::Address client_addr
Definition: HttpRequest.h:137
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1054
DownloaderContext(Downloader *dl, ClientHttpRequest *h)
Definition: Downloader.cc:39
SQUIDCEXTERN CSR clientGetMoreData
Definition: client_side.h:444
Cbc * valid() const
was set and is valid
Definition: CbcPointer.h:41
const unsigned int level_
Definition: Downloader.h:78
void fatal(const char *message)
Definition: fatal.cc:39
const char * c_str()
Definition: SBuf.cc:546
void handleReply(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
Definition: Downloader.cc:186
ClientStreamData data
Definition: clientStream.h:94
ClientRequestContext * calloutContext
struct StoreIOBuffer::@146 flags
RefCount< DownloaderContext > Pointer
Definition: Downloader.cc:28
URL url
the request URI
Definition: HttpRequest.h:103
AsyncCall::Pointer callback_
callback to call when download finishes
Definition: Downloader.h:76
struct ClientHttpRequest::Out out
Http::StatusCode status
Definition: Downloader.h:46
static HttpRequest * FromUrl(const char *url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:523
void callBack(Http::StatusCode const status)
Definition: Downloader.cc:254
ClientHttpRequest * http
Definition: Downloader.cc:35
unsigned short port() const
Definition: Address.cc:786
virtual void swanSong()
Definition: Downloader.cc:81
#define Must(cond)
Definition: TextException.h:89
unsigned error
Definition: StoreIOBuffer.h:52
HttpHeader header
Definition: Message.h:74
virtual ~Downloader()
Definition: Downloader.cc:75
void downloadFinished()
delays destruction to protect doCallouts()
Definition: Downloader.cc:245
static void downloaderDetach(clientStreamNode *node, ClientHttpRequest *http)
Definition: Downloader.cc:119
#define MYNAME
Definition: Debug.h:160
DownloaderContextPointer context_
Pointer to an object that stores the clientStream required info.
Definition: Downloader.h:83
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1063
CbcPointer< Downloader > downloader
The Downloader object which initiated the HTTP request if any.
Definition: HttpRequest.h:221
#define ScheduleCallHere(call)
Definition: AsyncCall.h:166
SQUIDCEXTERN CSS clientReplyStatus
Definition: client_side.h:445
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
bool canceled()
Definition: AsyncCall.h:58
Callback data to use with Downloader callbacks.
Definition: Downloader.h:35
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod & method
Definition: stub_store.cc:126
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:412
Definition: Debug.h:179
SBuf object_
the object body data
Definition: Downloader.h:77
bool buildRequest()
Initializes and starts the HTTP GET request to the remote server.
Definition: Downloader.cc:127
char requestBuffer[HTTP_REQBUF_SZ]
Definition: Downloader.cc:36
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
Ip::Address indirect_client_addr
Definition: HttpRequest.h:140
C * getRaw() const
Definition: RefCount.h:74
CbcPointer< Downloader > downloader
Definition: Downloader.cc:34
char * urlCanonicalClean(const HttpRequest *request)
Definition: url.cc:518

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors