helper.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 84 Helper process maintenance */
10 
11 #ifndef SQUID_HELPER_H
12 #define SQUID_HELPER_H
13 
14 #include "base/AsyncCall.h"
15 #include "base/InstanceId.h"
16 #include "cbdata.h"
17 #include "comm/forward.h"
18 #include "dlink.h"
19 #include "helper/ChildConfig.h"
20 #include "helper/forward.h"
21 #include "helper/Reply.h"
22 #include "helper/Request.h"
23 #include "helper/ReservationId.h"
24 #include "ip/Address.h"
25 #include "sbuf/SBuf.h"
26 
27 #include <list>
28 #include <map>
29 #include <queue>
30 #include <unordered_map>
31 
33 class HelperServerBase;
34 class MemBuf;
35 class Packable;
36 class wordlist;
37 
38 namespace Helper
39 {
41 class Xaction {
43 public:
44  Xaction(HLPCB *c, void *d, const char *b): request(c, d, b) {}
47 };
48 }
49 
65 class helper
66 {
68 
69 public:
70  explicit helper(const char *name) : id_name(name) {}
71  virtual ~helper();
72 
75 
77  bool trySubmit(const char *buf, HLPCB * callback, void *data);
78 
82 
84  void packStatsInto(Packable *p, const char *label = NULL) const;
87  bool willOverload() const;
88 
92  void handleKilledServer(HelperServerBase *srv, bool &needsNewServers);
93 
94 public:
95  wordlist *cmdline = nullptr;
97  std::queue<Helper::Xaction *> queue;
98  const char *id_name = nullptr;
100  int ipc_type = 0;
102  unsigned int droppedRequests = 0;
103  time_t overloadStart = 0;
104  time_t last_queue_warn = 0;
105  time_t last_restart = 0;
106  time_t timeout = 0;
107  bool retryTimedOut = false;
108  bool retryBrokenHelper = false;
110  char eom = '\n';
111 
112  struct _stats {
113  int requests = 0;
114  int replies = 0;
115  int timedout = 0;
116  int queue_size = 0;
117  int avg_svc_time = 0;
118  } stats;
119 
120 protected:
121  friend void helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data);
122  bool queueFull() const;
123  bool overloaded() const;
124  void syncQueueStats();
125  bool prepSubmit();
126  void submit(const char *buf, HLPCB * callback, void *data);
127 };
128 
129 class statefulhelper : public helper
130 {
132 
133 public:
134  typedef std::unordered_map<Helper::ReservationId, helper_stateful_server *> Reservations;
135 
136  explicit statefulhelper(const char *name) : helper(name) {}
137  virtual ~statefulhelper() = default;
138 
139 public:
142 
144  void cancelReservation(const Helper::ReservationId reservation);
145 
146 private:
147  friend void helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPCB * callback, void *data, const Helper::ReservationId & reservation);
148 
151 
152  void submit(const char *buf, HLPCB * callback, void *data, const Helper::ReservationId & reservation);
153  bool trySubmit(const char *buf, HLPCB * callback, void *data, const Helper::ReservationId & reservation);
154 
157 };
158 
161 {
162 public:
163  HelperServerBase(const char *desc, Ip::Address &, int aPid, void *aIpc, int rfd, int wfd);
164  virtual ~HelperServerBase();
165 
171  void closePipesSafely(const char *name);
172 
179  void closeWritePipeSafely(const char *name);
180 
181  // TODO: Teach each child to report its child-specific state instead.
183  virtual bool reserved() = 0;
184 
186  virtual void dropQueued();
187 
189  virtual helper *getParent() const = 0;
190 
191 public:
195  int pid = 0;
199  void *hIpc = nullptr;
200 
201  char *rbuf = nullptr;
202  size_t rbuf_sz = 0;
203  size_t roffset = 0;
204 
205  struct timeval dispatch_time;
206  struct timeval answer_time;
207 
209 
210  struct _helper_flags {
211  bool writing = false;
212  bool closing = false;
213  bool shutdown = false;
214  } flags;
215 
216  typedef std::list<Helper::Xaction *> Requests;
218 
219  struct {
220  uint64_t uses = 0;
221  uint64_t replies = 0;
222  uint64_t pending = 0;
223  uint64_t releases = 0;
224  uint64_t timedout = 0;
225  } stats;
226 };
227 
228 // TODO: Rename to StatelessHelperServer and rename HelperServerBase to HelperServer.
231 {
233 
234 public:
235  helper_server(helper *hlp, int pid, void *hIpc, int rfd, int wfd);
236  virtual ~helper_server();
237 
238  uint64_t nextRequestId = 0;
239 
240  MemBuf *wqueue = nullptr;
241  MemBuf *writebuf = nullptr;
242 
243  helper *parent = nullptr;
244 
250 
252  bool ignoreToEom = false;
253 
254  // STL says storing std::list iterators is safe when changing the list
255  typedef std::map<uint64_t, Requests::iterator> RequestIndex;
257 
262  Helper::Xaction *popRequest(int requestId);
263 
266  void checkForTimedOutRequests(bool const retry);
267 
268  /*HelperServerBase API*/
269  virtual bool reserved() override {return false;}
270  virtual void dropQueued() override;
271  virtual helper *getParent() const override {return parent;}
272 
274  static void requestTimeout(const CommTimeoutCbParams &io);
275 
277  static void HelperServerClosed(helper_server *srv);
278 };
279 
280 // TODO: Rename to StatefulHelperServer and rename HelperServerBase to HelperServer.
283 {
285 
286 public:
287  helper_stateful_server(statefulhelper *hlp, int pid, void *hIpc, int rfd, int wfd);
288  virtual ~helper_stateful_server();
289 
290  void reserve();
291  void clearReservation();
292 
293  /* HelperServerBase API */
294  virtual bool reserved() override {return reservationId.reserved();}
295  virtual helper *getParent() const override {return parent;}
296 
298  static void HelperServerClosed(helper_stateful_server *srv);
299 
300  statefulhelper *parent = nullptr;
301 
302  // Reservations temporary lock the server for an exclusive "client" use. The
303  // client keeps the reservation ID as a proof of her reservation. If a
304  // reservation expires, and the server is reserved for another client, then
305  // the reservation ID presented by the late client will not match ours.
307  time_t reservationStart = 0;
308 };
309 
310 /* helper.c */
311 void helperOpenServers(helper * hlp);
313 void helperSubmit(helper * hlp, const char *buf, HLPCB * callback, void *data);
314 void helperStatefulSubmit(statefulhelper * hlp, const char *buf, HLPCB * callback, void *data, uint64_t reservation);
315 void helperShutdown(helper * hlp);
317 
318 #endif /* SQUID_HELPER_H */
virtual helper * getParent() const override
the helper object that created this server
Definition: helper.h:271
std::list< Helper::Xaction * > Requests
Definition: helper.h:216
void closePipesSafely(const char *name)
Definition: helper.cc:63
dlink_node link
Definition: helper.h:208
time_t reservationStart
when the last reservation was made
Definition: helper.h:307
void submitRequest(Helper::Xaction *r)
Definition: helper.cc:431
Helper::ChildConfig childs
configuration settings for number running
Definition: helper.h:99
void syncQueueStats()
synchronizes queue-dependent measurements with the current queue state
Definition: helper.cc:477
represents a single "stateless helper" process
Definition: helper.h:230
wordlist * cmdline
Definition: helper.h:95
dlink_list servers
Definition: helper.h:96
statefulhelper(const char *name)
Definition: helper.h:136
bool reserved() const
Definition: ReservationId.h:22
void helperStatefulOpenServers(statefulhelper *hlp)
Definition: helper.cc:330
helper_stateful_server * findServer(const Helper::ReservationId &reservation)
Definition: helper.cc:599
struct helper::_stats stats
statefulhelper * parent
Definition: helper.h:300
virtual ~helper_stateful_server()
Definition: helper.cc:196
virtual ~helper_server()
Definition: helper.cc:160
virtual void dropQueued()
dequeues and sends a Helper::Unknown answer to all queued requests
Definition: helper.cc:113
virtual ~statefulhelper()=default
struct HelperServerBase::_helper_flags flags
friend void helperStatefulSubmit(statefulhelper *hlp, const char *buf, HLPCB *callback, void *data, const Helper::ReservationId &reservation)
Definition: helper.cc:552
uint64_t timedout
requests which timed-out
Definition: helper.h:224
const InstanceId< HelperServerBase > index
Definition: helper.h:194
int avg_svc_time
Definition: helper.h:117
#define CBDATA_CLASS(type)
Definition: cbdata.h:302
int ipc_type
Definition: helper.h:100
Definition: SBuf.h:86
void submit(const char *buf, HLPCB *callback, void *data)
dispatches or enqueues a helper requests; does not enforce queue limits
Definition: helper.cc:542
static void requestTimeout(const CommTimeoutCbParams &io)
Read timeout handler.
Definition: helper.cc:1542
time_t last_queue_warn
Definition: helper.h:104
static void HelperServerClosed(helper_stateful_server *srv)
close handler to handle exited server processes
Definition: helper.cc:861
std::queue< Helper::Xaction * > queue
Definition: helper.h:97
virtual bool reserved() override
whether the server is locked for exclusive use by a client
Definition: helper.h:269
virtual bool reserved() override
whether the server is locked for exclusive use by a client
Definition: helper.h:294
helper(const char *name)
Definition: helper.h:70
Helper::Reply reply
Definition: helper.h:46
struct timeval dispatch_time
Definition: helper.h:205
virtual void dropQueued() override
dequeues and sends a Helper::Unknown answer to all queued requests
Definition: helper.cc:184
CBDATA_CHILD(helper_stateful_server)
Xaction(HLPCB *c, void *d, const char *b)
Definition: helper.h:44
std::unordered_map< Helper::ReservationId, helper_stateful_server * > Reservations
Definition: helper.h:131
static void HelperServerClosed(helper_server *srv)
close handler to handle exited server processes
Definition: helper.cc:842
uint64_t uses
requests sent to this helper
Definition: helper.h:220
void submit(const char *buf, HLPCB *callback, void *data, const Helper::ReservationId &reservation)
Definition: helper.cc:630
Requests requests
requests in order of submission/expiration
Definition: helper.h:217
void cancelReservation(const Helper::ReservationId reservation)
undo reserveServer(), clear the reservation and kick the queue
Definition: helper.cc:583
time_t last_restart
Definition: helper.h:105
uint64_t releases
times release() has been called on this helper (if stateful)
Definition: helper.h:223
void helperStatefulSubmit(statefulhelper *hlp, const char *buf, HLPCB *callback, void *data, uint64_t reservation)
void reserveServer(helper_stateful_server *srv)
reserve the given server
Definition: helper.cc:570
HelperServerBase(const char *desc, Ip::Address &, int aPid, void *aIpc, int rfd, int wfd)
Definition: helper.cc:129
helper_stateful_server(statefulhelper *hlp, int pid, void *hIpc, int rfd, int wfd)
Definition: helper.cc:190
helper_server(helper *hlp, int pid, void *hIpc, int rfd, int wfd)
Definition: helper.cc:154
void * hIpc
Definition: helper.h:199
int timedout
Definition: helper.h:115
#define NULL
Definition: types.h:166
void helperShutdown(helper *hlp)
Definition: helper.cc:725
bool overloaded() const
Definition: helper.cc:471
represents a single helper process abstraction
Definition: helper.h:160
MemBuf * wqueue
Definition: helper.h:240
unsigned int droppedRequests
requests not sent during helper overload
Definition: helper.h:102
Definition: MemBuf.h:23
Definition: helper.h:65
void helperStatefulShutdown(statefulhelper *hlp)
Definition: helper.cc:762
uint64_t pending
queued lookups waiting to be sent to this helper
Definition: helper.h:222
std::map< uint64_t, Requests::iterator > RequestIndex
Definition: helper.h:255
struct timeval answer_time
Definition: helper.h:206
void const char HLPCB void * data
Definition: stub_helper.cc:16
Ip::Address addr
Definition: helper.h:196
int requests
Definition: helper.h:113
bool willOverload() const
Definition: helper.cc:720
virtual ~helper()
Definition: helper.cc:808
bool retryTimedOut
whether the timed-out requests must retried
Definition: helper.h:107
a (temporary) lock on a (stateful) helper channel
Definition: ReservationId.h:17
RequestIndex requestsIndex
maps request IDs to requests
Definition: helper.h:256
helper protocol primitives
Definition: ChildConfig.h:12
void packStatsInto(Packable *p, const char *label=NULL) const
Dump some stats about the helper state to a Packable object.
Definition: helper.cc:662
time_t overloadStart
when the helper became overloaded (zero if it is not)
Definition: helper.h:103
Comm::ConnectionPointer writePipe
Definition: helper.h:198
bool retryBrokenHelper
whether the requests must retried on BH replies
Definition: helper.h:108
const char * id_name
Definition: helper.h:98
uint64_t replies
replies received from this helper
Definition: helper.h:221
friend void helperSubmit(helper *hlp, const char *buf, HLPCB *callback, void *data)
Definition: helper.cc:458
size_t roffset
Definition: helper.h:203
MemBuf * writebuf
Definition: helper.h:241
CBDATA_CHILD(helper_server)
represents a single "stateful helper" process
Definition: helper.h:282
Helper::Xaction * popRequest(int requestId)
Definition: helper.cc:878
Helper::Xaction * nextRequest()
Definition: helper.cc:1247
char * rbuf
Definition: helper.h:201
void handleKilledServer(HelperServerBase *srv, bool &needsNewServers)
Definition: helper.cc:818
bool trySubmit(const char *buf, HLPCB *callback, void *data, const Helper::ReservationId &reservation)
If possible, submit request. Otherwise, either kill Squid or return false.
Definition: helper.cc:560
size_t rbuf_sz
Definition: helper.h:202
SBuf onTimedOutResponse
the response to use when helper response timedout
Definition: helper.h:109
struct HelperServerBase::@75 stats
virtual ~HelperServerBase()
Definition: helper.cc:146
void const char HLPCB * callback
Definition: stub_helper.cc:16
virtual helper * getParent() const =0
the helper object that created this server
helper * parent
Definition: helper.h:243
Helper::Xaction * replyXaction
Definition: helper.h:249
void helperSubmit(helper *hlp, const char *buf, HLPCB *callback, void *data)
Definition: helper.cc:458
bool queueFull() const
whether queuing an additional request would overload the helper
Definition: helper.cc:466
Helper::Request request
Definition: helper.h:45
virtual bool reserved()=0
whether the server is locked for exclusive use by a client
void HLPCB(void *, const Helper::Reply &)
Definition: forward.h:27
MEMPROXY_CLASS(Helper::Xaction)
bool ignoreToEom
Whether to ignore current message, because it is timed-out or other reason.
Definition: helper.h:252
Ip::Address addr
Definition: helper.h:101
int queue_size
Definition: helper.h:116
bool trySubmit(const char *buf, HLPCB *callback, void *data)
If possible, submit request. Otherwise, either kill Squid or return false.
Definition: helper.cc:531
void closeWritePipeSafely(const char *name)
Definition: helper.cc:89
time_t timeout
requests timeout
Definition: helper.h:106
bool prepSubmit()
Definition: helper.cc:504
Helper::ReservationId reservationId
"confirmation ID" of the last
Definition: helper.h:306
void helperOpenServers(helper *hlp)
Definition: helper.cc:215
void checkForTimedOutRequests(bool const retry)
Definition: helper.cc:1503
virtual helper * getParent() const override
the helper object that created this server
Definition: helper.h:295
char eom
the char which marks the end of (response) message
Definition: helper.h:110
void const char * buf
Definition: stub_helper.cc:16
Reservations reservations
reserved servers indexed by reservation IDs
Definition: helper.h:156
uint64_t nextRequestId
Definition: helper.h:238
Comm::ConnectionPointer readPipe
Definition: helper.h:197
Holds the required data to serve a helper request.
Definition: helper.h:41

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors