adaptation_service ACL This patch adds the new ACL adaptation_service, to match the name of: - an adaptation service or group that had been applied to the master transaction in the past - an adaptation service or group that is being applied to the master transaction now An adaptation group is formed by adaptation_service_chain or adaptation_service_set directives. Both REQMOD and RESPMOD services, successful or failed service applications matches this acl. This is a Measurement Factory project === modified file 'src/AclRegs.cc' --- src/AclRegs.cc 2013-10-25 00:13:46 +0000 +++ src/AclRegs.cc 2013-11-01 08:23:50 +0000 @@ -1,27 +1,31 @@ #include "squid.h" /** This file exists to provide satic registration code to executables that need ACLs. We cannot place this code in acl/lib*.la because it does not get linked in, because nobody is using these classes by name. */ +#if USE_ADAPTATION +#include "acl/AdaptationService.h" +#include "acl/AdaptationServiceData.h" +#endif #include "acl/AllOf.h" #include "acl/AnyOf.h" #if USE_SQUID_EUI #include "acl/Arp.h" #include "acl/Eui64.h" #endif #include "acl/Asn.h" #include "acl/Browser.h" #include "acl/Checklist.h" #include "acl/Data.h" #include "acl/DestinationAsn.h" #include "acl/DestinationDomain.h" #include "acl/DestinationIp.h" #include "acl/DomainData.h" #if USE_AUTH #include "acl/ExtUser.h" #endif #include "acl/FilledChecklist.h" #include "acl/Gadgets.h" #include "acl/HierCode.h" @@ -176,20 +180,25 @@ ACL::Prototype ACLProxyAuth::UserRegistryProtoype(&ACLProxyAuth::UserRegistryEntry_, "proxy_auth"); ACLProxyAuth ACLProxyAuth::UserRegistryEntry_(new ACLUserData, "proxy_auth"); ACL::Prototype ACLProxyAuth::RegexRegistryProtoype(&ACLProxyAuth::RegexRegistryEntry_, "proxy_auth_regex" ); ACLProxyAuth ACLProxyAuth::RegexRegistryEntry_(new ACLRegexData, "proxy_auth_regex"); ACL::Prototype ACLMaxUserIP::RegistryProtoype(&ACLMaxUserIP::RegistryEntry_, "max_user_ip"); ACLMaxUserIP ACLMaxUserIP::RegistryEntry_("max_user_ip"); #endif ACL::Prototype ACLTag::RegistryProtoype(&ACLTag::RegistryEntry_, "tag"); ACLStrategised ACLTag::RegistryEntry_(new ACLStringData, ACLTagStrategy::Instance(), "tag"); ACL::Prototype Acl::AnyOf::RegistryProtoype(&Acl::AnyOf::RegistryEntry_, "any-of"); Acl::AnyOf Acl::AnyOf::RegistryEntry_; ACL::Prototype Acl::AllOf::RegistryProtoype(&Acl::AllOf::RegistryEntry_, "all-of"); Acl::AllOf Acl::AllOf::RegistryEntry_; ACL::Prototype ACLNote::RegistryProtoype(&ACLNote::RegistryEntry_, "note"); ACLStrategised ACLNote::RegistryEntry_(new ACLNoteData, ACLNoteStrategy::Instance(), "note"); + +#if USE_ADAPTATION +ACL::Prototype ACLAdaptationService::RegistryProtoype(&ACLAdaptationService::RegistryEntry_, "adaptation_service"); +ACLStrategised ACLAdaptationService::RegistryEntry_(new ACLAdaptationServiceData, ACLAdaptationServiceStrategy::Instance(), "adaptation_service"); +#endif === added file 'src/acl/AdaptationService.cc' --- src/acl/AdaptationService.cc 1970-01-01 00:00:00 +0000 +++ src/acl/AdaptationService.cc 2013-11-06 15:54:59 +0000 @@ -0,0 +1,34 @@ +#include "squid.h" +#include "acl/Checklist.h" +#include "acl/IntRange.h" +#include "acl/AdaptationService.h" +#include "adaptation/Config.h" +#include "adaptation/History.h" +#include "HttpRequest.h" + +int +ACLAdaptationServiceStrategy::match (ACLData * &data, ACLFilledChecklist *checklist, ACLFlags &) +{ + HttpRequest *request = checklist->request; + if (!request) + return 0; + Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah == NULL) + return 0; + + Adaptation::History::AdaptationServices::iterator it; + for (it = ah->theAdaptationServices.begin(); it != ah->theAdaptationServices.end(); ++it) { + if (data->match(it->c_str())) + return 1; + } + + return 0; +} + +ACLAdaptationServiceStrategy * +ACLAdaptationServiceStrategy::Instance() +{ + return &Instance_; +} + +ACLAdaptationServiceStrategy ACLAdaptationServiceStrategy::Instance_; === added file 'src/acl/AdaptationService.h' --- src/acl/AdaptationService.h 1970-01-01 00:00:00 +0000 +++ src/acl/AdaptationService.h 2013-10-31 16:47:17 +0000 @@ -0,0 +1,35 @@ +#ifndef SQUID_ACLADAPTATIONSERVICE_H +#define SQUID_ACLADAPTATIONSERVICE_H + +#include "acl/Strategised.h" +#include "acl/Strategy.h" + +/// \ingroup ACLAPI +class ACLAdaptationServiceStrategy : public ACLStrategy +{ + +public: + virtual int match (ACLData * &, ACLFilledChecklist *, ACLFlags &); + static ACLAdaptationServiceStrategy *Instance(); + /** + * Not implemented to prevent copies of the instance. + */ + ACLAdaptationServiceStrategy(ACLAdaptationServiceStrategy const &); + +private: + static ACLAdaptationServiceStrategy Instance_; + ACLAdaptationServiceStrategy() {} + + ACLAdaptationServiceStrategy &operator = (ACLAdaptationServiceStrategy const &); +}; + +/// \ingroup ACLAPI +class ACLAdaptationService +{ + +private: + static ACL::Prototype RegistryProtoype; + static ACLStrategised RegistryEntry_; +}; + +#endif /* SQUID_ACLADAPTATIONSERVICE_H */ === added file 'src/acl/AdaptationServiceData.cc' --- src/acl/AdaptationServiceData.cc 1970-01-01 00:00:00 +0000 +++ src/acl/AdaptationServiceData.cc 2013-11-07 10:23:11 +0000 @@ -0,0 +1,39 @@ +#include "squid.h" +#include "acl/AdaptationServiceData.h" +#include "acl/Checklist.h" +#include "adaptation/Config.h" +#include "adaptation/ecap/Config.h" +#include "adaptation/icap/Config.h" +#include "adaptation/Service.h" +#include "adaptation/ServiceGroups.h" +#include "cache_cf.h" +#include "ConfigParser.h" +#include "Debug.h" +#include "wordlist.h" + +void +ACLAdaptationServiceData::parse() +{ + Adaptation::Config::needHistory = true; + while (char *t = ConfigParser::strtokFile()) { + if ( +#if USE_ECAP + Adaptation::Ecap::TheConfig.findServiceConfig(t) == NULL && +#endif +#if ICAP_CLIENT + Adaptation::Icap::TheConfig.findServiceConfig(t) == NULL && +#endif + Adaptation::FindGroup(t) == NULL) { + debugs(28, 0, "Adaptation service/group " << t << " in adaptation_service acl is not defined"); + self_destruct(); + } + insert(t); + } +} + +ACLData * +ACLAdaptationServiceData::clone() const +{ + return new ACLAdaptationServiceData(*this); +} + === added file 'src/acl/AdaptationServiceData.h' --- src/acl/AdaptationServiceData.h 1970-01-01 00:00:00 +0000 +++ src/acl/AdaptationServiceData.h 2013-11-06 15:59:00 +0000 @@ -0,0 +1,21 @@ + +#ifndef SQUID_ADAPTATIONSERVICEDATA_H +#define SQUID_ADAPTATIONSERVICEDATA_H + +#include "acl/Acl.h" +#include "acl/Data.h" +#include "acl/StringData.h" + +/// \ingroup ACLAPI +class ACLAdaptationServiceData : public ACLStringData +{ +public: + ACLAdaptationServiceData() : ACLStringData() {} + ACLAdaptationServiceData(ACLAdaptationServiceData const &old) : ACLStringData(old) {}; + // Not implemented + ACLAdaptationServiceData &operator= (ACLAdaptationServiceData const &); + virtual void parse(); + virtual ACLData *clone() const; +}; + +#endif /* SQUID_ADAPTATIONSERVICEDATA_H */ === modified file 'src/acl/Makefile.am' --- src/acl/Makefile.am 2013-05-26 01:08:42 +0000 +++ src/acl/Makefile.am 2013-11-07 09:47:34 +0000 @@ -131,30 +131,38 @@ ## TODO: move these to their respectful dirs when those dirs are created EXTRA_libacls_la_SOURCES = SSL_ACLS = \ CertificateData.cc \ CertificateData.h \ Certificate.cc \ Certificate.h \ ServerCertificate.cc \ ServerCertificate.h \ SslError.cc \ SslError.h \ SslErrorData.cc \ SslErrorData.h if ENABLE_SSL libacls_la_SOURCES += $(SSL_ACLS) endif +if USE_ADAPTATION +libacls_la_SOURCES += AdaptationService.h \ + AdaptationService.cc \ + AdaptationServiceData.h \ + AdaptationServiceData.cc +endif + + EXTRA_libacls_la_SOURCES += $(SSL_ACLS) ARP_ACLS = Arp.cc Arp.h Eui64.cc Eui64.h if USE_SQUID_EUI libacls_la_SOURCES += $(ARP_ACLS) endif EXTRA_libacls_la_SOURCES += $(ARP_ACLS) === modified file 'src/acl/StringData.h' --- src/acl/StringData.h 2013-10-25 00:13:46 +0000 +++ src/acl/StringData.h 2013-11-01 08:09:14 +0000 @@ -32,34 +32,34 @@ */ #ifndef SQUID_ACLSTRINGDATA_H #define SQUID_ACLSTRINGDATA_H #include "acl/Acl.h" #include "acl/Data.h" #include "splay.h" class ACLStringData : public ACLData { public: MEMPROXY_CLASS(ACLStringData); ACLStringData(); ACLStringData(ACLStringData const &); ACLStringData &operator= (ACLStringData const &); virtual ~ACLStringData(); bool match(char const *); wordlist *dump(); - void parse(); + virtual void parse(); bool empty() const; virtual ACLData *clone() const; /// Insert a string data value void insert(const char *); SplayNode *values; }; /* TODO move into .cci files */ MEMPROXY_CLASS_INLINE(ACLStringData); #endif /* SQUID_ACLSTRINGDATA_H */ === modified file 'src/adaptation/Config.cc' --- src/adaptation/Config.cc 2013-07-21 19:24:35 +0000 +++ src/adaptation/Config.cc 2013-11-07 10:22:38 +0000 @@ -50,74 +50,87 @@ int Adaptation::Config::send_username = false; int Adaptation::Config::use_indirect_client = true; const char *metasBlacklist[] = { "Methods", "Service", "ISTag", "Encapsulated", "Opt-body-type", "Max-Connections", "Options-TTL", "Date", "Service-ID", "Allow", "Preview", "Transfer-Preview", "Transfer-Ignore", "Transfer-Complete", NULL }; Notes Adaptation::Config::metaHeaders("ICAP header", metasBlacklist); +bool Adaptation::Config::needHistory = false; Adaptation::ServiceConfig* Adaptation::Config::newServiceConfig() const { return new ServiceConfig(); } void Adaptation::Config::removeService(const String& service) { removeRule(service); const Groups& groups = AllGroups(); for (unsigned int i = 0; i < groups.size(); ) { const ServiceGroupPointer group = groups[i]; const ServiceGroup::Store& services = group->services; typedef ServiceGroup::Store::const_iterator SGSI; for (SGSI it = services.begin(); it != services.end(); ++it) { if (*it == service) { group->removedServices.push_back(service); group->services.prune(service); debugs(93, 5, HERE << "adaptation service " << service << " removed from group " << group->id); break; } } if (services.empty()) { removeRule(group->id); AllGroups().prune(group); } else { ++i; } } } +Adaptation::ServiceConfigPointer +Adaptation::Config::findServiceConfig(const String &service) +{ + typedef ServiceConfigs::const_iterator SCI; + const ServiceConfigs& configs = serviceConfigs; + for (SCI cfg = configs.begin(); cfg != configs.end(); ++cfg) { + if ((*cfg)->key == service) + return *cfg; + } + return NULL; +} + void Adaptation::Config::removeRule(const String& id) { typedef AccessRules::const_iterator ARI; const AccessRules& rules = AllRules(); for (ARI it = rules.begin(); it != rules.end(); ++it) { AccessRule* rule = *it; if (rule->groupId == id) { debugs(93, 5, HERE << "removing access rules for:" << id); AllRules().prune(rule); delete (rule); break; } } } void Adaptation::Config::clear() { debugs(93, 3, HERE << "rules: " << AllRules().size() << ", groups: " << === modified file 'src/adaptation/Config.h' --- src/adaptation/Config.h 2013-10-25 00:13:46 +0000 +++ src/adaptation/Config.h 2013-11-07 10:22:27 +0000 @@ -32,50 +32,52 @@ public: static bool Enabled; // true if at least one adaptation mechanism is // these are global squid.conf options, documented elsewhere static char *masterx_shared_name; // global TODO: do we need TheConfig? static int service_iteration_limit; static int send_client_ip; static int send_username; static int use_indirect_client; // Options below are accessed via Icap::TheConfig or Ecap::TheConfig // TODO: move ICAP-specific options to Icap::Config and add TheConfig int onoff; int service_failure_limit; time_t oldest_service_failure; int service_revival_delay; static Notes metaHeaders; ///< The list of configured meta headers + static bool needHistory; ///< HttpRequest adaptation history should recorded + typedef Vector ServiceConfigs; ServiceConfigs serviceConfigs; Config(); virtual ~Config(); void parseService(void); void freeService(void); void dumpService(StoreEntry *, const char *) const; - ServicePointer findService(const String&); + ServiceConfigPointer findServiceConfig(const String&); /** * Creates and starts the adaptation services. In the case the adaptation * mechanism is disabled then removes any reference to the services from * access rules and service groups, and returns false. * \return true if the services are ready and running, false otherwise */ virtual bool finalize(); protected: /// Removes any reference to the services from configuration virtual void clear(); /// creates service configuration object that will parse and keep cfg info virtual ServiceConfig *newServiceConfig() const; /// Removes the given service from all service groups. void removeService(const String& service); /// Removes access rules of the given service or group === modified file 'src/adaptation/History.cc' --- src/adaptation/History.cc 2012-09-03 08:58:40 +0000 +++ src/adaptation/History.cc 2013-10-31 10:15:21 +0000 @@ -132,37 +132,43 @@ bool Adaptation::History::extractNextServices(String &value) { if (theNextServices == TheNullServices) return false; value = theNextServices; theNextServices = TheNullServices; // prevents resetting the plan twice return true; } void Adaptation::History::recordMeta(const HttpHeader *lm) { lastMeta.clean(); lastMeta.update(lm, NULL); allMeta.update(lm, NULL); allMeta.compact(); } void +Adaptation::History::recordAdaptationService(SBuf &srvId) +{ + theAdaptationServices.push_back(srvId); +} + +void Adaptation::History::setFutureServices(const DynamicGroupCfg &services) { if (!theFutureServices.empty()) debugs(93,3, HERE << "old future services: " << theFutureServices); debugs(93,3, HERE << "new future services: " << services); theFutureServices = services; // may be empty } bool Adaptation::History::extractFutureServices(DynamicGroupCfg &value) { if (theFutureServices.empty()) return false; value = theFutureServices; theFutureServices.clear(); return true; } === modified file 'src/adaptation/History.h' --- src/adaptation/History.h 2013-10-25 00:13:46 +0000 +++ src/adaptation/History.h 2013-10-31 10:15:05 +0000 @@ -1,77 +1,82 @@ #ifndef SQUID_ADAPT_HISTORY_H #define SQUID_ADAPT_HISTORY_H #include "adaptation/DynamicGroupCfg.h" #include "base/RefCount.h" #include "base/Vector.h" #include "HttpHeader.h" #include "Notes.h" +#include "SBuf.h" #include "SquidString.h" namespace Adaptation { /// collects information about adaptations related to a master transaction class History: public RefCountable { public: typedef RefCount Pointer; History(); /// record the start of a xact, return xact history ID int recordXactStart(const String &serviceId, const timeval &when, bool retrying); /// record the end of a xact identified by its history ID void recordXactFinish(int hid); /// dump individual xaction times to a string void allLogString(const char *serviceId, String &buf); /// dump xaction times, merging retried and retry times together void sumLogString(const char *serviceId, String &buf); /// sets or resets a cross-transactional database record void updateXxRecord(const char *name, const String &value); /// returns true and fills the record fields iff there is a db record bool getXxRecord(String &name, String &value) const; /// sets or resets next services for the Adaptation::Iterator to notice void updateNextServices(const String &services); /// returns true, fills the value, and resets iff next services were set bool extractNextServices(String &value); /// store the last meta header fields received from the adaptation service void recordMeta(const HttpHeader *lm); + void recordAdaptationService(SBuf &srvId); public: /// Last received meta header (REQMOD or RESPMOD, whichever comes last). HttpHeader lastMeta; /// All REQMOD and RESPMOD meta headers merged. Last field wins conflicts. HttpHeader allMeta; /// key:value pairs set by adaptation_meta, to be added to /// AccessLogEntry::notes when ALE becomes available NotePairs::Pointer metaHeaders; + typedef Vector AdaptationServices; + AdaptationServices theAdaptationServices; ///< The service groups used + /// sets future services for the Adaptation::AccessCheck to notice void setFutureServices(const DynamicGroupCfg &services); /// returns true, fills the value, and resets iff future services were set bool extractFutureServices(DynamicGroupCfg &services); private: /// single Xaction stats (i.e., a historical record entry) class Entry { public: Entry(const String &serviceId, const timeval &when); Entry(); // required by Vector<> void stop(); ///< updates stats on transaction end int rptm(); ///< returns response time [msec], calculates it if needed String service; ///< adaptation service ID timeval start; ///< when the xaction was started === modified file 'src/adaptation/Iterator.cc' --- src/adaptation/Iterator.cc 2013-10-25 00:13:46 +0000 +++ src/adaptation/Iterator.cc 2013-11-07 10:23:32 +0000 @@ -28,74 +28,93 @@ { if (theCause != NULL) HTTPMSGLOCK(theCause); if (theMsg != NULL) HTTPMSGLOCK(theMsg); } Adaptation::Iterator::~Iterator() { assert(!theLauncher); HTTPMSGUNLOCK(theMsg); HTTPMSGUNLOCK(theCause); } void Adaptation::Iterator::start() { Adaptation::Initiate::start(); thePlan = ServicePlan(theGroup, filter()); + + // Add adaptation group name once and now, before + // dynamic groups change it at step() time. + if (Adaptation::Config::needHistory && !thePlan.exhausted() && (dynamic_cast(theGroup.getRaw()) || dynamic_cast(theGroup.getRaw()))) { + HttpRequest *request = dynamic_cast(theMsg); + if (!request) + request = theCause; + Must(request); + Adaptation::History::Pointer ah = request->adaptHistory(true); + SBuf gid(theGroup->id); + ah->recordAdaptationService(gid); + } + step(); } void Adaptation::Iterator::step() { ++iterations; debugs(93,5, HERE << '#' << iterations << " plan: " << thePlan); Must(!theLauncher); if (thePlan.exhausted()) { // nothing more to do sendAnswer(Answer::Forward(theMsg)); Must(done()); return; } HttpRequest *request = dynamic_cast(theMsg); if (!request) request = theCause; assert(request); request->clearError(); if (iterations > Adaptation::Config::service_iteration_limit) { debugs(93,DBG_CRITICAL, "Adaptation iterations limit (" << Adaptation::Config::service_iteration_limit << ") exceeded:\n" << "\tPossible service loop with " << theGroup->kind << " " << theGroup->id << ", plan=" << thePlan); throw TexcHere("too many adaptations"); } ServicePointer service = thePlan.current(); Must(service != NULL); debugs(93,5, HERE << "using adaptation service: " << service->cfg().key); + if (Adaptation::Config::needHistory) { + Adaptation::History::Pointer ah = request->adaptHistory(true); + SBuf uid(thePlan.current()->cfg().key); + ah->recordAdaptationService(uid); + } + theLauncher = initiateAdaptation( service->makeXactLauncher(theMsg, theCause)); Must(initiated(theLauncher)); Must(!done()); } void Adaptation::Iterator::noteAdaptationAnswer(const Answer &answer) { switch (answer.kind) { case Answer::akForward: handleAdaptedHeader(const_cast(answer.message.getRaw())); break; case Answer::akBlock: handleAdaptationBlock(answer); break; case Answer::akError: handleAdaptationError(answer.final); === modified file 'src/cf.data.pre' --- src/cf.data.pre 2013-10-13 17:55:11 +0000 +++ src/cf.data.pre 2013-11-06 15:18:00 +0000 @@ -1056,40 +1056,49 @@ acl aclname tag tagvalue ... # string match on tag returned by external acl helper [slow] acl aclname hier_code codename ... # string match against squid hierarchy code(s); [fast] # e.g., DIRECT, PARENT_HIT, NONE, etc. # # NOTE: This has no effect in http_access rules. It only has # effect in rules that affect the reply data stream such as # http_reply_access. acl aclname note name [value ...] # match transaction annotation [fast] # Without values, matches any annotation with a given name. # With value(s), matches any annotation with a given name that # also has one of the given values. # Names and values are compared using a string equality test. # Annotation sources include note and adaptation_meta directives # as well as helper and eCAP responses. + acl aclname adaptation_service service ... + # Matches the name of any icap_service, ecap_service, + # adaptation_service_set, or adaptation_service_chain that Squid + # has used (or attempted to use) for the master transaction. + # This ACL must be defined after the corresponding adaptation + # service is named in squid.conf. This ACL is usable with + # adaptation_meta because it starts matching immediately after + # the service has been selected for adaptation. + IF USE_SSL acl aclname ssl_error errorname # match against SSL certificate validation error [fast] # # For valid error names see in @DEFAULT_ERROR_DIR@/templates/error-details.txt # template file. # # The following can be used as shortcuts for certificate properties: # [ssl::]certHasExpired: the "not after" field is in the past # [ssl::]certNotYetValid: the "not before" field is in the future # [ssl::]certUntrusted: The certificate issuer is not to be trusted. # [ssl::]certSelfSigned: The certificate is self signed. # [ssl::]certDomainMismatch: The certificate CN domain does not # match the name the name of the host we are connecting to. # # The ssl::certHasExpired, ssl::certNotYetValid, ssl::certDomainMismatch, # ssl::certUntrusted, and ssl::certSelfSigned can also be used as # predefined ACLs, just like the 'all' ACL. # # NOTE: The ssl_error ACL is only supported with sslproxy_cert_error,