Skip to content

Commit

Permalink
mod_proxy: Handle backend address renewal with address_ttl= parameter.
Browse files Browse the repository at this point in the history 
Define a new proxy_address struct holding the current/latest sockaddr in use
by each proxy worker and conn. Since backend addresses can be updated when
their TTL expires and while connections are being processed, each address is
refcounted and freed only when the last worker (or conn) using it grabs the
new one.

The lifetime of the addresses is handled at a single place by the new
ap_proxy_determine_address() function. It guarantees to bind the current/latest
backend address to the passed in conn (or do nothing if it's up to date already).
The function is called indirectly by ap_proxy_determine_connection() for the
proxy modules that use it, or directly by mod_proxy_ftp and mod_proxy_hcheck.
It also is called eventually by ap_proxy_connect_backend() when connect()ing all
the current addresses fails, to check (PROXY_DETERMINE_ADDRESS_CHECK) if some
new addrs are available.

This commit is also a rework of the lifetime of conn->addr, conn->hostname
and conn->forward, using the conn->uds_pool and conn->fwd_pool for the cases
where the backend is connected through a UDS socket and a remote CONNECT proxy
respectively.

* include/ap_mmn.h:
  Minor bump for new function/fields.

* modules/proxy/mod_proxy.h (struct proxy_address,
                             ap_proxy_determine_addresss()):
  Declare ap_proxy_determine_addresss() and opaque struct proxy_address,
  new fields to structs proxy_conn_rec/proxy_worker_shared/proxy_worker.

* modules/proxy/mod_proxy.c (set_worker_param):
  Parse/set the new worker->address_ttl parameter.

* modules/proxy/proxy_util.c (proxy_util_register_hooks(),
                              ap_proxy_initialize_worker(),
                              ap_proxy_connection_reusable(),
                              ap_proxyerror(), proxyerror_core(),
                              init_conn_pool(), make_conn_subpool(),
                              connection_make(), connection_cleanup(),
                              connection_constructor()):
 Initialize *proxy_start_time in proxy_util_register_hooks() as the epoch
 from which expiration times are relative (i.e. seconds stored in an uint32_t
 for atomic changes).
 Make sure worker->s->is_address_reusable and worker->s->disablereuse are
 consistant in ap_proxy_initialize_worker(), thus no need to check for both
 in ap_proxy_connection_reusable().
 New proxyerror_core() helper taking an apr_status_t to log, wrap in
 ap_proxyerror().
 New make_conn_subpool() to create worker->cp->{pool,dns} with their own
 allocator.
 New connection_make() helper to factorize code in connection_cleanup() and
 connection_constructor().

* modules/proxy/proxy_util.c (proxy_address_inc(), proxy_address_dec(),
                              proxy_address_cleanup(), proxy_address_set_expired(),
                              worker_address_get(), worker_address_set(),
                              worker_address_resolve(), proxy_addrs_equal(),
                              ap_proxy_determine_address(),
                              ap_proxy_determine_connection(),
                              ap_proxy_connect_backend()):
 Implement ap_proxy_determine_address() using the above helpers for atomic changes,
 and call it from ap_proxy_determine_connection() and ap_proxy_connect_backend().

* modules/proxy/mod_proxy_ftp.c (proxy_ftp_handler):
  Use ap_proxy_determine_address() and use the returned backend->addr.

* modules/proxy/mod_proxy_hcheck.c (hc_determine_connection, hc_get_backend,
                                    hc_init_worker, hc_watchdog_callback):
  Use ap_proxy_determine_address() in hc_determine_connection() and call the
  latter from hc_get_backend(), replace hc_init_worker() by hc_init_baton()
  which now calls hc_get_hcworker() and hc_get_backend() to resolve the first
  address at init time.

* modules/proxy/mod_proxy_http.c (proxy_http_handler):
  Use backend->addr and ->hostname instead of worker->cp->addr and
  worker->s->hostname_ex respectively.

* modules/proxy/mod_proxy_ajp.c (ap_proxy_ajp_request):
  Use backend->addr and ->hostname instead of worker->cp->addr and
  worker->s->hostname_ex respectively.


Closes #367



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1912459 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
@ylavic
ylavic committed Sep 21, 2023
1 parent 850f1a5 commit 3c7f67f
Show file tree
Hide file tree
Showing 8 changed files with 837 additions and 411 deletions.
3 changes: 2 additions & 1 deletion include/ap_mmn.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -719,14 +719,15 @@
* than username / password. Add autht_provider structure.
* 20211221.14 (2.5.1-dev) Add request_rec->final_resp_passed bit
* 20211221.15 (2.5.1-dev) Add ap_get_pollfd_from_conn()
* 20211221.16 (2.5.1-dev) Add ap_proxy_determine_address()
*/

#define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */

#ifndef MODULE_MAGIC_NUMBER_MAJOR
#define MODULE_MAGIC_NUMBER_MAJOR 20211221
#endif
#define MODULE_MAGIC_NUMBER_MINOR 15 /* 0...n */
#define MODULE_MAGIC_NUMBER_MINOR 16 /* 0...n */

/**
* Determine if the server's current MODULE_MAGIC_NUMBER is at least a
Expand Down
18 changes: 18 additions & 0 deletions modules/proxy/mod_proxy.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,24 @@ static const char *set_worker_param(apr_pool_t *p,
return "EnableReuse must be On|Off";
worker->s->disablereuse_set = 1;
}
else if (!strcasecmp(key, "addressttl")) {
/* Address TTL in seconds
*/
apr_interval_time_t ttl;
if (strcmp(val, "-1") == 0) {
worker->s->address_ttl = -1;
}
else if (ap_timeout_parameter_parse(val, &ttl, "s") == APR_SUCCESS
&& (ttl <= apr_time_from_sec(APR_INT32_MAX))
&& (ttl % apr_time_from_sec(1)) == 0) {
worker->s->address_ttl = apr_time_sec(ttl);
}
else {
return "AddressTTL must be -1 or a number of seconds not "
"exceeding " APR_STRINGIFY(APR_INT32_MAX);
}
worker->s->address_ttl_set = 1;
}
else if (!strcasecmp(key, "route")) {
/* Worker route.
*/
Expand Down
32 changes: 32 additions & 0 deletions modules/proxy/mod_proxy.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,8 @@ typedef struct {
apr_array_header_t* cookie_domains;
} proxy_req_conf;

struct proxy_address; /* opaque TTL'ed and refcount'ed address */

typedef struct {
conn_rec *connection;
request_rec *r; /* Request record of the backend request
Expand All @@ -294,6 +296,9 @@ typedef struct {
* and its scpool/bucket_alloc (NULL before),
* must be left cleaned when used (locally).
*/
apr_pool_t *uds_pool; /* Subpool for reusing UDS paths */
apr_pool_t *fwd_pool; /* Subpool for reusing ProxyRemote infos */
struct proxy_address *address; /* Current remote address */
} proxy_conn_rec;

typedef struct {
Expand Down Expand Up @@ -488,6 +493,9 @@ typedef struct {
unsigned int was_malloced:1;
unsigned int is_name_matchable:1;
unsigned int response_field_size_set:1;
unsigned int address_ttl_set:1;
apr_int32_t address_ttl; /* backend address' TTL (seconds) */
apr_uint32_t address_expiry; /* backend address' next expiry time */
} proxy_worker_shared;

#define ALIGNED_PROXY_WORKER_SHARED_SIZE (APR_ALIGN_DEFAULT(sizeof(proxy_worker_shared)))
Expand All @@ -504,6 +512,7 @@ struct proxy_worker {
#endif
void *context; /* general purpose storage */
ap_conf_vector_t *section_config; /* <Proxy>-section wherein defined */
struct proxy_address *volatile address; /* current worker address (if reusable) */
};

/* default to health check every 30 seconds */
Expand Down Expand Up @@ -1041,6 +1050,29 @@ PROXY_DECLARE(int) ap_proxy_post_request(proxy_worker *worker,
request_rec *r,
proxy_server_conf *conf);

/* Bitmask for ap_proxy_determine_address() */
#define PROXY_DETERMINE_ADDRESS_CHECK (1u << 0)
/**
* Resolve an address, reusing the one of the worker if any.
* @param proxy_function calling proxy scheme (http, ajp, ...)
* @param conn proxy connection the address is used for
* @param hostname host to resolve (should be the worker's if reusable)
* @param hostport port to resolve (should be the worker's if reusable)
* @param flags bitmask of PROXY_DETERMINE_ADDRESS_*
* @param r current request (if any)
* @param s current server (or NULL if r != NULL and ap_proxyerror()
* should be called on error)
* @return APR_SUCCESS or an error, APR_EEXIST if the address is still
* the same and PROXY_DETERMINE_ADDRESS_CHECK is asked
*/
PROXY_DECLARE(apr_status_t) ap_proxy_determine_address(const char *proxy_function,
proxy_conn_rec *conn,
const char *hostname,
apr_port_t hostport,
unsigned int flags,
request_rec *r,
server_rec *s);

/**
* Determine backend hostname and port
* @param p memory pool used for processing
Expand Down
35 changes: 12 additions & 23 deletions modules/proxy/mod_proxy_ajp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,10 +236,8 @@ static int ap_proxy_ajp_request(apr_pool_t *p, request_rec *r,
if (status != APR_SUCCESS) {
conn->close = 1;
ap_log_rerror(APLOG_MARK, APLOG_ERR, status, r, APLOGNO(00868)
"request failed to %pI (%s:%d)",
conn->worker->cp->addr,
conn->worker->s->hostname_ex,
(int)conn->worker->s->port);
"request failed to %pI (%s:%hu)",
conn->addr, conn->hostname, conn->port);
if (status == AJP_EOVERFLOW)
return HTTP_BAD_REQUEST;
else {
Expand Down Expand Up @@ -334,10 +332,8 @@ static int ap_proxy_ajp_request(apr_pool_t *p, request_rec *r,
conn->close = 1;
apr_brigade_destroy(input_brigade);
ap_log_rerror(APLOG_MARK, APLOG_ERR, status, r, APLOGNO(00876)
"send failed to %pI (%s:%d)",
conn->worker->cp->addr,
conn->worker->s->hostname_ex,
(int)conn->worker->s->port);
"send failed to %pI (%s:%hu)",
conn->addr, conn->hostname, conn->port);
/*
* It is fatal when we failed to send a (part) of the request
* body.
Expand Down Expand Up @@ -376,10 +372,8 @@ static int ap_proxy_ajp_request(apr_pool_t *p, request_rec *r,
conn->close = 1;
apr_brigade_destroy(input_brigade);
ap_log_rerror(APLOG_MARK, APLOG_ERR, status, r, APLOGNO(00878)
"read response failed from %pI (%s:%d)",
conn->worker->cp->addr,
conn->worker->s->hostname_ex,
(int)conn->worker->s->port);
"read response failed from %pI (%s:%hu)",
conn->addr, conn->hostname, conn->port);

/* If we had a successful cping/cpong and then a timeout
* we assume it is a request that cause a back-end timeout,
Expand Down Expand Up @@ -676,10 +670,8 @@ static int ap_proxy_ajp_request(apr_pool_t *p, request_rec *r,
}
else {
ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, APLOGNO(00892)
"got response from %pI (%s:%d)",
conn->worker->cp->addr,
conn->worker->s->hostname_ex,
(int)conn->worker->s->port);
"got response from %pI (%s:%hu)",
conn->addr, conn->hostname, conn->port);

if (ap_proxy_should_override(conf, r->status)) {
/* clear r->status for override error, otherwise ErrorDocument
Expand All @@ -701,10 +693,8 @@ static int ap_proxy_ajp_request(apr_pool_t *p, request_rec *r,

if (backend_failed) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, status, r, APLOGNO(00893)
"dialog to %pI (%s:%d) failed",
conn->worker->cp->addr,
conn->worker->s->hostname_ex,
(int)conn->worker->s->port);
"dialog to %pI (%s:%hu) failed",
conn->addr, conn->hostname, conn->port);
/*
* If we already send data, signal a broken backend connection
* upwards in the chain.
Expand Down Expand Up @@ -846,9 +836,8 @@ static int proxy_ajp_handler(request_rec *r, proxy_worker *worker,
if (status != APR_SUCCESS) {
backend->close = 1;
ap_log_rerror(APLOG_MARK, APLOG_ERR, status, r, APLOGNO(00897)
"cping/cpong failed to %pI (%s:%d)",
worker->cp->addr, worker->s->hostname_ex,
(int)worker->s->port);
"cping/cpong failed to %pI (%s:%hu)",
backend->addr, backend->hostname, backend->port);
status = HTTP_SERVICE_UNAVAILABLE;
retry++;
continue;
Expand Down
99 changes: 32 additions & 67 deletions modules/proxy/mod_proxy_ftp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -975,13 +975,8 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,
conn_rec *c = r->connection;
proxy_conn_rec *backend;
apr_socket_t *sock, *local_sock, *data_sock = NULL;
apr_sockaddr_t *connect_addr = NULL;
apr_status_t rv;
conn_rec *origin, *data = NULL;
apr_status_t err = APR_SUCCESS;
#if APR_HAS_THREADS
apr_status_t uerr = APR_SUCCESS;
#endif
apr_bucket_brigade *bb;
char *buf, *connectname;
apr_port_t connectport;
Expand All @@ -1005,8 +1000,8 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,
/* stuff for PASV mode */
int connect = 0, use_port = 0;
char dates[APR_RFC822_DATE_LEN];
apr_status_t rv;
int status;
apr_pool_t *address_pool;

/* is this for us? */
if (proxyhost) {
Expand Down Expand Up @@ -1120,53 +1115,8 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,
ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, APLOGNO(01036)
"connecting %s to %s:%d", url, connectname, connectport);

if (worker->s->is_address_reusable) {
if (!worker->cp->addr) {
#if APR_HAS_THREADS
if ((err = PROXY_THREAD_LOCK(worker)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, err, r, APLOGNO(01037) "lock");
return HTTP_INTERNAL_SERVER_ERROR;
}
#endif
}
connect_addr = AP_VOLATILIZE_T(apr_sockaddr_t *, worker->cp->addr);
address_pool = worker->cp->dns_pool;
}
else
address_pool = r->pool;

/* do a DNS lookup for the destination host */
if (!connect_addr)
err = apr_sockaddr_info_get(&(connect_addr),
connectname, APR_UNSPEC,
connectport, 0,
address_pool);
if (worker->s->is_address_reusable && !worker->cp->addr) {
worker->cp->addr = connect_addr;
#if APR_HAS_THREADS
if ((uerr = PROXY_THREAD_UNLOCK(worker)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, uerr, r, APLOGNO(01038) "unlock");
}
#endif
}
/*
* get all the possible IP addresses for the destname and loop through
* them until we get a successful connection
*/
if (APR_SUCCESS != err) {
return ap_proxyerror(r, HTTP_BAD_GATEWAY, apr_pstrcat(p,
"DNS lookup failure for: ",
connectname, NULL));
}

/* check if ProxyBlock directive on this host */
if (OK != ap_proxy_checkproxyblock(r, conf, connectname, connect_addr)) {
return ap_proxyerror(r, HTTP_FORBIDDEN,
"Connect to remote machine blocked");
}

/* create space for state information */
backend = (proxy_conn_rec *) ap_get_module_config(c->conn_config, &proxy_ftp_module);
backend = ap_get_module_config(c->conn_config, &proxy_ftp_module);
if (!backend) {
status = ap_proxy_acquire_connection("FTP", &backend, worker, r->server);
if (status != OK) {
Expand All @@ -1176,23 +1126,34 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,
}
return status;
}
/* TODO: see if ftp could use determine_connection */
backend->addr = connect_addr;
ap_set_module_config(c->conn_config, &proxy_ftp_module, backend);
}

/*
* get all the possible IP addresses for the destname and loop through
* them until we get a successful connection
*/
err = ap_proxy_determine_address("FTP", backend, connectname, connectport,
0, r, r->server);
if (APR_SUCCESS != err) {
return ftp_proxyerror(r, backend, HTTP_BAD_GATEWAY,
"Error resolving backend address");
}

/* check if ProxyBlock directive on this host */
if (OK != ap_proxy_checkproxyblock(r, conf, connectname, backend->addr)) {
return ftp_proxyerror(r, backend, HTTP_FORBIDDEN,
"Connect to remote machine blocked");
}


/*
* II: Make the Connection -----------------------
*
* We have determined who to connect to. Now make the connection.
*/


if (ap_proxy_connect_backend("FTP", backend, worker, r->server)) {
ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, APLOGNO(01039)
"an error occurred creating a new connection to %pI (%s)",
connect_addr, connectname);
proxy_ftp_cleanup(r, backend);
return HTTP_SERVICE_UNAVAILABLE;
}
Expand Down Expand Up @@ -1536,7 +1497,8 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,
"PASV contacting host %d.%d.%d.%d:%d",
h3, h2, h1, h0, pasvport);

if ((rv = apr_socket_create(&data_sock, connect_addr->family, SOCK_STREAM, 0, r->pool)) != APR_SUCCESS) {
if ((rv = apr_socket_create(&data_sock, backend->addr->family,
SOCK_STREAM, 0, r->pool)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, rv, r, APLOGNO(01045)
"error creating PASV socket");
proxy_ftp_cleanup(r, backend);
Expand All @@ -1558,11 +1520,13 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,
}

/* make the connection */
err = apr_sockaddr_info_get(&pasv_addr, apr_psprintf(p, "%d.%d.%d.%d", h3, h2, h1, h0), connect_addr->family, pasvport, 0, p);
err = apr_sockaddr_info_get(&pasv_addr, apr_psprintf(p, "%d.%d.%d.%d",
h3, h2, h1, h0),
backend->addr->family, pasvport, 0, p);
if (APR_SUCCESS != err) {
return ap_proxyerror(r, HTTP_BAD_GATEWAY, apr_pstrcat(p,
"DNS lookup failure for: ",
connectname, NULL));
return ftp_proxyerror(r, backend, HTTP_BAD_GATEWAY,
apr_pstrcat(p, "DNS lookup failure for: ",
connectname, NULL));
}
rv = apr_socket_connect(data_sock, pasv_addr);
if (rv != APR_SUCCESS) {
Expand All @@ -1586,7 +1550,8 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,
apr_port_t local_port;
unsigned int h0, h1, h2, h3, p0, p1;

if ((rv = apr_socket_create(&local_sock, connect_addr->family, SOCK_STREAM, 0, r->pool)) != APR_SUCCESS) {
if ((rv = apr_socket_create(&local_sock, backend->addr->family,
SOCK_STREAM, 0, r->pool)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, rv, r, APLOGNO(01049)
"error creating local socket");
proxy_ftp_cleanup(r, backend);
Expand All @@ -1608,9 +1573,9 @@ static int proxy_ftp_handler(request_rec *r, proxy_worker *worker,

err = apr_sockaddr_info_get(&local_addr, local_ip, APR_UNSPEC, local_port, 0, r->pool);
if (APR_SUCCESS != err) {
return ap_proxyerror(r, HTTP_BAD_GATEWAY, apr_pstrcat(p,
"DNS lookup failure for: ",
connectname, NULL));
return ftp_proxyerror(r, backend, HTTP_BAD_GATEWAY,
apr_pstrcat(p, "DNS lookup failure for: ",
connectname, NULL));
}

if ((rv = apr_socket_bind(local_sock, local_addr)) != APR_SUCCESS) {
Expand Down
Loading

0 comments on commit 3c7f67f

Please sign in to comment.