You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
302 lines
11 KiB
302 lines
11 KiB
commit 937604b4cfccddd607b8d4883815c4e3f9ab70d0
|
|
Author: Willy Tarreau <w@1wt.eu>
|
|
Date: Wed Jul 24 16:45:02 2019 +0200
|
|
|
|
BUG/MEDIUM: protocols: add a global lock for the init/deinit stuff
|
|
|
|
Dragan Dosen found that the listeners lock is not sufficient to protect
|
|
the listeners list when proxies are stopping because the listeners are
|
|
also unlinked from the protocol list, and under certain situations like
|
|
bombing with soft-stop signals or shutting down many frontends in parallel
|
|
from multiple CLI connections, it could be possible to provoke multiple
|
|
instances of delete_listener() to be called in parallel for different
|
|
listeners, thus corrupting the protocol lists.
|
|
|
|
Such operations are pretty rare, they are performed once per proxy upon
|
|
startup and once per proxy on shut down. Thus there is no point trying
|
|
to optimize anything and we can use a global lock to protect the protocol
|
|
lists during these manipulations.
|
|
|
|
This fix (or a variant) will have to be backported as far as 1.8.
|
|
|
|
(cherry picked from commit daacf3664506d56a1f3b050ccba504886a18b12a)
|
|
Signed-off-by: Christopher Faulet <cfaulet@haproxy.com>
|
|
|
|
diff --git a/include/proto/protocol.h b/include/proto/protocol.h
|
|
index 7bbebb8e..f25f77f0 100644
|
|
--- a/include/proto/protocol.h
|
|
+++ b/include/proto/protocol.h
|
|
@@ -23,9 +23,11 @@
|
|
#define _PROTO_PROTOCOL_H
|
|
|
|
#include <sys/socket.h>
|
|
+#include <common/hathreads.h>
|
|
#include <types/protocol.h>
|
|
|
|
extern struct protocol *__protocol_by_family[AF_CUST_MAX];
|
|
+__decl_hathreads(extern HA_SPINLOCK_T proto_lock);
|
|
|
|
/* Registers the protocol <proto> */
|
|
void protocol_register(struct protocol *proto);
|
|
diff --git a/include/types/protocol.h b/include/types/protocol.h
|
|
index 1d3404b9..f38baeb9 100644
|
|
--- a/include/types/protocol.h
|
|
+++ b/include/types/protocol.h
|
|
@@ -80,9 +80,9 @@ struct protocol {
|
|
int (*pause)(struct listener *l); /* temporarily pause this listener for a soft restart */
|
|
void (*add)(struct listener *l, int port); /* add a listener for this protocol and port */
|
|
|
|
- struct list listeners; /* list of listeners using this protocol */
|
|
- int nb_listeners; /* number of listeners */
|
|
- struct list list; /* list of registered protocols */
|
|
+ struct list listeners; /* list of listeners using this protocol (under proto_lock) */
|
|
+ int nb_listeners; /* number of listeners (under proto_lock) */
|
|
+ struct list list; /* list of registered protocols (under proto_lock) */
|
|
};
|
|
|
|
#define CONNECT_HAS_DATA 0x00000001 /* There's data available to be sent */
|
|
diff --git a/src/listener.c b/src/listener.c
|
|
index 40a774ed..b5fe2ac2 100644
|
|
--- a/src/listener.c
|
|
+++ b/src/listener.c
|
|
@@ -433,6 +433,9 @@ static void limit_listener(struct listener *l, struct list *list)
|
|
* used as a protocol's generic enable_all() primitive, for use after the
|
|
* fork(). It puts the listeners into LI_READY or LI_FULL states depending on
|
|
* their number of connections. It always returns ERR_NONE.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
int enable_all_listeners(struct protocol *proto)
|
|
{
|
|
@@ -447,6 +450,9 @@ int enable_all_listeners(struct protocol *proto)
|
|
* the polling lists when they are in the LI_READY or LI_FULL states. It is
|
|
* intended to be used as a protocol's generic disable_all() primitive. It puts
|
|
* the listeners into LI_LISTEN, and always returns ERR_NONE.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
int disable_all_listeners(struct protocol *proto)
|
|
{
|
|
@@ -516,6 +522,9 @@ void unbind_listener_no_close(struct listener *listener)
|
|
/* This function closes all listening sockets bound to the protocol <proto>,
|
|
* and the listeners end in LI_ASSIGNED state if they were higher. It does not
|
|
* detach them from the protocol. It always returns ERR_NONE.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
int unbind_all_listeners(struct protocol *proto)
|
|
{
|
|
@@ -580,14 +589,19 @@ int create_listeners(struct bind_conf *bc, const struct sockaddr_storage *ss,
|
|
* number of listeners is updated, as well as the global number of listeners
|
|
* and jobs. Note that the listener must have previously been unbound. This
|
|
* is the generic function to use to remove a listener.
|
|
+ *
|
|
+ * Will grab the proto_lock.
|
|
+ *
|
|
*/
|
|
void delete_listener(struct listener *listener)
|
|
{
|
|
HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock);
|
|
if (listener->state == LI_ASSIGNED) {
|
|
listener->state = LI_INIT;
|
|
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
|
|
LIST_DEL(&listener->proto_list);
|
|
listener->proto->nb_listeners--;
|
|
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
|
|
_HA_ATOMIC_SUB(&jobs, 1);
|
|
_HA_ATOMIC_SUB(&listeners, 1);
|
|
}
|
|
diff --git a/src/proto_sockpair.c b/src/proto_sockpair.c
|
|
index a4faa370..e7dd670d 100644
|
|
--- a/src/proto_sockpair.c
|
|
+++ b/src/proto_sockpair.c
|
|
@@ -80,6 +80,9 @@ INITCALL1(STG_REGISTER, protocol_register, &proto_sockpair);
|
|
/* Add <listener> to the list of sockpair listeners (port is ignored). The
|
|
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
|
|
* The number of listeners for the protocol is updated.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
static void sockpair_add_listener(struct listener *listener, int port)
|
|
{
|
|
@@ -97,6 +100,8 @@ static void sockpair_add_listener(struct listener *listener, int port)
|
|
* loose them across the fork(). A call to uxst_enable_listeners() is needed
|
|
* to complete initialization.
|
|
*
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
* The return value is composed from ERR_NONE, ERR_RETRYABLE and ERR_FATAL.
|
|
*/
|
|
static int sockpair_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
|
|
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
|
|
index 64ffb83c..bcbe27a7 100644
|
|
--- a/src/proto_tcp.c
|
|
+++ b/src/proto_tcp.c
|
|
@@ -1103,6 +1103,9 @@ int tcp_bind_listener(struct listener *listener, char *errmsg, int errlen)
|
|
* The sockets will be registered but not added to any fd_set, in order not to
|
|
* loose them across the fork(). A call to enable_all_listeners() is needed
|
|
* to complete initialization. The return value is composed from ERR_*.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
static int tcp_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
|
|
{
|
|
@@ -1121,6 +1124,9 @@ static int tcp_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
|
|
/* Add <listener> to the list of tcpv4 listeners, on port <port>. The
|
|
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
|
|
* The number of listeners for the protocol is updated.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
static void tcpv4_add_listener(struct listener *listener, int port)
|
|
{
|
|
@@ -1136,6 +1142,9 @@ static void tcpv4_add_listener(struct listener *listener, int port)
|
|
/* Add <listener> to the list of tcpv6 listeners, on port <port>. The
|
|
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
|
|
* The number of listeners for the protocol is updated.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
static void tcpv6_add_listener(struct listener *listener, int port)
|
|
{
|
|
diff --git a/src/proto_uxst.c b/src/proto_uxst.c
|
|
index 66093af6..7263240f 100644
|
|
--- a/src/proto_uxst.c
|
|
+++ b/src/proto_uxst.c
|
|
@@ -379,6 +379,9 @@ static int uxst_unbind_listener(struct listener *listener)
|
|
/* Add <listener> to the list of unix stream listeners (port is ignored). The
|
|
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
|
|
* The number of listeners for the protocol is updated.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
static void uxst_add_listener(struct listener *listener, int port)
|
|
{
|
|
@@ -594,6 +597,8 @@ static int uxst_connect_server(struct connection *conn, int flags)
|
|
* loose them across the fork(). A call to uxst_enable_listeners() is needed
|
|
* to complete initialization.
|
|
*
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
* The return value is composed from ERR_NONE, ERR_RETRYABLE and ERR_FATAL.
|
|
*/
|
|
static int uxst_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
|
|
@@ -613,6 +618,9 @@ static int uxst_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
|
|
/* This function stops all listening UNIX sockets bound to the protocol
|
|
* <proto>. It does not detaches them from the protocol.
|
|
* It always returns ERR_NONE.
|
|
+ *
|
|
+ * Must be called with proto_lock held.
|
|
+ *
|
|
*/
|
|
static int uxst_unbind_listeners(struct protocol *proto)
|
|
{
|
|
diff --git a/src/protocol.c b/src/protocol.c
|
|
index 96e01c82..ac45cf2e 100644
|
|
--- a/src/protocol.c
|
|
+++ b/src/protocol.c
|
|
@@ -18,18 +18,26 @@
|
|
#include <common/mini-clist.h>
|
|
#include <common/standard.h>
|
|
|
|
-#include <types/protocol.h>
|
|
+#include <proto/protocol.h>
|
|
|
|
/* List head of all registered protocols */
|
|
static struct list protocols = LIST_HEAD_INIT(protocols);
|
|
struct protocol *__protocol_by_family[AF_CUST_MAX] = { };
|
|
|
|
+/* This is the global spinlock we may need to register/unregister listeners or
|
|
+ * protocols. Its main purpose is in fact to serialize the rare stop/deinit()
|
|
+ * phases.
|
|
+ */
|
|
+__decl_spinlock(proto_lock);
|
|
+
|
|
/* Registers the protocol <proto> */
|
|
void protocol_register(struct protocol *proto)
|
|
{
|
|
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
|
|
LIST_ADDQ(&protocols, &proto->list);
|
|
if (proto->sock_domain >= 0 && proto->sock_domain < AF_CUST_MAX)
|
|
__protocol_by_family[proto->sock_domain] = proto;
|
|
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
|
|
}
|
|
|
|
/* Unregisters the protocol <proto>. Note that all listeners must have
|
|
@@ -37,8 +45,10 @@ void protocol_register(struct protocol *proto)
|
|
*/
|
|
void protocol_unregister(struct protocol *proto)
|
|
{
|
|
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
|
|
LIST_DEL(&proto->list);
|
|
LIST_INIT(&proto->list);
|
|
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
|
|
}
|
|
|
|
/* binds all listeners of all registered protocols. Returns a composition
|
|
@@ -50,6 +60,7 @@ int protocol_bind_all(char *errmsg, int errlen)
|
|
int err;
|
|
|
|
err = 0;
|
|
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
|
|
list_for_each_entry(proto, &protocols, list) {
|
|
if (proto->bind_all) {
|
|
err |= proto->bind_all(proto, errmsg, errlen);
|
|
@@ -57,6 +68,7 @@ int protocol_bind_all(char *errmsg, int errlen)
|
|
break;
|
|
}
|
|
}
|
|
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
|
|
return err;
|
|
}
|
|
|
|
@@ -71,11 +83,13 @@ int protocol_unbind_all(void)
|
|
int err;
|
|
|
|
err = 0;
|
|
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
|
|
list_for_each_entry(proto, &protocols, list) {
|
|
if (proto->unbind_all) {
|
|
err |= proto->unbind_all(proto);
|
|
}
|
|
}
|
|
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
|
|
return err;
|
|
}
|
|
|
|
@@ -89,11 +103,13 @@ int protocol_enable_all(void)
|
|
int err;
|
|
|
|
err = 0;
|
|
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
|
|
list_for_each_entry(proto, &protocols, list) {
|
|
if (proto->enable_all) {
|
|
err |= proto->enable_all(proto);
|
|
}
|
|
}
|
|
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
|
|
return err;
|
|
}
|
|
|
|
@@ -107,11 +123,13 @@ int protocol_disable_all(void)
|
|
int err;
|
|
|
|
err = 0;
|
|
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
|
|
list_for_each_entry(proto, &protocols, list) {
|
|
if (proto->disable_all) {
|
|
err |= proto->disable_all(proto);
|
|
}
|
|
}
|
|
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
|
|
return err;
|
|
}
|
|
|