From f00476a192a37427da37d2cbe8bcc1acad1306d4 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Fri, 26 Jan 2024 07:45:20 -0800 Subject: [PATCH 1/4] broker: add FLUX_IPADDR_INTERFACE environment var Problem: there is no way to force a flux instance to use a specific network interface for overlay communication. Add an environment variable: FLUX_IPADDR_INTERFACE. If set to an interface name, that will select the preferred network. --- src/common/libutil/ipaddr.c | 47 ++++++++++++++++++++++++++++++------- src/common/libutil/ipaddr.h | 3 +++ 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/common/libutil/ipaddr.c b/src/common/libutil/ipaddr.c index a9203375eecd..05f6d1b2c1bb 100644 --- a/src/common/libutil/ipaddr.c +++ b/src/common/libutil/ipaddr.c @@ -100,16 +100,17 @@ static struct ifaddrs *find_ifaddr (struct ifaddrs *ifaddr, return ifa; } -static int getprimary_ifaddr (char *buf, int len, int prefer_family, - char *errstr, int errstrsz) +static int getnamed_ifaddr (char *buf, + int len, + const char *name, + int prefer_family, + char *errstr, + int errstrsz) { struct ifaddrs *ifaddr; struct ifaddrs *ifa; - char name[64]; int error; - if (getprimary_iface4 (name, sizeof (name), errstr, errstrsz) < 0) - return -1; if (getifaddrs (&ifaddr) < 0) { esprintf (errstr, errstrsz, "getifaddrs: %s", strerror (errno)); return -1; @@ -141,6 +142,18 @@ static int getprimary_ifaddr (char *buf, int len, int prefer_family, return 0; } +static int getprimary_ifaddr (char *buf, + int len, + int prefer_family, + char *errstr, + int errstrsz) +{ + char name[64]; + if (getprimary_iface4 (name, sizeof (name), errstr, errstrsz) < 0) + return -1; + return getnamed_ifaddr (buf, len, name, prefer_family, errstr, errstrsz); +} + static struct addrinfo *find_addrinfo (struct addrinfo *addrinfo, int family) { struct addrinfo *ai; @@ -205,12 +218,28 @@ int ipaddr_getprimary (char *buf, int len, char *errstr, int errstrsz) { int prefer_family = getenv ("FLUX_IPADDR_V6") ? AF_INET6 : AF_INET; + const char *iface_name; int rc = -1; - if (getenv ("FLUX_IPADDR_HOSTNAME") == NULL) - rc = getprimary_ifaddr (buf, len, prefer_family, errstr, errstrsz); - if (rc < 0) - rc = getprimary_hostaddr (buf, len, prefer_family, errstr, errstrsz); + if ((iface_name = getenv ("FLUX_IPADDR_INTERFACE"))) { + rc = getnamed_ifaddr (buf, + len, + iface_name, + prefer_family, + errstr, + errstrsz); + } + else { + if (getenv ("FLUX_IPADDR_HOSTNAME") == NULL) + rc = getprimary_ifaddr (buf, len, prefer_family, errstr, errstrsz); + if (rc < 0) { + rc = getprimary_hostaddr (buf, + len, + prefer_family, + errstr, + errstrsz); + } + } return rc; } diff --git a/src/common/libutil/ipaddr.h b/src/common/libutil/ipaddr.h index 9a31062d0c21..fdfc7de783e0 100644 --- a/src/common/libutil/ipaddr.h +++ b/src/common/libutil/ipaddr.h @@ -18,6 +18,7 @@ * 1. Find the interface associated with the default route, then * look up address of that interface. * 2. Look up address associated with the hostname + * 3. Look up address associated with a specific interface. * * Main use case: determine bind address for a PMI-bootstrapped flux broker. * @@ -29,6 +30,8 @@ * FLUX_IPADDR_HOSTNAME * if set, only method 2 is tried above * if unset, first method 1 is tried, then if that fails, method 2 is tried + * FLUX_IPADDR_INTERFACE + * if set, only method 3 is tried above * * Return address as a string in buf (up to len bytes, always null terminated) * Return 0 on success, -1 on error with error message written to errstr From 20015bf609ccfe1be212be0e703dabace83e396c Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Fri, 26 Jan 2024 08:05:50 -0800 Subject: [PATCH 2/4] flux-environment(7): add FLUX_IPADDR_INTERFACE Problem: FLUX_IPADDR_INTERFACE is not documented. Add it to the envrironment man page. --- doc/man7/flux-environment.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/man7/flux-environment.rst b/doc/man7/flux-environment.rst index ef75149d0610..6c68ce51d6ec 100644 --- a/doc/man7/flux-environment.rst +++ b/doc/man7/flux-environment.rst @@ -229,6 +229,11 @@ affect the broker's PMI client. version 4 addresses. Setting this variable to any value in the broker's environment causes it to prefer version 6 addresses. +.. envvar:: FLUX_IPADDR_HOSTNAME + + Force PMI bootstrap to assign the broker an address associated with a + particular network interface, like ``eth0``. + CUSTOM OUTPUT FORMATS ===================== From 259d90dc060c0eceba213d8d2034ca3651a5eb53 Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Mon, 29 Jan 2024 13:12:53 -0800 Subject: [PATCH 3/4] testsuite: cover FLUX_IPADDR_INTERFACE Problem: there is no test coverage for the FLUX_IPADDR_INTERFACE environment variable. Add a couple checks. --- t/t0001-basic.t | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t0001-basic.t b/t/t0001-basic.t index 5c45ae19aef4..7f6e9702292a 100755 --- a/t/t0001-basic.t +++ b/t/t0001-basic.t @@ -429,6 +429,17 @@ test_expect_success 'tbon.endpoint uses tcp:// if tbon.prefertcp is set' ' flux getattr tbon.endpoint >endpoint2.out && grep "^tcp" endpoint2.out ' +test_expect_success 'FLUX_IPADDR_INTERFACE=lo works' ' + FLUX_IPADDR_INTERFACE=lo flux start \ + ${ARGS} -s2 -o,-Stbon.prefertcp=1 \ + flux getattr tbon.endpoint >endpoint3.out && + grep "127.0.0.1" endpoint3.out +' +test_expect_success 'FLUX_IPADDR_INTERFACE=badiface fails' ' + (export FLUX_IPADDR_INTERFACE=badiface; \ + test_expect_code 137 flux start \ + ${ARGS} --test-exit-timeout=1s -s2 -o,-Stbon.prefertcp=1 true) +' test_expect_success 'tbon.endpoint cannot be set' ' test_must_fail flux start ${ARGS} -s2 \ -o,--setattr=tbon.endpoint=ipc:///tmp/customflux /bin/true From fa4fdf5c3b5d978b6296f3c22fbe3097eae75c0e Mon Sep 17 00:00:00 2001 From: Jim Garlick Date: Mon, 29 Jan 2024 13:15:10 -0800 Subject: [PATCH 4/4] broker: clean up ipaddr log message in PMI boot Problem: when ipaddr_getprimary() fails during PMI bootstrap, the broker prints the textual error plus, unnecessarily, the errno string. This results in messages like this: flux-broker: could not find address of badiface: Success Drop the errno string suffix. --- src/broker/boot_pmi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/broker/boot_pmi.c b/src/broker/boot_pmi.c index 167f736ceedb..d30073a0749b 100644 --- a/src/broker/boot_pmi.c +++ b/src/broker/boot_pmi.c @@ -151,7 +151,7 @@ static int format_bind_uri (char *buf, int bufsz, attr_t *attrs, int rank) if (ipaddr_getprimary (ipaddr, sizeof (ipaddr), error, sizeof (error)) < 0) { - log_err ("%s", error); + log_msg ("%s", error); return -1; } if (snprintf (buf, bufsz, "tcp://%s:*", ipaddr) >= bufsz)