opticsmon: Introduce opticsmon tool

The optics monitoring tool opticsmon implements the user-space portion
of reporting optics data to the SE. Its basic functionality is to
collect optical module information equivalent to "ethtool --module-info"
for PCI Physical Functions and forwards this data to the SE using the
new SCLP Write Event Data Action Qualifier 3.

For the part of finding all PFs we need to look at all PCI
functions and determine which ones are PFs and what netdevs they
correspond to. This is a generally useful functionality so this part as
well as the SCLP issuing code go into a new libzpci library which also
includes a standalone example for listing PCI functions and their s390x
specific attributes. Medium term we plan to add this functionality to
lszdev.

For the opticsmon tool itself there are 2 basic operating modes:

* One-shot Mode: Without parameters opticsmon collects optical module
  data and prints a summary of the netdevice in JSON format. With
  --module-data it also includes a base64 encoded raw dump equivalent to
  ethtool --module-info <netdev> raw on.
* Monitor Mode: With the --monitor flag opticsmon runs continuously
  usually started via a systemd unit and collects new optical module
  data on a time interval (default 24h) or when the operational state
  ("/sys/class/net/<netdev/operstate") changes. The tool listens for
  changes via netlink so no polling on sysfs is necessary

Note: Both modes will *NOT* issues SCLPs without adding the
--send-report flag but will output a JSON summary for each data
collection so can be tested without firmware impact.

Reviewed-by: Halil Pasic <[email protected]>
Signed-off-by: Niklas Schnelle <[email protected]>
Signed-off-by: Jan Höppner <[email protected]>

Makefile

@@ -15,7 +15,7 @@ TOOL_DIRS = zipl zdump fdasd dasdfmt dasdview tunedasd \
 	   vmcp man mon_tools dasdinfo vmur cpuplugd ipl_tools \
 	   ziomon iucvterm hyptop cmsfs-fuse qethqoat zfcpdump zdsfs cpumf \
 	   systemd hmcdrvfs cpacfstats zdev dump2tar zkey netboot etc zpcictl \
-	   lsstp hsci hsavmcore chreipl-fcp-mpath ap_tools rust
+	   lsstp hsci hsavmcore chreipl-fcp-mpath ap_tools rust opticsmon
 
 else
 BASELIB_DIRS =

include/lib/pci_sclp.h

@@ -20,8 +20,10 @@
 #define SCLP_ERRNOTIFY_AQ_RESET			0
 #define SCLP_ERRNOTIFY_AQ_DECONF		1
 #define SCLP_ERRNOTIFY_AQ_REPORT_ERR		2
+#define SCLP_ERRNOTIFY_AQ_OPTICS_DATA		3
 
 #define SCLP_ERRNOTIFY_ID_ZPCICTL		0x4713
+#define SCLP_ERRNOTIFY_ID_OPTICSMON		0x4714
 
 #define SCLP_ERRNOTIFY_DATA_SIZE		4054
 
@@ -31,6 +33,7 @@ struct zpci_report_error_header {
 			 * 0: Adapter Reset Request
 			 * 1: Deconfigure and repair action requested
 			 * 2: Informational Report
+			 * 3: Optics Data
 			 */
 	__u16 length;	/* Length of Subsequent Data (up to 4K – SCLP header) */
 } __packed;

opticsmon/Makefile

@@ -0,0 +1,83 @@
+include ../common.mak
+
+TESTS := tests/
+
+libs =$(rootdir)/libzpci/libzpci.a $(rootdir)/libutil/libutil.a
+
+ifneq (${HAVE_OPENSSL},0)
+check_dep_openssl:
+	$(call check_dep, \
+		"opticsmon", \
+		"openssl/evp.h", \
+		"openssl-devel", \
+		"HAVE_OPENSSL=0")
+BUILDTARGET += check_dep_openssl
+endif # HAVE_OPENSSL
+
+ifneq (${HAVE_LIBNL3},0)
+check_dep_libnl3:
+	$(call check_dep, \
+		"opticsmon", \
+		"netlink/socket.h", \
+		"libnl3-devel", \
+		"HAVE_LIBNL3=0")
+BUILDTARGET += check_dep_libnl3
+endif # HAVE_LIBNL3
+
+ifeq (${HAVE_OPENSSL},0)
+
+all:
+	$(SKIP) HAVE_OPENSSL=0
+
+install:
+	$(SKIP) HAVE_OPENSSL=0
+
+else ifeq (${HAVE_LIBNL3},0)
+all:
+	$(SKIP) HAVE_LIBNL3=0
+
+install:
+	$(SKIP) HAVE_LIBNL3=0
+
+else
+
+ifneq ($(shell sh -c 'command -v pkg-config'),)
+LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-3.0)
+LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-genl-3.0)
+LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libnl-route-3.0)
+
+LIB_CFLAGS += $(shell pkg-config --silence-errors --cflags libcrypto)
+
+LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-3.0)
+LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-genl-3.0)
+LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libnl-route-3.0)
+
+LIB_LFLAGS += $(shell pkg-config --silence-errors --libs libcrypto)
+else
+LIB_CFLAGS += -I /usr/include/libnl3/
+LIB_LFLAGS += -lnl-route-3 -lnl-genl-3 -lnl-3
+
+LIB_CFLAGS += -I /usr/include/openssl/
+LIB_LFLAGS += -lcrypto
+endif
+
+ALL_CPPFLAGS += $(LIB_CFLAGS)
+LDLIBS += $(LIB_LFLAGS)
+
+BUILDTARGET += opticsmon
+
+all: ${BUILDTARGET}
+
+opticsmon: opticsmon.o optics_info.o optics_sclp.o ethtool.o link_mon.o $(libs)
+
+install: all
+	$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR) $(DESTDIR)$(MANDIR)/man8
+	$(INSTALL) -g $(GROUP) -o $(OWNER) -m 755 opticsmon $(DESTDIR)$(BINDIR)
+	$(INSTALL) -g $(GROUP) -o $(OWNER) -m 644 opticsmon.8 \
+		$(DESTDIR)$(MANDIR)/man8
+endif # HAVE_OPENSSL3=0 or HAVE_LIBNL3=0
+
+clean:
+	rm -f *.o *~ opticsmon core
+
+.PHONY: all install clean

opticsmon/ethtool.c

@@ -0,0 +1,279 @@
+#include <execinfo.h>
+#include <sys/wait.h>
+#include <sys/socket.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include <linux/netlink.h>
+#include <linux/ethtool_netlink.h>
+
+#include <netlink/socket.h>
+#include <netlink/msg.h>
+#include <netlink/genl/ctrl.h>
+#include <netlink/genl/genl.h>
+#include <netlink/handlers.h>
+#include <netlink/attr.h>
+
+#include "lib/util_libc.h"
+
+#include "ethtool.h"
+
+static int ethtool_nl_cb(struct nl_msg *msg, void *arg)
+{
+	struct nlattr *attrs[ETHTOOL_A_MODULE_EEPROM_DATA + 1] = {};
+	struct nlmsghdr *hdr = nlmsg_hdr(msg);
+	struct optics **oi = arg;
+	int rc = 0;
+	size_t len;
+
+	rc = genlmsg_parse(hdr, 0, attrs, ETHTOOL_A_MODULE_EEPROM_DATA, NULL);
+	if (rc) {
+		nl_perror(rc, "genlmsg parse");
+		return NL_STOP;
+	}
+
+	len = nla_len(attrs[ETHTOOL_A_MODULE_EEPROM_DATA]);
+
+	/* Extend optics info*/
+	if (!(*oi)->raw)
+		(*oi)->raw = util_malloc(len);
+	else
+		(*oi)->raw = util_realloc((*oi)->raw, (*oi)->size + len);
+	memcpy((*oi)->raw + (*oi)->size, nla_data(attrs[ETHTOOL_A_MODULE_EEPROM_DATA]), len);
+	(*oi)->size += len;
+
+	return NL_OK;
+}
+
+int ethtool_nl_connect(struct ethtool_nl_ctx *ctx)
+{
+	struct nl_sock *sk;
+	int ethtool_id;
+	int rc = 0;
+
+	sk = nl_socket_alloc();
+	if (!sk) {
+		nl_perror(NLE_NOMEM, "alloc");
+		return EXIT_FAILURE;
+	}
+
+	rc = genl_connect(sk);
+	if (rc) {
+		nl_perror(rc, "connect");
+		rc = EXIT_FAILURE;
+		goto err_free;
+	}
+
+	ethtool_id = genl_ctrl_resolve(sk, ETHTOOL_GENL_NAME);
+	if (ethtool_id < 0) {
+		if (ethtool_id == -NLE_OBJ_NOTFOUND)
+			fprintf(stderr, "Ethtool netlink family not found\n");
+		else
+			nl_perror(ethtool_id, "ctrl resolve");
+		rc = EXIT_FAILURE;
+		goto err_close;
+	}
+	ctx->sk = sk;
+	ctx->ethtool_id = ethtool_id;
+	return rc;
+
+err_close:
+	nl_close(sk);
+err_free:
+	nl_socket_free(sk);
+	return rc;
+}
+
+void ethtool_nl_close(struct ethtool_nl_ctx *ctx)
+{
+	nl_close(ctx->sk);
+	nl_socket_free(ctx->sk);
+}
+
+static int ethtool_nl_put_req_hdr(struct ethtool_nl_ctx *ctx, struct nl_msg *msg, uint8_t cmd,
+				  const char *netdev)
+{
+	struct nlattr *opts;
+	void *user_hdr;
+	int rc = 0;
+
+	user_hdr = genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, ctx->ethtool_id, 0,
+			       NLM_F_REQUEST | NLM_F_ACK, cmd, ETHTOOL_GENL_VERSION);
+	if (!user_hdr) {
+		fprintf(stderr, "genlmsg put failed\n");
+		return EXIT_FAILURE;
+	}
+
+	opts = nla_nest_start(msg, ETHTOOL_A_MODULE_EEPROM_HEADER);
+	if (!opts) {
+		fprintf(stderr, "nla nest for start failed\n");
+		return EXIT_FAILURE;
+	}
+
+	NLA_PUT_STRING(msg, ETHTOOL_A_HEADER_DEV_NAME, netdev);
+	nla_nest_end(msg, opts);
+	return rc;
+
+nla_put_failure:
+	nla_nest_cancel(msg, opts);
+	return EXIT_FAILURE;
+}
+
+static int ethtool_nl_put_eeprom_get_attrs(struct nl_msg *msg, uint8_t addr, uint8_t page,
+					   uint32_t offset)
+{
+	NLA_PUT_U32(msg, ETHTOOL_A_MODULE_EEPROM_LENGTH, SFF8636_PAGE_SIZE);
+	NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_PAGE, page);
+	NLA_PUT_U32(msg, ETHTOOL_A_MODULE_EEPROM_OFFSET, offset);
+	NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_BANK, 0);
+	NLA_PUT_U8(msg, ETHTOOL_A_MODULE_EEPROM_I2C_ADDRESS, addr);
+
+	return 0;
+
+nla_put_failure:
+	return EXIT_FAILURE;
+}
+
+static int ethtool_nl_get_page(struct ethtool_nl_ctx *ctx, const char *netdev, uint8_t addr,
+			       uint8_t page, uint32_t offset)
+{
+	struct nl_msg *msg;
+	int rc = 0;
+
+	msg = nlmsg_alloc();
+	if (!msg) {
+		nl_perror(NLE_NOMEM, "nlmsg alloc");
+		return -ENOMEM;
+	}
+	ethtool_nl_put_req_hdr(ctx, msg, ETHTOOL_MSG_MODULE_EEPROM_GET, netdev);
+	ethtool_nl_put_eeprom_get_attrs(msg, addr, page, offset);
+	rc = nl_send_auto(ctx->sk, msg);
+	if (rc < 0) {
+		nl_perror(rc, "Failed to send netlink message");
+		rc = -EIO;
+		goto free_msg;
+	}
+
+	rc = nl_recvmsgs_default(ctx->sk);
+	if (rc < 0) {
+		if (rc == -NLE_NODEV) {
+			rc = -ENODEV;
+		} else {
+			nl_perror(rc, "Failed to receive netlink message");
+			rc = -EIO;
+		}
+		goto free_msg;
+	}
+
+	/* Ethtool netlink sends ACKs need to pick them up */
+	rc = nl_wait_for_ack(ctx->sk);
+	if (rc < 0) {
+		nl_perror(rc, "Failed to wait for netlink ack");
+		rc = -EIO;
+		goto free_msg;
+	}
+free_msg:
+	nlmsg_free(msg);
+	return rc;
+}
+
+static int ethtool_nl_get_sfp(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics *oi)
+{
+	int rc = 0;
+
+	/* Page A0h upper */
+	rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, SFF8636_PAGE_SIZE);
+	if (rc < 0)
+		return rc;
+
+	/* If page A2h is not present we're done */
+	if (!(oi->raw[SFF8472_DIAGNOSTICS_TYPE_OFFSET] & SFF8472_DIAGNOSTICS_TYPE_MASK))
+		return 0;
+
+	/* Page A2h lower */
+	rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_HIGH, 0x0, 0);
+	if (rc < 0)
+		return rc;
+
+	/* Page A2h upper */
+	rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_HIGH, 0x0, SFF8636_PAGE_SIZE);
+	if (rc < 0)
+		return rc;
+
+	return 0;
+}
+
+static int ethtool_nl_get_qsfp(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics *oi)
+{
+	int rc = 0;
+
+	/* Page 00h upper */
+	rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, SFF8636_PAGE_SIZE);
+	if (rc)
+		return rc;
+
+	/* Page 01h  */
+	if (oi->raw[SFF8636_PAGE_OFFSET] & SFF8636_P01H) {
+		/* Page 01h upper only */
+		rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x1,
+					 SFF8636_PAGE_SIZE);
+		if (rc < 0)
+			return rc;
+	}
+
+	/* Page 02h  */
+	if (oi->raw[SFF8636_PAGE_OFFSET] & SFF8636_P02H) {
+		/* Page 02h upper only */
+		rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x2,
+					 SFF8636_PAGE_SIZE);
+		if (rc < 0)
+			return rc;
+	}
+
+	/* Page 03h is present if flatmem is not set */
+	if (!(oi->raw[SFF8636_STATUS_2_OFFSET] & SFF8636_STATUS_FLAT_MEM)) {
+		/* Page 03h upper only */
+		rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x3,
+					 SFF8636_PAGE_SIZE);
+		if (rc < 0)
+			return rc;
+	}
+
+	return 0;
+}
+
+int ethtool_nl_get_optics(struct ethtool_nl_ctx *ctx, const char *netdev, struct optics **oi)
+{
+	int rc = 0;
+	int type;
+
+	*oi = util_zalloc(sizeof(**oi));
+	nl_socket_modify_cb(ctx->sk, NL_CB_VALID, NL_CB_CUSTOM, ethtool_nl_cb, oi);
+
+	/* Page 00h lower */
+	rc = ethtool_nl_get_page(ctx, netdev, SFF8079_I2C_ADDRESS_LOW, 0x0, 0);
+	if (rc < 0)
+		goto out_err_free_oi;
+
+	type = optics_type(*oi);
+	switch (type) {
+	case OPTICS_TYPE_SFP:
+		rc = ethtool_nl_get_sfp(ctx, netdev, *oi);
+		break;
+	case OPTICS_TYPE_QSFP28:
+		rc = ethtool_nl_get_qsfp(ctx, netdev, *oi);
+		break;
+	};
+	if (rc < 0)
+		goto out_err_free_oi;
+
+	return rc;
+
+out_err_free_oi:
+	free(*oi);
+	*oi = NULL;
+	return rc;
+}