CP-49690 Create smlib to replace sm-core-libs

The sm-core-libs package mostly contained copies of libraries from SM,
but has the same LGPL License. It did install in the correct python
site-packages location, but it meant that any change to these libraries
had to be done in two different places which was both error- and
omission-prone.

This provides the same libraries that sm-core-libs provided, by moving
the SM originals from /opt/xensource/sm into the same place sm-core-libs
was using, so that sm-core-libs can be obsoleted.

It leaves behind in /opt/xensource/sm some wrapper libraries which
import the public APIs from the moved code, which should external code to
continue working until it can be updated. It also helps with the way
iscsilib was renamed to libiscsi by sm-core-libs.

Signed-off-by: Tim Smith <[email protected]>

Author: Tim Smith

.github/workflows/main.yml

@@ -19,5 +19,5 @@ jobs:
       - name: Test
         run: |
           make precheck
-          PYTHONPATH="./mocks:./drivers/:./misc/fairlock" coverage3 run --branch --source='./drivers,./tests,./misc/fairlock' -m unittest discover -s tests -p "*.py" -v
+          PYTHONPATH="./mocks:./libs:./drivers/:./misc/fairlock" coverage3 run --branch --source='./drivers,./tests,./misc/fairlock' -m unittest discover -s tests -p "*.py" -v
           coverage3 report --include='./*'

Makefile

@@ -1,4 +1,5 @@
 PYLINT=$(shell command -v pylint-3 || echo pylint)
+PYTHONLIBDIR = $(shell python3 -c "import sys; print(sys.path.pop())")
 
 SM_DRIVERS := File
 SM_DRIVERS += NFS
@@ -14,36 +15,56 @@ SM_DRIVERS += LVHDoHBA
 SM_DRIVERS += SHM
 SM_DRIVERS += SMB
 
+# Libraries which have moved in to sm.core
+SM_CORE_LIBS := util
+SM_CORE_LIBS += scsiutil
+SM_CORE_LIBS += mpath_dmp
+SM_CORE_LIBS += mpath_cli
+SM_CORE_LIBS += xs_errors
+SM_CORE_LIBS += libiscsi
+SM_CORE_LIBS += wwid_conf
+SM_CORE_LIBS += lock
+SM_CORE_LIBS += flock
+SM_CORE_LIBS += f_exceptions
+
+# Libraries which remain in drivers/ and get installed in
+# /opt/xensource/sm as wrappers around sm.core libs for
+# backwards compatibility and can hopefully be one day dropped
+SM_COMPAT_LIBS := util
+SM_COMPAT_LIBS += scsiutil
+SM_COMPAT_LIBS += mpath_dmp
+SM_COMPAT_LIBS += mpath_cli
+SM_COMPAT_LIBS += xs_errors
+SM_COMPAT_LIBS += iscsilib
+SM_COMPAT_LIBS += wwid_conf
+SM_COMPAT_LIBS += lock
+SM_COMPAT_LIBS += flock
+
+# Libraries and other code still maintained in
+# drivers/ and installed in /opt/xensource/sm which
+# has not yet been moved elsewhere.
 SM_LIBS := SR
 SM_LIBS += SRCommand
 SM_LIBS += VDI
 SM_LIBS += BaseISCSI
 SM_LIBS += cleanup
 SM_LIBS += lvutil
 SM_LIBS += lvmcache
-SM_LIBS += util
 SM_LIBS += verifyVHDsOnSR
-SM_LIBS += scsiutil
 SM_LIBS += scsi_host_rescan
 SM_LIBS += vhdutil
 SM_LIBS += lvhdutil
 SM_LIBS += cifutils
-SM_LIBS += xs_errors
 SM_LIBS += nfs
 SM_LIBS += devscan
 SM_LIBS += sysdevice
-SM_LIBS += iscsilib
-SM_LIBS += mpath_dmp
 SM_LIBS += mpath_null
-SM_LIBS += mpath_cli
 SM_LIBS += mpathutil
 SM_LIBS += LUNperVDI
 SM_LIBS += mpathcount
 SM_LIBS += refcounter
 SM_LIBS += journaler
 SM_LIBS += fjournaler
-SM_LIBS += lock
-SM_LIBS += flock
 SM_LIBS += lock_queue
 SM_LIBS += ipc
 SM_LIBS += srmetadata
@@ -52,12 +73,12 @@ SM_LIBS += lvmanager
 SM_LIBS += blktap2
 SM_LIBS += lcache
 SM_LIBS += resetvdis
-SM_LIBS += wwid_conf
 SM_LIBS += trim_util
 SM_LIBS += pluginutil
 SM_LIBS += constants
 SM_LIBS += cbtutil
 SM_LIBS += sr_health_check
+SM_LIBS += $(SM_COMPAT_LIBS)
 
 UDEV_RULES = 65-multipath 55-xs-mpath-scsidev 57-usb 58-xapi 99-purestorage
 MPATH_CUSTOM_CONF = custom.conf
@@ -70,7 +91,8 @@ DEBUG_DEST := /opt/xensource/debug/
 BIN_DEST := /opt/xensource/bin/
 MASTER_SCRIPT_DEST := /etc/xensource/master.d/
 PLUGIN_SCRIPT_DEST := /etc/xapi.d/plugins/
-LIBEXEC := /opt/xensource/libexec/
+SM_LIBEXEC := /opt/xensource/libexec/
+SM_DATADIR := /usr/share/sm
 UDEV_RULES_DIR := /etc/udev/rules.d/
 UDEV_SCRIPTS_DIR := /etc/udev/scripts/
 SYSTEMD_SERVICE_DIR := /usr/lib/systemd/system/
@@ -84,6 +106,7 @@ SM_STAGING := $(DESTDIR)
 SM_STAMP := $(MY_OBJ_DIR)/.staging_stamp
 
 SM_PY_FILES = $(foreach LIB, $(SM_LIBS), drivers/$(LIB).py) $(foreach DRIVER, $(SM_DRIVERS), drivers/$(DRIVER)SR.py)
+SM_CORE_PY_FILES = $(foreach LIB, $(SM_CORE_LIBS), libs/sm/core/$(LIB).py) libs/sm/core/__init__.py
 
 .PHONY: build
 build:
@@ -92,10 +115,10 @@ build:
 .PHONY: precommit
 precommit: build
 	@ QUIT=0; \
-	CHANGED=$$(git status --porcelain $(SM_PY_FILES) | awk '{print $$2}'); \
+	CHANGED=$$(git status --porcelain $(SM_PY_FILES) $(SM_CORE_PY_FILES) | awk '{print $$2}'); \
 	for i in $$CHANGED; do \
 		echo Checking $${i} ...; \
-		PYTHONPATH=./drivers:./misc/fairlock:$$PYTHONPATH $(PYLINT) --rcfile=tests/pylintrc $${i}; \
+		PYTHONPATH=./drivers:./libs:./misc/fairlock:$$PYTHONPATH $(PYLINT) --rcfile=tests/pylintrc $${i}; \
 		[ $$? -ne 0 ] && QUIT=1 ; \
 	done; \
 	if [ $$QUIT -ne 0 ]; then \
@@ -106,7 +129,7 @@ precommit: build
 
 .PHONY: precheck
 precheck: build
-	PYTHONPATH=./drivers:./misc/fairlock:$$PYTHONPATH $(PYLINT) --rcfile=tests/pylintrc $(SM_PY_FILES)
+	PYTHONPATH=./drivers:./libs:./misc/fairlock:$$PYTHONPATH $(PYLINT) --rcfile=tests/pylintrc $(SM_PY_FILES) $(SM_CORE_PY_FILES)
 	echo "Precheck succeeded with no outstanding issues found."
 
 .PHONY: install
@@ -128,6 +151,19 @@ install: precheck
 	mkdir -p $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	mkdir -p $(SM_STAGING)$(EXTENSION_SCRIPT_DEST)
 	mkdir -p $(SM_STAGING)/sbin
+	# Core libs (including XML error definitions)
+	mkdir -p $(SM_STAGING)/$(PYTHONLIBDIR)/sm/core
+	install -D -m 644 libs/sm/__init__.py $(SM_STAGING)$(PYTHONLIBDIR)/sm/__init__.py
+	for i in $(SM_CORE_PY_FILES); do \
+	  install -D -m 644 $$i $(SM_STAGING)$(PYTHONLIBDIR)/sm/core/; \
+	done
+	mkdir -p $(SM_STAGING)$(SM_DATADIR)
+	# This should go in SM_DATADIR but that breaks the unit tests.
+	# Leave it next to xs_errors.py until we can fix that.
+	for i in $(SM_XML); do \
+	  install -D -m 644 libs/sm/core/$$i.xml $(SM_STAGING)$(PYTHONLIBDIR)/sm/core/; \
+	done
+	# Legacy SM python files
 	for i in $(SM_PY_FILES); do \
 	  install -m 755 $$i $(SM_STAGING)$(SM_DEST); \
 	done
@@ -162,9 +198,6 @@ install: precheck
 	for i in $(UDEV_RULES); do \
 	  install -m 644 udev/$$i.rules \
 	    $(SM_STAGING)$(UDEV_RULES_DIR); done
-	for i in $(SM_XML); do \
-	  install -m 755 drivers/$$i.xml \
-	    $(SM_STAGING)$(SM_DEST); done
 	cd $(SM_STAGING)$(SM_DEST) && for i in $(SM_DRIVERS); do \
 	  ln -sf $$i"SR.py" $$i"SR"; \
 	done
@@ -183,21 +216,20 @@ install: precheck
 	install -m 755 drivers/tapdisk-pause $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	install -m 755 drivers/intellicache-clean $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	install -m 755 drivers/trim $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
-	install -m 755 drivers/iscsilib.py $(SM_STAGING)$(SM_DEST)
-	mkdir -p $(SM_STAGING)$(LIBEXEC)
-	install -m 755 scripts/local-device-change $(SM_STAGING)$(LIBEXEC)
-	install -m 755 scripts/check-device-sharing $(SM_STAGING)$(LIBEXEC)
-	install -m 755 scripts/usb_change $(SM_STAGING)$(LIBEXEC)
-	install -m 755 scripts/kickpipe $(SM_STAGING)$(LIBEXEC)
-	install -m 755 scripts/set-iscsi-initiator $(SM_STAGING)$(LIBEXEC)
+	mkdir -p $(SM_STAGING)$(SM_LIBEXEC)
+	install -m 755 scripts/local-device-change $(SM_STAGING)$(SM_LIBEXEC)
+	install -m 755 scripts/check-device-sharing $(SM_STAGING)$(SM_LIBEXEC)
+	install -m 755 scripts/usb_change $(SM_STAGING)$(SM_LIBEXEC)
+	install -m 755 scripts/kickpipe $(SM_STAGING)$(SM_LIBEXEC)
+	install -m 755 scripts/set-iscsi-initiator $(SM_STAGING)$(SM_LIBEXEC)
 	mkdir -p $(SM_STAGING)/etc/xapi.d/xapi-pre-shutdown/
 	install -m 755 scripts/stop_all_gc $(SM_STAGING)/etc/xapi.d/xapi-pre-shutdown/
 	$(MAKE) -C dcopy install DESTDIR=$(SM_STAGING)
 	ln -sf $(SM_DEST)blktap2.py $(SM_STAGING)$(BIN_DEST)/blktap2
 	ln -sf $(SM_DEST)lcache.py $(SM_STAGING)$(BIN_DEST)tapdisk-cache-stats
 	install -m 755 scripts/xs-mpath-scsidev.sh $(SM_STAGING)$(UDEV_SCRIPTS_DIR)
-	install -m 755 scripts/make-dummy-sr $(SM_STAGING)$(LIBEXEC)
-	install -m 755 scripts/storage-init $(SM_STAGING)$(LIBEXEC)
+	install -m 755 scripts/make-dummy-sr $(SM_STAGING)$(SM_LIBEXEC)
+	install -m 755 scripts/storage-init $(SM_STAGING)$(SM_LIBEXEC)
 
 .PHONY: clean
 clean:

drivers/BaseISCSI.py

@@ -19,17 +19,17 @@
 #
 
 import SR
-import util
+from sm.core import util
 import time
 import LUNperVDI
 import os
 import sys
 import re
 import glob
 import xml.dom.minidom
-import scsiutil
-import iscsilib
-import xs_errors
+from sm.core import scsiutil
+from sm.core import libiscsi as iscsilib
+from sm.core import xs_errors
 
 INITIATORNAME_FILE = '/etc/iscsi/initiatorname.iscsi'
 SECTOR_SHIFT = 9

drivers/DummySR.py

@@ -20,7 +20,7 @@
 import SR
 import VDI
 import SRCommand
-import util
+from sm.core import util
 import time
 import xs_errors
 

drivers/EXTSR.py

@@ -21,7 +21,7 @@
 from SR import deviceCheck
 import SRCommand
 import FileSR
-import util
+from sm.core import util
 import lvutil
 import scsiutil
 

drivers/FileSR.py

@@ -20,12 +20,12 @@
 import SR
 import VDI
 import SRCommand
-import util
-import scsiutil
+from sm.core import util
+from sm.core import scsiutil
 import vhdutil
 import os
 import errno
-import xs_errors
+from sm.core import xs_errors
 import cleanup
 import blktap2
 import time

drivers/HBASR.py

@@ -22,12 +22,12 @@
 import SR
 import SRCommand
 import devscan
-import scsiutil
-import util
+from sm.core import scsiutil
+from sm.core import util
 import LUNperVDI
 import os
 import time
-import xs_errors
+from sm.core import xs_errors
 import xml.dom.minidom
 
 CAPABILITIES = ["SR_PROBE", "VDI_CREATE", "VDI_DELETE", "VDI_ATTACH",

drivers/ISOSR.py

@@ -20,11 +20,11 @@
 import SR
 import VDI
 import SRCommand
-import util
+from sm.core import util
 import nfs
 import os
 import re
-import xs_errors
+from sm.core import xs_errors
 import cifutils
 
 CAPABILITIES = ["VDI_CREATE", "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH",

drivers/LUNperVDI.py

@@ -20,9 +20,9 @@
 
 import os
 import VDI
-import util
-import scsiutil
-import xs_errors
+from sm.core import util
+from sm.core import scsiutil
+from sm.core import xs_errors
 
 MAX_TIMEOUT = 15
 

drivers/LVHDSR.py

@@ -22,7 +22,7 @@
 from SR import deviceCheck
 import VDI
 import SRCommand
-import util
+from sm.core import util
 import lvutil
 import lvmcache
 import vhdutil
@@ -32,11 +32,11 @@
 import sys
 import time
 import errno
-import xs_errors
+from sm.core import xs_errors
 import cleanup
 import blktap2
 from journaler import Journaler
-from lock import Lock
+from sm.core.lock import Lock
 from refcounter import RefCounter
 from ipc import IPCFlag
 from lvmanager import LVActivator

drivers/LVHDoHBASR.py

@@ -29,7 +29,7 @@
 import sys
 import xs_errors
 import xmlrpc.client
-import util
+from sm.core import util
 import scsiutil
 import mpath_cli
 import glob

drivers/LVHDoISCSISR.py

@@ -22,19 +22,18 @@
 import LVHDSR
 import BaseISCSI
 import SRCommand
-import util
-import scsiutil
+from sm.core import util
+from sm.core import scsiutil
 import lvutil
 import time
 import os
 import sys
-import xs_errors
+from sm.core import xs_errors
 import xmlrpc.client
-import mpath_cli
-import iscsilib
+from sm.core import mpath_cli
+from sm.core import libiscsi as iscsilib
 import glob
 import copy
-import scsiutil
 import xml.dom.minidom
 
 CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_METADATA", "SR_TRIM",

drivers/NFSSR.py

@@ -22,15 +22,15 @@
 import SR
 import SRCommand
 import FileSR
-import util
+from sm.core import util
 import errno
 import os
 import sys
 import xmlrpc.client
-import xs_errors
+from sm.core import xs_errors
 import nfs
 import vhdutil
-from lock import Lock
+from sm.core.lock import Lock
 import cleanup
 
 CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_CACHING",

drivers/RawISCSISR.py

@@ -22,7 +22,7 @@
 import SRCommand
 import BaseISCSI
 import LUNperVDI
-import util
+from sm.core import util
 
 CAPABILITIES = ["SR_PROBE", "VDI_CREATE", "VDI_DELETE", "VDI_ATTACH",
                 "VDI_DETACH", "VDI_INTRODUCE"]

drivers/SHMSR.py

@@ -18,7 +18,7 @@
 import SR
 import VDI
 import SRCommand
-import util
+from sm.core import util
 import os
 import xs_errors
 

drivers/SMBSR.py

@@ -20,7 +20,7 @@
 import SR
 import SRCommand
 import FileSR
-import util
+from sm.core import util
 import errno
 import os
 import xmlrpc.client