From 4baa60a029421761446161cfd6df4a46aeca8791 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 17 Aug 2015 15:35:54 -0700 Subject: [PATCH 01/70] Remove 'level' from UserBackedStorageObject TCMUv2 has no more level -- it's full pass-through. Signed-off-by: Andy Grover --- rtslib/tcm.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/rtslib/tcm.py b/rtslib/tcm.py index a7688c6..f8279b3 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -741,7 +741,7 @@ class UserBackedStorageObject(StorageObject): An interface to configFS storage objects for userspace-backed backstore. ''' - def __init__(self, name, config=None, level=None, size=None, wwn=None): + def __init__(self, name, config=None, size=None, wwn=None): ''' @param name: The name of the UserBackedStorageObject. @type name: string @@ -755,37 +755,31 @@ class UserBackedStorageObject(StorageObject): @param config: user-handler-specific config string. - e.g. "rbd/machine1@snap4" @type config: string - @param level: TCMU emulation level, 0 or 1. Level 0 will pass all SCSI - commands, 1 will just pass I/O commands, READ, WRITE, etc. - @type level: int @return: A UserBackedStorageObject object. ''' if size is not None: - if level is None or config is None: - raise RTSLibError("'size', 'level', and 'config' must be set when " + if config is None: + raise RTSLibError("'size' and 'config' must be set when " "creating a new UserBackedStorageObject") if '/' not in config: raise RTSLibError("'config' must contain a '/' separating subtype " "from its configuration string") super(UserBackedStorageObject, self).__init__(name, 'create') try: - self._configure(config, level, size, wwn) + self._configure(config, size, wwn) except: self.delete() raise else: super(UserBackedStorageObject, self).__init__(name, 'lookup') - def _configure(self, config, level, size, wwn): + def _configure(self, config, size, wwn): self._check_self() if ':' in config: raise RTSLibError("':' not allowed in config string") - if level not in (0, 1): - raise RTSLibError("Current allowable levels are 0 or 1") self._control("dev_config=%s" % config) - self._control("pass_level=%d" % level) self._control("dev_size=%d" % size) self._enable() @@ -795,10 +789,6 @@ class UserBackedStorageObject(StorageObject): self._check_self() return int(self._parse_info('Size')) - def _get_level(self): - self._check_self() - return int(self._parse_info('PassLevel')) - def _get_config(self): self._check_self() val = self._parse_info('Config') @@ -808,8 +798,6 @@ class UserBackedStorageObject(StorageObject): size = property(_get_size, doc="Get the size in bytes.") - level = property(_get_level, - doc="Get the command emulation level.") config = property(_get_config, doc="Get the TCMU config.") @@ -817,7 +805,6 @@ class UserBackedStorageObject(StorageObject): d = super(UserBackedStorageObject, self).dump() d['wwn'] = self.wwn d['size'] = self.size - d['level'] = self.level d['config'] = self.config return d -- GitLab From 3cddf352d5cf0f23e2b7ead601050367eda2d237 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 31 Aug 2015 09:29:20 -0700 Subject: [PATCH 02/70] Fix some things in the deb/rpm Makefile Still some lingering issues since we renamed the example packaging directories. Signed-off-by: Andy Grover --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f975c6e..987d4fd 100644 --- a/Makefile +++ b/Makefile @@ -61,11 +61,11 @@ build/release-stamp: build/${PKGNAME}-${VERSION}/${NAME}/__init__.py @echo "Generating rpm specfile from template..." @cd build/${PKGNAME}-${VERSION}; \ - for spectmpl in rpm/*.spec.tmpl; do \ + for spectmpl in example-rpm/*.spec.tmpl; do \ sed -i "s/Version:\( *\).*/Version:\1${VERSION}/g" $${spectmpl}; \ mv $${spectmpl} $$(basename $${spectmpl} .tmpl); \ done; \ - rm -r rpm + rm -r example-rpm @echo "Generating rpm changelog..." @( \ version=$$(basename $$(git describe HEAD --tags | tr - .)); \ @@ -93,7 +93,7 @@ build/release-stamp: echo; \ echo " -- $${author} $${date}"; \ echo; \ - ) > build/${PKGNAME}-${VERSION}/debian/changelog + ) > build/${PKGNAME}-${VERSION}/example-debian/changelog @find build/${PKGNAME}-${VERSION}/ -exec \ touch -t $$(date -d @$$(git show -s --format="format:%at") \ +"%Y%m%d%H%M.%S") {} \; -- GitLab From 290738c20213a9fabff1d0a0d8eb726706473aad Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 6 Nov 2015 09:13:35 -0800 Subject: [PATCH 03/70] Fix regex in get_size_for_disk_name Would break with 'sda10'. We need to non-greedily match the first subgroup. Signed-off-by: Andy Grover --- rtslib/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index 5531e8f..79a4d9d 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -152,7 +152,7 @@ def get_size_for_disk_name(name): return get_size("/sys/block/%s" % name) except IOError: # Maybe it's a partition? - m = re.search(r'^([a-z0-9_\-!]+)(\d+)$', name) + m = re.search(r'^([a-z0-9_\-!]+?)(\d+)$', name) if m: # If disk name ends with a digit, Linux sticks a 'p' between it and # the partition number in the blockdev name. -- GitLab From bddfe573e553b9e6777880e973bc664e018a9414 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 16 Nov 2015 10:09:30 -0800 Subject: [PATCH 04/70] Handle errors from kmod.modprobe() If a module fails to load it can throw KmodError. Catch this and convert to RTSLibError. Signed-off-by: Andy Grover --- rtslib/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index 79a4d9d..286dbc1 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -382,7 +382,6 @@ def modprobe(module): try: import kmod - kmod.Kmod().modprobe(module) except ImportError: process = subprocess.Popen(("modprobe", module), stdout=subprocess.PIPE, @@ -390,6 +389,12 @@ def modprobe(module): (stdoutdata, stderrdata) = process.communicate() if process.returncode != 0: raise RTSLibError(stderrdata) + return + + try: + kmod.Kmod().modprobe(module) + except kmod.error.KmodError: + raise RTSLibError("Could not load module: %s" % module) def mount_configfs(): if not os.path.ismount("/sys/kernel/config"): -- GitLab From 6a417e32148db009351e02480fc437c659ea1e9b Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 16 Nov 2015 10:11:25 -0800 Subject: [PATCH 05/70] Tidy up _BaseFabricModule init specs are gone, fix comment. name should already be a string, if it isn't then it's a programming error. Signed-off-by: Andy Grover --- rtslib/fabric.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rtslib/fabric.py b/rtslib/fabric.py index eae6325..65f8195 100644 --- a/rtslib/fabric.py +++ b/rtslib/fabric.py @@ -137,12 +137,11 @@ class _BaseFabricModule(CFSNode): def __init__(self, name): ''' Instantiate a FabricModule object, according to the provided name. - @param name: the name of the FabricModule object. It must match an - existing target fabric module specfile (name.spec). + @param name: the name of the FabricModule object. @type name: str ''' super(_BaseFabricModule, self).__init__() - self.name = str(name) + self.name = name self.spec_file = "N/A" self._path = "%s/%s" % (self.configfs_dir, self.name) self.features = ('discovery_auth', 'acls', 'auth', 'nps', 'tpgts') -- GitLab From 2bcc4cd809409525bca62f09b08e97c2b97b0d43 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 16 Nov 2015 10:15:11 -0800 Subject: [PATCH 06/70] Add support for xen-scsiback Based on seeing https://github.com/Datera/rtslib/pull/10 we need to also add a class to support this new fabric, since we don't use specs. I can't test since Fedora doesn't include Xen dom0, but it *should* work. :-) Signed-off-by: Andy Grover --- rtslib/fabric.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rtslib/fabric.py b/rtslib/fabric.py index 65f8195..b5cc4d9 100644 --- a/rtslib/fabric.py +++ b/rtslib/fabric.py @@ -435,6 +435,14 @@ class VhostFabricModule(_BaseFabricModule): self.wwn_types = ('naa',) self.kernel_module = "tcm_vhost" +class XenPvScsiFabricModule(_BaseFabricModule): + def __init__(self): + super(XenPvScsiFabricModule, self).__init__('xen_pvscsi') + self._path = "%s/%s" % (self.configfs_dir, 'xen-pvscsi') + self.features = ("nexus", "tpgts") + self.wwn_types = ('naa',) + self.kernel_module = "xen-scsiback" + fabric_modules = { "srpt": SRPTFabricModule, @@ -445,6 +453,7 @@ fabric_modules = { "tcm_fc": FCoEFabricModule, # "usb_gadget": USBGadgetFabricModule, # very rare, don't show "vhost": VhostFabricModule, + "xen_pvscsi": XenPvScsiFabricModule, } # -- GitLab From 7ccca210c7f00176cfcf2aa2f40df3d0ee9c5a30 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Tue, 17 Nov 2015 10:07:19 -0800 Subject: [PATCH 07/70] Restrict auth parameters to 255 chars See https://bugzilla.redhat.com/show_bug.cgi?id=1279942 The kernel will truncate all auth paramaters over 255 characters. Instead of silently altering what the user entered, throw an exception. Make exception RTSLibError for consistency with the rest of the library. Change check for 'NULL' to match. Signed-off-by: Andy Grover --- rtslib/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index 286dbc1..46f10c3 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -451,7 +451,9 @@ def _set_auth_attr(self, value, attribute, ignore=False): path = "%s/%s" % (self.path, attribute) value = value.strip() if value == "NULL": - raise ValueError("'NULL' is not a permitted value") + raise RTSLibError("'NULL' is not a permitted value") + if len(value) > 255: + raise RTSLibError("Value longer than maximum length of 255") if value == '': value = "NULL" try: -- GitLab From 5afe6d98dc23b050a7649150fdf8e09dfba367c1 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Tue, 24 Nov 2015 14:54:21 -0800 Subject: [PATCH 08/70] Do not use realpath() on fileio paths RBD creates block device nodes named /dev/rbd0 -> rbdX, but these may refer to different volumes after reboot. Users should use consistent names like /dev/rbd/foo/bar, these are symlinks to rbdX. rtslib should not use realpath() to get the rbdX name, but should just use what was given, which may be a symlink, but it will point to the right rbdX. Resolves #57 Signed-off-by: Andy Grover --- rtslib/tcm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/rtslib/tcm.py b/rtslib/tcm.py index f8279b3..941db1c 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -565,8 +565,6 @@ class FileIOStorageObject(StorageObject): def _configure(self, dev, size, wwn, write_back): self._check_self() - - dev = os.path.realpath(dev) block_type = get_blockdev_type(dev) if block_type is None: # a file if os.path.exists(dev) and not os.path.isfile(dev): @@ -816,7 +814,6 @@ class StorageObjectFactory(object): """ def __new__(cls, path): - path = os.path.realpath(path) name = path.strip("/").replace("/", "-") if os.path.exists(path): s = os.stat(path) -- GitLab From ae4c72b54f252a4fc0184adeff8429590918efcf Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 30 Nov 2015 15:44:12 -0800 Subject: [PATCH 09/70] version 2.1.fb58 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ac759d5..923a6b9 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.57', + version = '2.1.58', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From 3f828e7971d035f455021d8011c3f7eafb3866fb Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Tue, 1 Dec 2015 10:45:19 -0800 Subject: [PATCH 10/70] Save/restore alias values for LUN and MappedLUN if present We generate aliases -- the symlink names that link LUNs to storage objects, and MappedLUNs to LUNs -- using uuid, and generating a uuid needs the system to have a seeded source of randomness. If we're restoring a config on boot, we may not have enough randomness and this can cause long delays. Instead, save alias values (by including them in dump()), and then restore them if present. If not present, then a fresh alias value is generated. See https://github.com/agrover/targetcli-fb/issues/58 Reported-by: Timofey Signed-off-by: Andy Grover --- rtslib/target.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/rtslib/target.py b/rtslib/target.py index 206ef12..db44ebc 100644 --- a/rtslib/target.py +++ b/rtslib/target.py @@ -637,7 +637,7 @@ class LUN(CFSNode): return try: - cls(tpg_obj, lun['index'], storage_object=match_so) + cls(tpg_obj, lun['index'], storage_object=match_so, alias=lun.get('alias')) except (RTSLibError, KeyError): err_func("Creating TPG %d LUN index %d failed" % (tpg_obj.tag, lun['index'])) @@ -647,6 +647,7 @@ class LUN(CFSNode): d['storage_object'] = "/backstores/%s/%s" % \ (self.storage_object.plugin, self.storage_object.name) d['index'] = self.lun + d['alias'] = self.alias return d @@ -998,7 +999,7 @@ class MappedLUN(CFSNode): self.parent_nodeacl.parent_tpg.tag, self.tpg_lun.lun) def __init__(self, parent_nodeacl, mapped_lun, - tpg_lun=None, write_protect=None): + tpg_lun=None, write_protect=None, alias=None): ''' A MappedLUN object can be instanciated in two ways: - B{Creation mode}: If I{tpg_lun} is specified, the underlying @@ -1046,14 +1047,14 @@ class MappedLUN(CFSNode): if tpg_lun is not None: self._create_in_cfs_ine('create') try: - self._configure(tpg_lun, write_protect) + self._configure(tpg_lun, write_protect, alias) except: self.delete() raise else: self._create_in_cfs_ine('lookup') - def _configure(self, tpg_lun, write_protect): + def _configure(self, tpg_lun, write_protect, alias): self._check_self() if isinstance(tpg_lun, LUN): tpg_lun = tpg_lun.lun @@ -1071,8 +1072,10 @@ class MappedLUN(CFSNode): if not (isinstance(tpg_lun, LUN) and tpg_lun): raise RTSLibError("LUN %s does not exist in this TPG" % str(tpg_lun)) - os.symlink(tpg_lun.path, "%s/%s" - % (self.path, str(uuid.uuid4())[-10:])) + + if not alias: + alias = str(uuid.uuid4())[-10:] + os.symlink(tpg_lun.path, "%s/%s" % (self.path, alias)) try: self.write_protect = int(write_protect) > 0 @@ -1170,7 +1173,8 @@ class MappedLUN(CFSNode): try: mlun_obj = cls(acl_obj, mlun['index'], - tpg_lun_obj, mlun.get('write_protect')) + tpg_lun_obj, mlun.get('write_protect'), + mlun.get('alias')) mlun_obj.tag = mlun.get("tag", None) except (RTSLibError, KeyError): err_func("Creating MappedLUN object %d failed" % mlun['index']) @@ -1180,6 +1184,7 @@ class MappedLUN(CFSNode): d['write_protect'] = self.write_protect d['index'] = self.mapped_lun d['tpg_lun'] = self.tpg_lun.lun + d['alias'] = self.alias return d -- GitLab From 110d0482f837b36ab30490090dd28e6cd303f544 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 2 Dec 2015 21:50:32 -0800 Subject: [PATCH 11/70] Make sure get_size_for_blk_dev() works for symlinks Now that we don't call realpath() in other places, we need to call it here, because we really do need the real path here. Signed-off-by: Andy Grover --- rtslib/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rtslib/utils.py b/rtslib/utils.py index 46f10c3..4cee7a5 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -117,6 +117,7 @@ def get_size_for_blk_dev(path): @type path: string @return: The size in logical blocks of the device ''' + path = os.path.realpath(str(path)) rdev = os.lstat(path).st_rdev maj, min = os.major(rdev), os.minor(rdev) -- GitLab From ba2badbc40aa1798ee667b6b4b31f5733b50611a Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 10 Dec 2015 17:46:10 -0500 Subject: [PATCH 12/70] Make use of pyudev package as much as possible in utils module. Signed-off-by: mulhern --- rtslib/utils.py | 117 +++++++++++++++++++++++++++++++----------------- 1 file changed, 77 insertions(+), 40 deletions(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index 4cee7a5..e5f1c4d 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -27,6 +27,10 @@ import subprocess import uuid from contextlib import contextmanager +import pyudev + +_CONTEXT = pyudev.Context() + class RTSLibError(Exception): ''' Generic rtslib error. @@ -111,22 +115,36 @@ def is_dev_in_use(path): os.close(file_fd) return False +def _get_size_for_dev(device): + ''' + @param device: the device + @type device: pyudev.Device + @return: the size in logical blocks, 0 if none found + @rtype: int + ''' + attributes = device.attributes + try: + sect_size = attributes.asint('size') + except (KeyError, UnicodeDecodeError, ValueError): + return 0 + + try: + logical_block_size = attributes.asint('queue/logical_block_size') + except (KeyError, UnicodeDecodeError, ValueError): + return 0 + + return (sect_size * 512) // logical_block_size + def get_size_for_blk_dev(path): ''' @param path: The path to a block device @type path: string @return: The size in logical blocks of the device + @raises: DeviceNotFoundError if corresponding device not found + @raises: EnvironmentError, ValueError in some situations ''' - path = os.path.realpath(str(path)) - rdev = os.lstat(path).st_rdev - maj, min = os.major(rdev), os.minor(rdev) - - for line in list(open("/proc/partitions"))[2:]: - xmaj, xmin, size, name = line.split() - if (maj, min) == (int(xmaj), int(xmin)): - return get_size_for_disk_name(name) - else: - return 0 + device = Device.from_device_file(_CONTEXT, os.path.realpath(str(path))) + return _get_size_for_dev(device) get_block_size = get_size_for_blk_dev @@ -135,23 +153,26 @@ def get_size_for_disk_name(name): @param name: a kernel disk name, as found in /proc/partitions @type name: string @return: The size in logical blocks of a disk-type block device. + @raises: DeviceNotFoundError ''' # size is in 512-byte sectors, we want to return number of logical blocks - def get_size(path, is_partition=False): - sect_size = int(fread("%s/size" % path)) - if is_partition: - path = os.path.split(path)[0] - logical_block_size = int(fread("%s/queue/logical_block_size" % path)) - return sect_size / (logical_block_size / 512) + def get_size(name): + """ + :param str name: name of block device + :raises DeviceNotFoundError: if device not found + """ + device = pyudev.Device.from_name(_CONTEXT, 'block', name) + return _get_size_for_dev(device) # Disk names can include '/' (e.g. 'cciss/c0d0') but these are changed to # '!' when listed in /sys/block. + # in pyudev 0.19 it should no longer be necessary to swap '/'s in name name = name.replace("/", "!") try: - return get_size("/sys/block/%s" % name) - except IOError: + return get_size(name) + except pyudev.DeviceNotFoundError: # Maybe it's a partition? m = re.search(r'^([a-z0-9_\-!]+?)(\d+)$', name) if m: @@ -160,7 +181,7 @@ def get_size_for_disk_name(name): disk = m.groups()[0] if disk[-1] == 'p' and disk[-2].isdigit(): disk = disk[:-1] - return get_size("/sys/block/%s/%s" % (disk, m.group()), True) + return get_size(m.group()) else: raise @@ -184,26 +205,41 @@ def get_blockdev_type(path): @type path: string @return: An int for the block device type, or None if not a block device. ''' - dev = os.path.realpath(path) - - # is dev a block device? try: - mode = os.stat(dev) - except OSError: + device = pyudev.Device.from_device_file(_CONTEXT, path) + except (pyudev.DeviceNotFoundError, EnvironmentError, ValueError): return None - if not stat.S_ISBLK(mode[stat.ST_MODE]): + if device.subsystem != u'block': return None - # assume disk if device/type is missing - disk_type = 0 - with ignored(IOError): - disk_type = int(fread("/sys/block/%s/device/type" % os.path.basename(dev))) + attributes = device.attributes + disk_type = 0 + try: + disk_type = attributes.asint('device/type') + except (KeyError, UnicodeDecodeError, ValueError): + pass return disk_type get_block_type = get_blockdev_type +def _hctl_from_dev(device): + ''' + @param device: the device + @type device: pyudev.Device + @returns: H:C:T:L specifier or None if not found + @rtype: list of int * 4 or NoneType + ''' + parent = device.parent + if parent is None or parent.subsystem != 'scsi': + return None + + try: + return [int(data) for data in parent.sys_name.split(':')] + except (ValueError, TypeError): + return None + def convert_scsi_path_to_hctl(path): ''' This function returns the SCSI ID in H:C:T:L form for the block @@ -229,14 +265,11 @@ def convert_scsi_path_to_hctl(path): values representing the SCSI ID of the device, or None if no match is found. ''' - devname = os.path.basename(os.path.realpath(path)) try: - hctl = os.listdir("/sys/block/%s/device/scsi_device" - % devname)[0].split(':') - except: + device = pyudev.Device.from_device_file(context, os.path.realpath(path)) + except (pyudev.DeviceNotFoundError, EnvironmentError, ValueError): return None - - return [int(data) for data in hctl] + return _hctl_from_dev(device) def convert_scsi_hctl_to_path(host, controller, target, lun): ''' @@ -270,11 +303,15 @@ def convert_scsi_hctl_to_path(host, controller, target, lun): raise RTSLibError( "The host, controller, target and lun parameter must be integers") - for devname in os.listdir("/sys/block"): - path = "/dev/%s" % devname - hctl = [host, controller, target, lun] - if convert_scsi_path_to_hctl(path) == hctl: - return os.path.realpath(path) + hctl = [host, controller, target, lun] + try: + scsi_device = pyudev.Device.from_name(_CONTEXT, 'scsi', ':'.join(hctl)) + except DeviceNotFoundError: + return '' + + for device in context.list_devices(subsystem='block'): + if device.parent == scsi_device: + return device.device_node return '' def generate_wwn(wwn_type): -- GitLab From 690d1905066e4e880891496648d5f42befa707bb Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 11 Dec 2015 10:47:59 -0800 Subject: [PATCH 13/70] version 2.1.fb59 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 923a6b9..9e710cc 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.58', + version = '2.1.59', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From 501521813ce29b494b38b4d017599afee250e79c Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 18 Dec 2015 18:36:25 -0800 Subject: [PATCH 14/70] Update README.md We no longer use 2to3. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b40a43..b3957bd 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Python object API for managing the Linux LIO kernel target rtslib-fb is an object-based Python library for configuring the LIO generic SCSI target, present in 3.x Linux kernel versions. -This runs with Python 2 and 2to3 is run by setup.py to run on Python 3. +This work with both Python 2 and Python 3, thanks to the python-six library. rtslib-fb development --------------------- -- GitLab From f17b77d47def7141e10a740d04b721d3d9f97b05 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 18 Dec 2015 18:37:30 -0800 Subject: [PATCH 15/70] Update README.md again Gaah, grammar. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b3957bd..8825b8f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Python object API for managing the Linux LIO kernel target rtslib-fb is an object-based Python library for configuring the LIO generic SCSI target, present in 3.x Linux kernel versions. -This work with both Python 2 and Python 3, thanks to the python-six library. +It supports both Python 2 and Python 3, thanks to the python-six library. rtslib-fb development --------------------- -- GitLab From c4d26669558674805620f96e070939cca964ab09 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 8 Jan 2016 15:48:35 -0500 Subject: [PATCH 16/70] Use find_parent instead of parent. Better, because it is sure to find the scsi parent. You never know what intermediate parent might get interposed. Better, becaus it does not rely on subsystem property (which sometimes raises an AttributeError (fixed in pyudev 0.19)). Signed-off-by: mulhern --- rtslib/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index e5f1c4d..a763bcc 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -231,8 +231,8 @@ def _hctl_from_dev(device): @returns: H:C:T:L specifier or None if not found @rtype: list of int * 4 or NoneType ''' - parent = device.parent - if parent is None or parent.subsystem != 'scsi': + parent = device.find_parent(subsystem='scsi') + if parent is None: return None try: -- GitLab From 2393c1fd4fdda71c2b594212a6f5471852536e3d Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 11 Jan 2016 10:09:41 -0500 Subject: [PATCH 17/70] Make sure to properly avoid treating partitions the same as disk. Return None if device path is a partition in convert_scsi_path_to_hctl. In convert_scsi_hctl_to_path ensure that device is not a partition in the most correct and efficient way. Get exactly the set of devices that are descendants of the scsi device and are block devices and have type disk. This should be a singleton, or the empty set. Signed-off-by: mulhern --- rtslib/utils.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index a763bcc..acf8aa5 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -224,22 +224,6 @@ def get_blockdev_type(path): get_block_type = get_blockdev_type -def _hctl_from_dev(device): - ''' - @param device: the device - @type device: pyudev.Device - @returns: H:C:T:L specifier or None if not found - @rtype: list of int * 4 or NoneType - ''' - parent = device.find_parent(subsystem='scsi') - if parent is None: - return None - - try: - return [int(data) for data in parent.sys_name.split(':')] - except (ValueError, TypeError): - return None - def convert_scsi_path_to_hctl(path): ''' This function returns the SCSI ID in H:C:T:L form for the block @@ -266,10 +250,22 @@ def convert_scsi_path_to_hctl(path): match is found. ''' try: - device = pyudev.Device.from_device_file(context, os.path.realpath(path)) + path = os.path.realpath(path) + device = pyudev.Device.from_device_file(_CONTEXT, path) except (pyudev.DeviceNotFoundError, EnvironmentError, ValueError): return None - return _hctl_from_dev(device) + + if device.device_type != u'disk': + return None + + parent = device.find_parent(subsystem='scsi') + if parent is None: + return None + + try: + return [int(data) for data in parent.sys_name.split(':')] + except (ValueError, TypeError): + return None def convert_scsi_hctl_to_path(host, controller, target, lun): ''' @@ -309,10 +305,13 @@ def convert_scsi_hctl_to_path(host, controller, target, lun): except DeviceNotFoundError: return '' - for device in context.list_devices(subsystem='block'): - if device.parent == scsi_device: - return device.device_node - return '' + devices = _CONTEXT.list_devices( + subsystem='block', + DEVTYPE='disk', + parent=scsi_device + ) + + return next((dev.device_node for dev in devices), '') def generate_wwn(wwn_type): ''' -- GitLab From b625f61a03d2127239480b45fed80028f82f8a50 Mon Sep 17 00:00:00 2001 From: Jon Magrini Date: Tue, 19 Jan 2016 09:36:51 -0500 Subject: [PATCH 18/70] [rtslib-fb] Ensure internal buffers are flushed when saveconfig.json is written to disk. A power-outage, unexpected reboot, etc can lead to zero byte file after a saveconfig. Signed-off-by: Jon Magrini --- rtslib/root.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rtslib/root.py b/rtslib/root.py index a4fbb2e..083a2c2 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -249,7 +249,9 @@ class RTSRoot(CFSNode): os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IWUSR) f.write(json.dumps(self.dump(), sort_keys=True, indent=2)) f.write("\n") + f.flush() os.fsync(f.fileno()) + f.close() os.rename(save_file+".temp", save_file) -- GitLab From 2372b244d57325117a477514f28c748f2aca8142 Mon Sep 17 00:00:00 2001 From: Fam Zheng Date: Tue, 15 Mar 2016 14:00:26 +0800 Subject: [PATCH 19/70] UserBackedStorageObject: Fix docstring for __init__ The present parameter list is "name, config, size, wwn", update the docstring to match this. Signed-off-by: Fam Zheng --- rtslib/tcm.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rtslib/tcm.py b/rtslib/tcm.py index 941db1c..bdeafd2 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -743,16 +743,13 @@ class UserBackedStorageObject(StorageObject): ''' @param name: The name of the UserBackedStorageObject. @type name: string - @param dev: The path to the backend block device to be used. - - Example: I{dev="/dev/sda"}. - - The only device type that is accepted I{TYPE_DISK}. - For other device types, use pscsi. - @type dev: string - @param size: The size of the device to create, in bytes. - @type size: int @param config: user-handler-specific config string. - e.g. "rbd/machine1@snap4" @type config: string + @param size: The size of the device to create, in bytes. + @type size: int + @param wwn: T10 WWN Unit Serial, will generate if None + @type wwn: string @return: A UserBackedStorageObject object. ''' -- GitLab From 7576593de778e21220a7821234e1748230321b8d Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 25 Mar 2016 16:26:43 -0700 Subject: [PATCH 20/70] Fix 'make rpm' Include symlinks in dist tarball fix some paths in specfile. Signed-off-by: Andy Grover --- Makefile | 3 ++- example-rpm/python-rtslib.spec.tmpl | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 987d4fd..bfc9010 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,8 @@ build/release-stamp: @cd build; tar -c --owner=0 --group=0 --numeric-owner \ --format=gnu -b20 --quoting-style=escape \ -f ../dist/${PKGNAME}-${VERSION}.tar \ - $$(find ${PKGNAME}-${VERSION} -type f | sort) + $$(find ${PKGNAME}-${VERSION} -type f | sort) \ + $$(find ${PKGNAME}-${VERSION} -type l | sort) @gzip -6 -n dist/${PKGNAME}-${VERSION}.tar @echo "Generated release tarball:" @echo " $$(ls dist/${PKGNAME}-${VERSION}.tar.gz)" diff --git a/example-rpm/python-rtslib.spec.tmpl b/example-rpm/python-rtslib.spec.tmpl index ae566b5..2191b7a 100644 --- a/example-rpm/python-rtslib.spec.tmpl +++ b/example-rpm/python-rtslib.spec.tmpl @@ -1,4 +1,4 @@ -%define oname rtslib +%define oname rtslib-fb Name: python-rtslib License: Apache License 2.0 @@ -22,7 +22,7 @@ API for RisingTide Systems generic SCSI target. %build %{__python} setup.py build mkdir -p doc -epydoc --no-sourcecode --html -n %{oname} --exclude configobj %{oname}/*.py +epydoc --no-sourcecode --html -n %{oname} --exclude configobj rtslib_fb/*.py mv html doc/ %install @@ -37,6 +37,7 @@ rm -rf %{buildroot} %files %defattr(-,root,root,-) %{python_sitelib} +%{_bindir}/targetctl /usr/share/doc/python-rtslib-doc-%{version} %doc COPYING README.md -- GitLab From 5061cf0c717eaf41a7e75420a84b44da07ccac13 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 7 Apr 2016 17:34:18 -0700 Subject: [PATCH 21/70] version 2.1.fb60 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9e710cc..2087187 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.59', + version = '2.1.60', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From ebf20306d6bfe2fd0e508d4233b26ca5a8352487 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Tue, 10 May 2016 20:53:10 -0700 Subject: [PATCH 22/70] Initial version of convert-to-json script This script attempts to convert from Datera 3.0's scsi_target.lio format to rtslib-fb's json format. It currently is not fully parsing the .lio file according to its grammar, it's kind of ad-hoc, shall we say, so that it hopefully works on most existing .lio files. Signed-off-by: Andy Grover --- scripts/convert-to-json | 312 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100755 scripts/convert-to-json diff --git a/scripts/convert-to-json b/scripts/convert-to-json new file mode 100755 index 0000000..6f84c73 --- /dev/null +++ b/scripts/convert-to-json @@ -0,0 +1,312 @@ +#!/usr/bin/python3 +''' +convert-to-json + +This file is part of RTSLib-fb. +Copyright (c) 2013-2016 by Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. +''' + +# +# A script to convert .lio format save files to json format. +# + +import json +import re + +def human_to_bytes(hsize, kilo=1024): + ''' + This function converts human-readable amounts of bytes to bytes. + It understands the following units : + - I{B} or no unit present for Bytes + - I{k}, I{K}, I{kB}, I{KB} for kB (kilobytes) + - I{m}, I{M}, I{mB}, I{MB} for MB (megabytes) + - I{g}, I{G}, I{gB}, I{GB} for GB (gigabytes) + - I{t}, I{T}, I{tB}, I{TB} for TB (terabytes) + + Note: The definition of I{kilo} defaults to 1kB = 1024Bytes. + Strictly speaking, those should not be called I{kB} but I{kiB}. + You can override that with the optional kilo parameter. + + @param hsize: The human-readable version of the Bytes amount to convert + @type hsize: string or int + @param kilo: Optional base for the kilo prefix + @type kilo: int + @return: An int representing the human-readable string converted to bytes + ''' + size = hsize.replace('i', '') + size = size.lower() + if not re.match("^[0-9\.]+[k|m|g|t]?[b]?$", size): + raise Exception("Cannot interpret size, wrong format: %s" % hsize) + + size = size.rstrip('ib') + + units = ['k', 'm', 'g', 't'] + try: + power = units.index(size[-1]) + 1 + except ValueError: + power = 0 + size = int(size) + else: + try: + size = int(size[:-1]) + except ValueError: + size = int(float(size[:-1])) + + return size * (int(kilo) ** power) + +def parse_yesno(val): + if val == "yes": + return 1 + elif val == "no": + return 0 + else: + try: + return int(val) + except: + return val + +def parse_attributes(txt, cur): + attribs = {} + while txt[cur] != "}": + name = txt[cur] + val = txt[cur+1] + attribs[name] = parse_yesno(val) + cur += 2 + return (cur+1, attribs) + +def parse_fileio(txt, cur): + so = dict(plugin="fileio") + while txt[cur] != "}": + if txt[cur] == "path": + so["dev"] = txt[cur+1] + cur += 2 + continue + if txt[cur] == "attribute": + cur, so["attributes"] = parse_attributes(txt, cur+2) + continue + return (cur+1, so) + +def parse_block(txt, cur): + so = dict(plugin="block") + while txt[cur] != "}": + if txt[cur] == "path": + so["dev"] = txt[cur+1] + cur += 2 + continue + if txt[cur] == "attribute": + cur, so["attributes"] = parse_attributes(txt, cur+2) + continue + return (cur+1, so) + +def parse_ramdisk(txt, cur): + so = dict(plugin="ramdisk") + while txt[cur] != "}": + if txt[cur] == "nullio": + so["nullio"] = parse_yesno(txt[cur+1]) + cur += 2 + continue + if txt[cur] == "size": + so["size"] = human_to_bytes(txt[cur+1]) + cur += 2 + continue + if txt[cur] == "attribute": + cur, so["attributes"] = parse_attributes(txt, cur+2) + continue + return (cur+1, so) + +so_types = { + "fileio": parse_fileio, + "rd_mcp": parse_ramdisk, + "iblock": parse_block, +} + +def parse_storage(txt, cur): + name = txt[cur+3] + ty = txt[cur+1] + cur += 5 + (cur, d) = so_types[ty](txt, cur) + d["name"] = name + return (cur, d) + +def parse_lun(txt, cur): + index = int(txt[cur+1]) + plugin, name = txt[cur+3].split(":") + return cur+4, dict(index=index, plugin=plugin, name=name) + +def parse_mapped_lun(txt, cur): + mlun = dict(index=txt[cur+1]) + cur += 3 + while txt[cur] != "}": + if txt[cur] == "target_lun": + mlun["tpg_lun"] = parse_yesno(txt[cur+1]) + cur += 2 + continue + if txt[cur] == "write_protect": + mlun["write_protect"] = bool(parse_yesno(txt[cur+1])) + cur += 2 + continue + return cur, mlun + +def parse_acl(txt, cur): + acl = dict(node_wwn=txt[cur+1]) + mapped_luns = [] + cur += 3 + while txt[cur] != "}": + if txt[cur] == "attribute": + cur, acl["attributes"] = parse_attributes(txt, cur+2) + continue + if txt[cur] == "auth": + cur, auth = parse_attributes(txt, cur+2) + if len(auth): + acl["auth"] = auth + continue + if txt[cur] == "mapped_lun": + cur, mlun = parse_mapped_lun(txt, cur) + mapped_luns.append(mlun) + acl["mapped_luns"] = mapped_luns + return cur+1, acl + +def parse_tpgt(txt, cur): + tpgt = dict(tag = int(txt[cur+1])) + luns = [] + acls = [] + portals = [] + cur += 3 + while txt[cur] != "}": + if txt[cur] == "enable": + tpgt["enable"] = parse_yesno(txt[cur+1]) + cur += 2 + continue + if txt[cur] == "attribute": + cur, tpgt["attributes"] = parse_attributes(txt, cur+2) + continue + if txt[cur] == "parameter": + cur, tpgt["parameters"] = parse_attributes(txt, cur+2) + continue + if txt[cur] == "auth": + cur, auth = parse_attributes(txt, cur+2) + if len(auth): + tpgt["auth"] = auth + continue + if txt[cur] == "lun": + cur, l = parse_lun(txt, cur) + luns.append(l) + continue + if txt[cur] == "acl": + cur, acl = parse_acl(txt, cur) + acls.append(acl) + continue + if txt[cur] == "portal": + ip, port = txt[cur+1].split(":") + portal = dict(ip_address=ip, port=port) + portals.append(portal) + cur += 2 + if len(luns): + tpgt["luns"] = luns + if len(acls): + tpgt["node_acls"] = acls + if len(portals): + tpgt["portals"] = portals + return cur+1, tpgt + + +def parse_target(fabric, txt, cur): + target = dict(wwn=txt[cur+1], fabric=fabric) + tpgts = [] + extra = 0 + # handle multiple tpgts + if txt[cur+2] == "{": + extra = 1 + cur += 2 + extra + while txt[cur] != "}": + cur, tpgt = parse_tpgt(txt, cur) + tpgts.append(tpgt) + target["tpgs"] = tpgts + return cur+extra, target + +def parse_fabric(txt, cur): + fabric = txt[cur+1] + cur += 3 + while txt[cur] != "}": + if txt[cur] == "discovery_auth": + cur, disco = parse_attributes(txt, cur+2) + new_disco = {} + if disco.get("enable"): + new_disco["discovery_enable_auth"] = disco.get("enable") + if disco.get("userid"): + new_disco["discovery_userid"] = disco.get("userid") + if disco.get("password"): + new_disco["discovery_password"] = disco.get("password") + if disco.get("mutual_userid"): + new_disco["discovery_mutual_userid"] = disco.get("mutual_userid") + if disco.get("mutual_password"): + new_disco["discovery_mutual_password"] = disco.get("mutual_password") + new_disco["name"] = "iscsi" + fabs.append(new_disco) + continue + if txt[cur] == "target": + cur, t = parse_target(fabric, txt, cur) + targs.append(t) + continue + return cur + +sos = [] +fabs = [] +targs = [] + +# a basic tokenizer that splits on whitespace and handles double quotes +def split(s): + new_lst = [] + in_quotes = False + new_str = [] + for c in s: + if c not in " \n\t\"": + new_str.append(c) + elif c == '\"' and in_quotes == False: + in_quotes = True + elif c == '\"' and in_quotes == True: + in_quotes = False + if len(new_str) == 0: + # don't include things that are set to '""' + del new_lst[-1] + elif in_quotes == True: # append ws if in quotes + new_str.append(c) + elif len(new_str): # not in quotes, break on ws if anything in new_str + new_lst.append("".join(new_str)) + new_str = [] + else: + pass # drop ws + + return new_lst + +def parse(txt, cur): + cur = 0 + end = len(txt) - 1 + while cur != end: + if txt[cur] == "storage": + cur, d = parse_storage(txt, cur) + sos.append(d) + elif txt[cur] == "fabric": + cur = parse_fabric(txt, cur) + +with open("/etc/target/scsi_target.lio") as f: + txt = f.read() + txt = split(txt) + cur = parse(txt, 0) + +output = dict(storage_objects=sos, fabric_modules=fabs, targets=targs) + +print(json.dumps(output, indent=2, sort_keys=True)) + -- GitLab From c58895226e62be26d3ef940b21e09289ca358eed Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 12 May 2016 11:09:49 -0700 Subject: [PATCH 23/70] convert-to-json: Fix to handle fileio and other issues parse_fileio was a cut-n-paste from block, missing key attributes like size and buffered. Fixed. Also, bodge up better handling for single vs multiple tpgs. Clarify referring to tpgs versus the tpg tag (tpgt, an integer). Signed-off-by: Andy Grover --- scripts/convert-to-json | 47 +++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/scripts/convert-to-json b/scripts/convert-to-json index 6f84c73..3483e0a 100755 --- a/scripts/convert-to-json +++ b/scripts/convert-to-json @@ -93,6 +93,14 @@ def parse_fileio(txt, cur): so["dev"] = txt[cur+1] cur += 2 continue + if txt[cur] == "size": + so["size"] = human_to_bytes(txt[cur+1]) + cur += 2 + continue + if txt[cur] == "buffered": + # skip, recent LIO doesn't use for fileio + cur += 2 + continue if txt[cur] == "attribute": cur, so["attributes"] = parse_attributes(txt, cur+2) continue @@ -157,7 +165,7 @@ def parse_mapped_lun(txt, cur): mlun["write_protect"] = bool(parse_yesno(txt[cur+1])) cur += 2 continue - return cur, mlun + return cur+1, mlun def parse_acl(txt, cur): acl = dict(node_wwn=txt[cur+1]) @@ -178,27 +186,30 @@ def parse_acl(txt, cur): acl["mapped_luns"] = mapped_luns return cur+1, acl -def parse_tpgt(txt, cur): - tpgt = dict(tag = int(txt[cur+1])) +def parse_tpg(tag, txt, cur): + if tag is None: + tag = int(txt[cur+1]) + cur += 2 + tpg = dict(tag=tag) luns = [] acls = [] portals = [] cur += 3 while txt[cur] != "}": if txt[cur] == "enable": - tpgt["enable"] = parse_yesno(txt[cur+1]) + tpg["enable"] = parse_yesno(txt[cur+1]) cur += 2 continue if txt[cur] == "attribute": - cur, tpgt["attributes"] = parse_attributes(txt, cur+2) + cur, tpg["attributes"] = parse_attributes(txt, cur+2) continue if txt[cur] == "parameter": - cur, tpgt["parameters"] = parse_attributes(txt, cur+2) + cur, tpg["parameters"] = parse_attributes(txt, cur+2) continue if txt[cur] == "auth": cur, auth = parse_attributes(txt, cur+2) if len(auth): - tpgt["auth"] = auth + tpg["auth"] = auth continue if txt[cur] == "lun": cur, l = parse_lun(txt, cur) @@ -213,27 +224,31 @@ def parse_tpgt(txt, cur): portal = dict(ip_address=ip, port=port) portals.append(portal) cur += 2 + continue if len(luns): - tpgt["luns"] = luns + tpg["luns"] = luns if len(acls): - tpgt["node_acls"] = acls + tpg["node_acls"] = acls if len(portals): - tpgt["portals"] = portals - return cur+1, tpgt + tpg["portals"] = portals + return cur+1, tpg def parse_target(fabric, txt, cur): target = dict(wwn=txt[cur+1], fabric=fabric) - tpgts = [] - extra = 0 + tpgs = [] + tpgt = None # handle multiple tpgts if txt[cur+2] == "{": extra = 1 + else: + extra = 0 + tpgt = int(txt[cur+3]) cur += 2 + extra while txt[cur] != "}": - cur, tpgt = parse_tpgt(txt, cur) - tpgts.append(tpgt) - target["tpgs"] = tpgts + cur, tpg = parse_tpg(tpgt, txt, cur) + tpgs.append(tpg) + target["tpgs"] = tpgs return cur+extra, target def parse_fabric(txt, cur): -- GitLab From 9134f76aab9a124e6aa1cb996c05fa8076f7debd Mon Sep 17 00:00:00 2001 From: colml Date: Fri, 3 Jun 2016 09:32:47 +0100 Subject: [PATCH 24/70] Fix mount race condition --- rtslib/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index acf8aa5..c9f8213 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -440,7 +440,8 @@ def mount_configfs(): stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = process.communicate() - if process.returncode != 0: + if process.returncode != 0 and not os.path.ismount( + "/sys/kernel/config"): raise RTSLibError("Cannot mount configfs") def dict_remove(d, items): -- GitLab From cf92f069fe5786a4caf1655933d6a3791f1ed176 Mon Sep 17 00:00:00 2001 From: Nayef Ghattas Date: Mon, 13 Jun 2016 23:19:22 +0200 Subject: [PATCH 25/70] Changed example-debian folder for compatibility with dpkg-buildpackage --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bfc9010..8716fe2 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ build/release-stamp: mv $${spectmpl} $$(basename $${spectmpl} .tmpl); \ done; \ rm -r example-rpm + @mv build/${PKGNAME}-${VERSION}/example-debian build/${PKGNAME}-${VERSION}/debian @echo "Generating rpm changelog..." @( \ version=$$(basename $$(git describe HEAD --tags | tr - .)); \ @@ -93,7 +94,7 @@ build/release-stamp: echo; \ echo " -- $${author} $${date}"; \ echo; \ - ) > build/${PKGNAME}-${VERSION}/example-debian/changelog + ) > build/${PKGNAME}-${VERSION}/debian/changelog @find build/${PKGNAME}-${VERSION}/ -exec \ touch -t $$(date -d @$$(git show -s --format="format:%at") \ +"%Y%m%d%H%M.%S") {} \; -- GitLab From 95754007b2e1cbf7b65f8472c69b30d72ceffe2a Mon Sep 17 00:00:00 2001 From: Jared Hulbert Date: Tue, 21 Jun 2016 16:02:14 -0700 Subject: [PATCH 26/70] fixes to debian package scripts --- example-debian/control | 2 +- example-debian/rules | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example-debian/control b/example-debian/control index 86ddfc2..0c831db 100644 --- a/example-debian/control +++ b/example-debian/control @@ -2,7 +2,7 @@ Source: rtslib-fb Section: python Priority: optional Maintainer: Andy Grover -Build-Depends: debhelper(>= 8), python, python3 , python-epydoc, python-setuptools, python3-setuptools +Build-Depends: debhelper(>= 8), python, python3 , python-epydoc, python-setuptools, python3-setuptools, python-six, python-pyudev, dh-python Standards-Version: 3.9.4 X-Python-Version: >= 2.6 X-Python3-Version: >= 3.1 diff --git a/example-debian/rules b/example-debian/rules index 6250ceb..3eae1a1 100755 --- a/example-debian/rules +++ b/example-debian/rules @@ -21,7 +21,7 @@ export http_proxy = http://127.0.0.1:9 build-python%: python$* setup.py build -override_dh_auto_build: $(PYTHON3:%=build-python%) +override_dh_auto_build: $(PYTHON3:%=build-python%) $(PYTHON2:%=build-python%) dh_auto_build @@ -29,7 +29,7 @@ install-python%: python$* setup.py install --root=$(install_dir) --install-layout=deb -override_dh_auto_install: $(PYTHON3:%=install-python%) +override_dh_auto_install: $(PYTHON3:%=install-python%) $(PYTHON2:%=install-python%) dh_auto_install # for using python 3 for targetctl # even though it is overwritten by python2 setup -- GitLab From 8277306b4d30e93b6985302c25910550d4172fff Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 11 Aug 2016 10:26:20 -0700 Subject: [PATCH 27/70] Handle if LIO components are compiled in the kernel Attempt to do the thing that would require the module first, and if it fails then attempt to load the module and retry. This should handle all combinations of modules, compiled-in etc. and return an error if a needed component is neither compiled in or as a module. Fixes #64 Signed-off-by: Andy Grover --- rtslib/fabric.py | 7 +++++-- rtslib/root.py | 15 +++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/rtslib/fabric.py b/rtslib/fabric.py index b5cc4d9..885727e 100644 --- a/rtslib/fabric.py +++ b/rtslib/fabric.py @@ -152,8 +152,11 @@ class _BaseFabricModule(CFSNode): def _check_self(self): if not self.exists: - modprobe(self.kernel_module) - self._create_in_cfs_ine('any') + try: + self._create_in_cfs_ine('any') + except RTSLibError: + modprobe(self.kernel_module) + self._create_in_cfs_ine('any') super(_BaseFabricModule, self)._check_self() def has_feature(self, feature): diff --git a/rtslib/root.py b/rtslib/root.py index 083a2c2..b65825f 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -62,10 +62,17 @@ class RTSRoot(CFSNode): base kernel modules (tcm) ''' super(RTSRoot, self).__init__() - modprobe('configfs') - mount_configfs() - modprobe('target_core_mod') - self._create_in_cfs_ine('any') + try: + mount_configfs() + except RTSLibError: + modprobe('configfs') + mount_configfs() + + try: + self._create_in_cfs_ine('any') + except RTSLibError: + modprobe('target_core_mod') + self._create_in_cfs_ine('any') def _list_targets(self): self._check_self() -- GitLab From ba87a191edffe01235f86cfb145aa6834f5f0057 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 12 Aug 2016 10:19:14 -0700 Subject: [PATCH 28/70] Rework convert_scsi_path_to_hctl and convert_scsi_hctl_to_path Do not check for type=disk, this is not a requirement. Raise an RTSLibError if not found, rather than returning None. Rework PSCSIStorageObject._configure to use the new semantics, and avoid else clauses in try blocks, since they are confusing and not strictly needed. Properly qualify DeviceNotFoundError as pyudev.DeviceNotFoundError. Signed-off-by: Andy Grover --- rtslib/tcm.py | 23 ++++++++++++----------- rtslib/utils.py | 31 +++++++++++-------------------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/rtslib/tcm.py b/rtslib/tcm.py index bdeafd2..2ea5d0f 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -313,11 +313,14 @@ class PSCSIStorageObject(StorageObject): def _configure(self, dev): self._check_self() - # Use H:C:T:L format or preserve the path given by the user. + # Use H:C:T:L format or use the path given by the user. try: + # assume 'dev' is the path, try to get h:c:t:l values (hostid, channelid, targetid, lunid) = \ convert_scsi_path_to_hctl(dev) - except TypeError: + udev_path = dev.strip() + except: + # Oops, maybe 'dev' is in h:c:t:l format, try to get udev_path try: (hostid, channelid, targetid, lunid) = dev.split(':') hostid = int(hostid) @@ -329,15 +332,13 @@ class PSCSIStorageObject(StorageObject): + "path, and dev " + "parameter not in H:C:T:L " + "format: %s" % dev) - else: - udev_path = convert_scsi_hctl_to_path(hostid, - channelid, - targetid, - lunid) - if not udev_path: - raise RTSLibError("SCSI device does not exist") - else: - udev_path = dev.strip() + + udev_path = convert_scsi_hctl_to_path(hostid, + channelid, + targetid, + lunid) + + # -- now have all 5 values or have errored out -- if is_dev_in_use(udev_path): raise RTSLibError("Cannot configure StorageObject because " diff --git a/rtslib/utils.py b/rtslib/utils.py index c9f8213..969d2d7 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -246,26 +246,15 @@ def convert_scsi_path_to_hctl(path): @param path: The udev path to the SCSI block device. @type path: string @return: An (host, controller, target, lun) tuple of integer - values representing the SCSI ID of the device, or None if no - match is found. + values representing the SCSI ID of the device, or raise RTSLibError. ''' try: path = os.path.realpath(path) device = pyudev.Device.from_device_file(_CONTEXT, path) - except (pyudev.DeviceNotFoundError, EnvironmentError, ValueError): - return None - - if device.device_type != u'disk': - return None - - parent = device.find_parent(subsystem='scsi') - if parent is None: - return None - - try: + parent = device.find_parent(subsystem='scsi') return [int(data) for data in parent.sys_name.split(':')] - except (ValueError, TypeError): - return None + except: + raise RTSLibError("Could not convert scsi path to hctl") def convert_scsi_hctl_to_path(host, controller, target, lun): ''' @@ -288,7 +277,7 @@ def convert_scsi_hctl_to_path(host, controller, target, lun): @type target: int @param lun: The SCSI Logical Unit Number. @type lun: int - @return: A string for the canonical path to the device, or empty string. + @return: A string for the canonical path to the device, or raise RTSLibError. ''' try: host = int(host) @@ -302,16 +291,18 @@ def convert_scsi_hctl_to_path(host, controller, target, lun): hctl = [host, controller, target, lun] try: scsi_device = pyudev.Device.from_name(_CONTEXT, 'scsi', ':'.join(hctl)) - except DeviceNotFoundError: - return '' + except pyudev.DeviceNotFoundError: + raise RTSLibError("Could not find path for SCSI hctl") devices = _CONTEXT.list_devices( subsystem='block', - DEVTYPE='disk', parent=scsi_device ) - return next((dev.device_node for dev in devices), '') + path = next((dev.device_node for dev in devices), '') + if path == None: + raise RTSLibError("Could not find path for SCSI hctl") + return path def generate_wwn(wwn_type): ''' -- GitLab From 3d47f832115a71a914c9658ebcbc6105f97345e3 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Mon, 22 Aug 2016 15:59:56 +0200 Subject: [PATCH 29/70] Change the URL of the GitHub repo to the open-iscsi organization Signed-off-by: Christophe Vu-Brugier --- README.md | 4 ++-- doc/saveconfig.json.5 | 2 +- doc/targetctl.8 | 4 ++-- setup.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8825b8f..a049e2c 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ Since rtslib-fb is used most often with targetcli-fb, the targetcli-fb mailing should be used for rtslib-fb discussion. * Mailing list: [targetcli-fb-devel](https://lists.fedorahosted.org/mailman/listinfo/targetcli-fb-devel) - * Source repo: [GitHub](https://github.com/agrover/rtslib-fb) - * Bugs: [GitHub](https://github.com/agrover/rtslib-fb/issues) or [Trac](https://fedorahosted.org/targetcli-fb/) + * Source repo: [GitHub](https://github.com/open-iscsi/rtslib-fb) + * Bugs: [GitHub](https://github.com/open-iscsi/rtslib-fb/issues) or [Trac](https://fedorahosted.org/targetcli-fb/) * Tarballs: [fedorahosted](https://fedorahosted.org/releases/t/a/targetcli-fb/) In-repo packaging diff --git a/doc/saveconfig.json.5 b/doc/saveconfig.json.5 index b22efb1..cc4513e 100644 --- a/doc/saveconfig.json.5 +++ b/doc/saveconfig.json.5 @@ -243,4 +243,4 @@ Man page written by Andy Grover . .SH REPORTING BUGS Report bugs via .br -or +or diff --git a/doc/targetctl.8 b/doc/targetctl.8 index f87aaaf..923383c 100644 --- a/doc/targetctl.8 +++ b/doc/targetctl.8 @@ -25,7 +25,7 @@ permissions will be set to only allow root access. If .B config-file is not supplied, .B targetctl -will use the default file location, +will use the default file location, .BR /etc/target/saveconfig.json. .P .B targetctl restore [config-file] @@ -60,4 +60,4 @@ Man page written by Andy Grover . .SH REPORTING BUGS Report bugs via .br -or +or diff --git a/setup.py b/setup.py index 2087187..b90a76c 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup ( license = 'Apache 2.0', maintainer = 'Andy Grover', maintainer_email = 'agrover@redhat.com', - url = 'http://github.com/agrover/rtslib-fb', + url = 'http://github.com/open-iscsi/rtslib-fb', packages = ['rtslib_fb', 'rtslib'], scripts = ['scripts/targetctl'], classifiers = [ -- GitLab From e04d87b5f8e5c2252e18dab7a8778ee03a1bf171 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 3 Oct 2016 10:54:33 -0700 Subject: [PATCH 30/70] Implement support for cxgbit offload Just like the 'iser' value within np/, save and restore the 'cxgbit' value. However, the property is called 'offload'. This will let us handle other offloads with the same property in the future if and when they are supported. Signed-off-by: Andy Grover --- doc/saveconfig.json.5 | 2 ++ rtslib/target.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/doc/saveconfig.json.5 b/doc/saveconfig.json.5 index cc4513e..42c6317 100644 --- a/doc/saveconfig.json.5 +++ b/doc/saveconfig.json.5 @@ -195,6 +195,8 @@ values are (number). .I iser (boolean) is an optional value to enable iSER. +.I offload +(boolean) is an optional value to enable hardware offload. .SS node_acls This contains information about explicit initiator LUN mappings. diff --git a/rtslib/target.py b/rtslib/target.py index db44ebc..c2836db 100644 --- a/rtslib/target.py +++ b/rtslib/target.py @@ -728,10 +728,27 @@ class NetworkPortal(CFSNode): if os.path.isfile(path): raise RTSLibError("Cannot change iser") + def _get_offload(self): + try: + # only offload at the moment is cxgbit + return bool(int(fread("%s/cxgbit" % self.path))) + except IOError: + return False + + def _set_offload(self, boolean): + path = "%s/cxgbit" % self.path + try: + fwrite(path, str(int(boolean))) + except IOError: + # b/w compat: don't complain if cxgbit entry is missing + if os.path.isfile(path): + raise RTSLibError("Cannot change offload") + # NetworkPortal public stuff def delete(self): self.iser = False + self.offload = False super(NetworkPortal, self).delete() parent_tpg = property(_get_parent_tpg, @@ -743,6 +760,9 @@ class NetworkPortal(CFSNode): iser = property(_get_iser, _set_iser, doc="Get or set a boolean value representing if this " \ + "NetworkPortal supports iSER.") + offload = property(_get_offload, _set_offload, + doc="Get or set a boolean value representing if this " \ + + "NetworkPortal supports offload.") @classmethod def setup(cls, tpg_obj, p, err_func): @@ -756,6 +776,7 @@ class NetworkPortal(CFSNode): try: np = cls(tpg_obj, p['ip_address'], p['port']) np.iser = p.get('iser', False) + np.offload = p.get('offload', False) except (RTSLibError, KeyError) as e: err_func("Creating NetworkPortal object %s:%s failed: %s" % (p['ip_address'], p['port'], e)) @@ -765,6 +786,7 @@ class NetworkPortal(CFSNode): d['port'] = self.port d['ip_address'] = self.ip_address d['iser'] = self.iser + d['offload'] = self.offload return d -- GitLab From d7ccfcf592a2051ddba9eecb2843aee3f0da8a8d Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 3 Oct 2016 10:58:42 -0700 Subject: [PATCH 31/70] version 2.1.fb61 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b90a76c..48f6282 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.60', + version = '2.1.61', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From b3f692f60bc933dce0c8c727d9028003014eeddb Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Fri, 21 Oct 2016 10:42:03 +0200 Subject: [PATCH 32/70] Remove build scripts for RPM and Debian packages Remove the "example-debian" and "example-rpm" directories because our in-repo packaging is less complete than the packaging made by distributions. Finally, this patch adds a few links to RPM and Debian build scripts in our README.md. Signed-off-by: Christophe Vu-Brugier --- Makefile | 70 ------------------- README.md | 16 ++--- example-debian/README.Debian | 13 ---- example-debian/compat | 1 - example-debian/control | 29 -------- example-debian/copyright | 24 ------- example-debian/python-rtslib-fb-docs.doc-base | 9 --- example-debian/python-rtslib-fb.install | 2 - example-debian/python-rtslib-fb.manpages | 2 - example-debian/python3-rtslib-fb.install | 2 - example-debian/python3-rtslib-fb.manpages | 2 - example-debian/python3-rtslib-fb.target.init | 41 ----------- example-debian/rules | 51 -------------- example-debian/source/format | 1 - example-rpm/python-rtslib.spec.tmpl | 44 ------------ 15 files changed, 7 insertions(+), 300 deletions(-) delete mode 100644 example-debian/README.Debian delete mode 100644 example-debian/compat delete mode 100644 example-debian/control delete mode 100644 example-debian/copyright delete mode 100644 example-debian/python-rtslib-fb-docs.doc-base delete mode 100644 example-debian/python-rtslib-fb.install delete mode 100644 example-debian/python-rtslib-fb.manpages delete mode 100644 example-debian/python3-rtslib-fb.install delete mode 100644 example-debian/python3-rtslib-fb.manpages delete mode 100644 example-debian/python3-rtslib-fb.target.init delete mode 100755 example-debian/rules delete mode 100644 example-debian/source/format delete mode 100644 example-rpm/python-rtslib.spec.tmpl diff --git a/Makefile b/Makefile index 8716fe2..f72f324 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,6 @@ VERSION = $$(basename $$(git describe --tags | tr - . | grep -o '[0-9].*$$')) all: @echo "Usage:" @echo - @echo " make deb - Builds debian packages." - @echo " make rpm - Builds rpm packages." @echo " make release - Generates the release tarball." @echo @echo " make clean - Cleanup the local repository build files." @@ -32,15 +30,7 @@ all: clean: @rm -fv ${NAME}/*.pyc ${NAME}/*.html @rm -frv ${NAME}.egg-info MANIFEST build - @rm -frv debian/tmp - @rm -fv build-stamp - @rm -fv dpkg-buildpackage.log dpkg-buildpackage.version - @rm -frv *.rpm - @rm -fv debian/files debian/*.log debian/*.substvars - @rm -frv debian/${PKGNAME}-doc/ debian/python2.5-${PKGNAME}/ - @rm -frv debian/python2.6-${PKGNAME}/ debian/python-${PKGNAME}/ @rm -frv results - @rm -fv rpm/*.spec *.spec rpm/sed* sed* @rm -frv ${PKGNAME}-* @echo "Finished cleanup." @@ -59,42 +49,6 @@ build/release-stamp: @echo "Fixing version string..." @sed -i "s/__version__ = .*/__version__ = '${VERSION}'/g" \ build/${PKGNAME}-${VERSION}/${NAME}/__init__.py - @echo "Generating rpm specfile from template..." - @cd build/${PKGNAME}-${VERSION}; \ - for spectmpl in example-rpm/*.spec.tmpl; do \ - sed -i "s/Version:\( *\).*/Version:\1${VERSION}/g" $${spectmpl}; \ - mv $${spectmpl} $$(basename $${spectmpl} .tmpl); \ - done; \ - rm -r example-rpm - @mv build/${PKGNAME}-${VERSION}/example-debian build/${PKGNAME}-${VERSION}/debian - @echo "Generating rpm changelog..." - @( \ - version=$$(basename $$(git describe HEAD --tags | tr - .)); \ - author=$$(git show HEAD --format="format:%an <%ae>" -s); \ - date=$$(git show HEAD --format="format:%ad" -s \ - | awk '{print $$1,$$2,$$3,$$5}'); \ - hash=$$(git show HEAD --format="format:%H" -s); \ - echo '* '"$${date} $${author} $${version}-1"; \ - echo " - Generated from git commit $${hash}."; \ - ) >> $$(ls build/${PKGNAME}-${VERSION}/*.spec) - @echo "Generating debian changelog..." - @( \ - version=$$(basename $$(git describe HEAD --tags | tr - . | grep -o '[0-9].*$$')); \ - author=$$(git show HEAD --format="format:%an <%ae>" -s); \ - date=$$(git show HEAD --format="format:%aD" -s); \ - day=$$(git show HEAD --format='format:%ai' -s \ - | awk '{print $$1}' \ - | awk -F '-' '{print $$3}' | sed 's/^0/ /g'); \ - date=$$(echo $${date} \ - | awk '{print $$1, "'"$${day}"'", $$3, $$4, $$5, $$6}'); \ - hash=$$(git show HEAD --format="format:%H" -s); \ - echo "${PKGNAME} ($${version}) unstable; urgency=low"; \ - echo; \ - echo " * Generated from git commit $${hash}."; \ - echo; \ - echo " -- $${author} $${date}"; \ - echo; \ - ) > build/${PKGNAME}-${VERSION}/debian/changelog @find build/${PKGNAME}-${VERSION}/ -exec \ touch -t $$(date -d @$$(git show -s --format="format:%at") \ +"%Y%m%d%H%M.%S") {} \; @@ -108,27 +62,3 @@ build/release-stamp: @echo "Generated release tarball:" @echo " $$(ls dist/${PKGNAME}-${VERSION}.tar.gz)" @touch build/release-stamp - -deb: release build/deb-stamp -build/deb-stamp: - @echo "Building debian packages..." - @cd build/${PKGNAME}-${VERSION}; \ - dpkg-buildpackage -rfakeroot -us -uc - @mv build/*_${VERSION}_*.deb dist/ - @echo "Generated debian packages:" - @for pkg in $$(ls dist/*_${VERSION}_*.deb); do echo " $${pkg}"; done - @touch build/deb-stamp - -rpm: release build/rpm-stamp -build/rpm-stamp: - @echo "Building rpm packages..." - @mkdir -p build/rpm - @build=$$(pwd)/build/rpm; dist=$$(pwd)/dist/; rpmbuild \ - --define "_topdir $${build}" --define "_sourcedir $${dist}" \ - --define "_rpmdir $${build}" --define "_buildir $${build}" \ - --define "_srcrpmdir $${build}" -ba build/${PKGNAME}-${VERSION}/*.spec - @mv build/rpm/*-${VERSION}*.src.rpm dist/ - @mv build/rpm/*/*-${VERSION}*.rpm dist/ - @echo "Generated rpm packages:" - @for pkg in $$(ls dist/*-${VERSION}*.rpm); do echo " $${pkg}"; done - @touch build/rpm-stamp diff --git a/README.md b/README.md index a049e2c..a17a21f 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,13 @@ mailing should be used for rtslib-fb discussion. * Bugs: [GitHub](https://github.com/open-iscsi/rtslib-fb/issues) or [Trac](https://fedorahosted.org/targetcli-fb/) * Tarballs: [fedorahosted](https://fedorahosted.org/releases/t/a/targetcli-fb/) -In-repo packaging ------------------ -Packaging scripts for RPM and DEB are included, but these are to make end-user -custom packaging easier -- distributions tend to maintain their own packaging -scripts separately. If you run into issues with packaging, start with opening -a bug on your distro's bug reporting system. - -Some people do use these scripts, so we want to keep them around. Fixes for -any breakage you encounter are welcome. +Packages +-------- +rtslib-fb is packaged for a number of Linux distributions including +RHEL, +[Fedora](https://apps.fedoraproject.org/packages/python-rtslib), +openSUSE, Arch Linux, and +[Debian](https://tracker.debian.org/pkg/python-rtslib-fb). "fb" -- "free branch" --------------------- diff --git a/example-debian/README.Debian b/example-debian/README.Debian deleted file mode 100644 index 78a1f14..0000000 --- a/example-debian/README.Debian +++ /dev/null @@ -1,13 +0,0 @@ -Copyright (c) 2011-2013 by Datera, Inc - -Licensed under the Apache License, Version 2.0 (the "License"); you may -not use this file except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. diff --git a/example-debian/compat b/example-debian/compat deleted file mode 100644 index 7f8f011..0000000 --- a/example-debian/compat +++ /dev/null @@ -1 +0,0 @@ -7 diff --git a/example-debian/control b/example-debian/control deleted file mode 100644 index 0c831db..0000000 --- a/example-debian/control +++ /dev/null @@ -1,29 +0,0 @@ -Source: rtslib-fb -Section: python -Priority: optional -Maintainer: Andy Grover -Build-Depends: debhelper(>= 8), python, python3 , python-epydoc, python-setuptools, python3-setuptools, python-six, python-pyudev, dh-python -Standards-Version: 3.9.4 -X-Python-Version: >= 2.6 -X-Python3-Version: >= 3.1 - -Package: python-rtslib-fb-docs -Architecture: all -Description: RisingTide Systems LIO target Python API (Documentation). - -Package: python-rtslib-fb -Architecture: all -Depends: ${python:Depends}, ${misc:Depends} -Provides: ${python:Provides} -Suggests: python-rtslib-fb-doc -Conflicts: python-rtslib, rtsadmin-frozen -Description: RisingTide Systems LIO target Python 2 API (free branch). - -Package: python3-rtslib-fb -Architecture: all -Depends: ${python3:Depends}, ${misc:Depends} -Suggests: python-rtslib-fb-doc -Conflicts: lio-utils -Replaces: targetcli-fb (<< 2.1.fb32) -Description: RisingTide Systems LIO target Python 3 API (free branch). - This includes the LIO rc scripts to start and stop the target. diff --git a/example-debian/copyright b/example-debian/copyright deleted file mode 100644 index 638eeb9..0000000 --- a/example-debian/copyright +++ /dev/null @@ -1,24 +0,0 @@ -This package was originally debianized by Jerome Martin -on Fri Nov 18 12:00:01 UTC 2009. It is currently maintained by Andy Grover -. - -Upstream Author: Jerome Martin - -Copyright: - -This file is part of RTSLib. -Copyright (c) 2011-2013 by Datera, Inc - -Licensed under the Apache License, Version 2.0 (the "License"); you may -not use this file except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - - diff --git a/example-debian/python-rtslib-fb-docs.doc-base b/example-debian/python-rtslib-fb-docs.doc-base deleted file mode 100644 index 9d070d3..0000000 --- a/example-debian/python-rtslib-fb-docs.doc-base +++ /dev/null @@ -1,9 +0,0 @@ -Document: python-rtslib-fb -Title: python-rtslib documentation -Author: Jerome Martin -Abstract: Python library for configuring the Linux kernel-based multiprotocol SCSI target (LIO) -Section: Programming/Python - -Format: HTML -Index: /usr/share/doc/python-rtslib-fb-docs/html/index.html -Files: /usr/share/doc/python-rtslib-fb-docs/html/*.html diff --git a/example-debian/python-rtslib-fb.install b/example-debian/python-rtslib-fb.install deleted file mode 100644 index 05ca243..0000000 --- a/example-debian/python-rtslib-fb.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/lib/python2* -usr/bin diff --git a/example-debian/python-rtslib-fb.manpages b/example-debian/python-rtslib-fb.manpages deleted file mode 100644 index f620da6..0000000 --- a/example-debian/python-rtslib-fb.manpages +++ /dev/null @@ -1,2 +0,0 @@ -doc/saveconfig.json.5 -doc/targetctl.8 diff --git a/example-debian/python3-rtslib-fb.install b/example-debian/python3-rtslib-fb.install deleted file mode 100644 index bf7cd62..0000000 --- a/example-debian/python3-rtslib-fb.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/lib/python3* -usr/bin diff --git a/example-debian/python3-rtslib-fb.manpages b/example-debian/python3-rtslib-fb.manpages deleted file mode 100644 index f620da6..0000000 --- a/example-debian/python3-rtslib-fb.manpages +++ /dev/null @@ -1,2 +0,0 @@ -doc/saveconfig.json.5 -doc/targetctl.8 diff --git a/example-debian/python3-rtslib-fb.target.init b/example-debian/python3-rtslib-fb.target.init deleted file mode 100644 index b011013..0000000 --- a/example-debian/python3-rtslib-fb.target.init +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -### BEGIN INIT INFO -# Provides: target -# Default-Start: 3 4 5 -# Default-Stop: 0 1 2 6 -# Required-Start: $local_fs $network -# Required-Stop: $local_fs $network -# Short-Description: Start LIO targets -# Description: Loads configfs and restores LIO config with targetctl -### END INIT INFO - - -case "$1" in - start) - echo "Loading lio configuration" - /usr/bin/targetctl restore - if [[ $? -gt 0 ]]; then - exit 1 - fi - ;; - - stop) - echo "Unloading lio configuration" - /usr/bin/targetctl clear - if [[ $? -gt 0 ]]; then - exit 1 - fi - ;; - - restart|force-reload) - $0 stop - sleep 3 - $0 start - ;; - - *) - echo "usage: $0 {start|stop|restart|force-reload}" -esac - -exit 0 diff --git a/example-debian/rules b/example-debian/rules deleted file mode 100755 index 3eae1a1..0000000 --- a/example-debian/rules +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/make -f - -build_dir = build -install_dir = $(CURDIR)/debian/tmp -pkgname = rtslib-fb -name = rtslib - -#export DH_VERBOSE=1 - -PYTHON2=$(shell pyversions -vr) -PYTHON3=$(shell py3versions -vr) - -#prevent internet access to use PyPi -export http_proxy = http://127.0.0.1:9 - - -%: - dh $@ --with python2,python3 - - -build-python%: - python$* setup.py build - -override_dh_auto_build: $(PYTHON3:%=build-python%) $(PYTHON2:%=build-python%) - dh_auto_build - - -install-python%: - python$* setup.py install --root=$(install_dir) --install-layout=deb - - -override_dh_auto_install: $(PYTHON3:%=install-python%) $(PYTHON2:%=install-python%) - dh_auto_install - # for using python 3 for targetctl - # even though it is overwritten by python2 setup - sed -i '1s|/usr/bin/python$$|/usr/bin/python3|' $(install_dir)/usr/bin/targetctl - - -override_dh_installdocs: - cd $(build_dir); epydoc --no-sourcecode --html -n $(pkgname) \ - --exclude configobj ../$(name)/*.py - dh_installdocs $(build_dir)/html - -override_dh_installinit: - dh_installinit --name target - - -override_dh_auto_clean: - dh_auto_clean - rm -rf build - rm -rf *.egg-info diff --git a/example-debian/source/format b/example-debian/source/format deleted file mode 100644 index 89ae9db..0000000 --- a/example-debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/example-rpm/python-rtslib.spec.tmpl b/example-rpm/python-rtslib.spec.tmpl deleted file mode 100644 index 2191b7a..0000000 --- a/example-rpm/python-rtslib.spec.tmpl +++ /dev/null @@ -1,44 +0,0 @@ -%define oname rtslib-fb - -Name: python-rtslib -License: Apache License 2.0 -Group: System Environment/Libraries -Summary: A framework to implement simple but nice CLIs. -Version: VERSION -Release: 1%{?dist} -URL: http://www.risingtidesystems.com/git/ -Source: %{oname}-%{version}.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-rpmroot -BuildArch: noarch -BuildRequires: python-devel, epydoc -Vendor: Datera, Inc. - -%description -API for RisingTide Systems generic SCSI target. - -%prep -%setup -q -n %{oname}-%{version} - -%build -%{__python} setup.py build -mkdir -p doc -epydoc --no-sourcecode --html -n %{oname} --exclude configobj rtslib_fb/*.py -mv html doc/ - -%install -rm -rf %{buildroot} -%{__python} setup.py install --skip-build --root %{buildroot} --prefix usr -mkdir -p %{buildroot}/usr/share/doc/python-rtslib-doc-%{version} -cp -r doc/* %{buildroot}/usr/share/doc/python-rtslib-doc-%{version}/ - -%clean -rm -rf %{buildroot} - -%files -%defattr(-,root,root,-) -%{python_sitelib} -%{_bindir}/targetctl -/usr/share/doc/python-rtslib-doc-%{version} -%doc COPYING README.md - -%changelog -- GitLab From d420b805b1c098b08813bfc29421269ef7455570 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Fri, 28 Oct 2016 14:53:33 +0200 Subject: [PATCH 33/70] Add Gentoo to the list of distributions that ship rtslib-fb Signed-off-by: Christophe Vu-Brugier --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a17a21f..89f6978 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ Packages rtslib-fb is packaged for a number of Linux distributions including RHEL, [Fedora](https://apps.fedoraproject.org/packages/python-rtslib), -openSUSE, Arch Linux, and +openSUSE, Arch Linux, +[Gentoo](https://packages.gentoo.org/packages/dev-python/rtslib-fb), and [Debian](https://tracker.debian.org/pkg/python-rtslib-fb). "fb" -- "free branch" -- GitLab From 925281b9f036552f7fe252da0e05407f4ed2db62 Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Thu, 3 Nov 2016 22:35:29 +0100 Subject: [PATCH 34/70] Depend on pyudev during runtime In v2.1.fb60 an unconditional dependency to pyudev was added, which needs to be specified in the egg info, otherwise pip install might fail with e.g. File "local/lib/python2.7/site-packages/rtslib/utils.py", line 30, in import pyudev ImportError: No module named pyudev --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 48f6282..7770032 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup ( url = 'http://github.com/open-iscsi/rtslib-fb', packages = ['rtslib_fb', 'rtslib'], scripts = ['scripts/targetctl'], + install_requires = ['pyudev >= 0.16.1'], classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", -- GitLab From 76b3acea16b58852952c2ea813b4777b99cab7c8 Mon Sep 17 00:00:00 2001 From: Steven Royer Date: Wed, 9 Nov 2016 12:42:45 -0600 Subject: [PATCH 35/70] Add IBM PowerVM vscsi fabric Adds support for the IBM PowerVM virtual SCSI fabric. This uses the ibmvscsis kernel module that merged in 4.8 and later kernels. Signed-off-by: Steven Royer --- rtslib/fabric.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rtslib/fabric.py b/rtslib/fabric.py index 885727e..2887783 100644 --- a/rtslib/fabric.py +++ b/rtslib/fabric.py @@ -447,6 +447,19 @@ class XenPvScsiFabricModule(_BaseFabricModule): self.kernel_module = "xen-scsiback" +class IbmvscsisFabricModule(_BaseFabricModule): + def __init__(self): + super(IbmvscsisFabricModule, self).__init__('ibmvscsis') + self.features = () + self.kernel_module = "ibmvscsis" + + @property + def wwns(self): + for wwn_file in glob("/sys/module/ibmvscsis/drivers/vio:ibmvscsis/*/devspec"): + name = fread(wwn_file) + yield name[name.find("@") + 1:] + + fabric_modules = { "srpt": SRPTFabricModule, "iscsi": ISCSIFabricModule, @@ -457,6 +470,7 @@ fabric_modules = { # "usb_gadget": USBGadgetFabricModule, # very rare, don't show "vhost": VhostFabricModule, "xen_pvscsi": XenPvScsiFabricModule, + "ibmvscsis": IbmvscsisFabricModule, } # -- GitLab From 39aba4c939fdaa93b59045df6169e2fc7c181719 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Tue, 3 Jan 2017 13:07:51 -0600 Subject: [PATCH 36/70] Add basic ALUA support This patch adds support for creating ALUA groups for storage objects and setting up their state. The next patch will add support to map these groups to LUNs. This patchset does not add support for LU groups and secondary groups. Signed-off-by: Mike Christie --- rtslib/__init__.py | 2 + rtslib/alua.py | 385 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 rtslib/alua.py diff --git a/rtslib/__init__.py b/rtslib/__init__.py index c4b3cb6..48aab2a 100644 --- a/rtslib/__init__.py +++ b/rtslib/__init__.py @@ -33,6 +33,8 @@ from .tcm import FileIOStorageObject, BlockStorageObject from .tcm import PSCSIStorageObject, RDMCPStorageObject, UserBackedStorageObject from .tcm import StorageObjectFactory +from .alua import ALUATargetPortGroup + __version__ = 'GIT_VERSION' __author__ = "Jerome Martin " __url__ = "http://www.risingtidesystems.com" diff --git a/rtslib/alua.py b/rtslib/alua.py new file mode 100644 index 0000000..86a4dd3 --- /dev/null +++ b/rtslib/alua.py @@ -0,0 +1,385 @@ +''' +Implements the RTS ALUA Target Port Group class. + +This file is part of RTSLib. +Copyright (c) 2016 by Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. +''' + +from .node import CFSNode +from .utils import RTSLibError, fread, fwrite + +alua_rw_params = ['alua_access_state', 'alua_access_status', + 'alua_write_metadata', 'alua_access_type', 'preferred', + 'nonop_delay_msecs', 'trans_delay_msecs', + 'implicit_trans_secs', 'alua_support_offline', + 'alua_support_standby', 'alua_support_transitioning', + 'alua_support_active_nonoptimized', + 'alua_support_unavailable', 'alua_support_active_optimized'] +alua_ro_params = ['tg_pt_gp_id', 'members', 'alua_support_lba_dependent'] +alua_types = ['None', 'Implicit', 'Explicit', 'Implicit and Explicit'] +alua_statuses = ['None', 'Altered by Explicit STPG', 'Altered by Implicit ALUA'] + +class ALUATargetPortGroup(CFSNode): + """ + ALUA Target Port Group interface + """ + + def __repr__(self): + return "" % self.name + + def __init__(self, storage_object, name, tag=None): + """ + @param storage_object: backstore storage object to create ALUA group for + @param name: name of ALUA group + @param tag: target port group id. If not passed in, try to look + up existing ALUA TPG with the same name + """ + + # default_tg_pt_gp takes tag 1 + if tag is not None and (tag > 65535 or tag < 1): + raise RTSLibError("The TPG Tag must be between 1 and 65535") + + super(ALUATargetPortGroup, self).__init__() + self.name = name + self.storage_object = storage_object + + self._path = "%s/alua/%s" % (storage_object.path, name) + + if tag is not None: + try: + self._create_in_cfs_ine('create') + except OSError as msg: + raise RTSLibError(msg) + + try: + fwrite("%s/tg_pt_gp_id" % self._path, tag) + except IOError as msg: + self.delete() + raise RTSLibError("Cannot set id to %d: %s" % (tag, str(msg))) + else: + try: + self._create_in_cfs_ine('lookup') + except OSError as msg: + raise RTSLibError(msg) + + # Public + + def delete(self): + """ + Delete ALUA TPG and unmap from LUNs + """ + self._check_self() + + # default_tg_pt_gp created by the kernel and cannot be deleted + if self.name == "default_tg_pt_gp": + raise RTSLibError("Can not delete default_tg_pt_gp") + + # This will reset the ALUA tpg to default_tg_pt_gp + super(ALUATargetPortGroup, self).delete() + + def _get_alua_access_state(self): + self._check_self() + path = "%s/alua_access_state" % self.path + return int(fread(path)) + + def _set_alua_access_state(self, newstate): + self._check_self() + path = "%s/alua_access_state" % self.path + try: + fwrite(path, str(int(newstate))) + except IOError as e: + raise RTSLibError("Cannot change ALUA state: %s" % e) + + def _get_alua_access_status(self): + self._check_self() + path = "%s/alua_access_status" % self.path + status = fread(path) + return alua_statuses.index(status) + + def _set_alua_access_status(self, newstatus): + self._check_self() + path = "%s/alua_access_status" % self.path + try: + fwrite(path, str(int(newstatus))) + except IOError as e: + raise RTSLibError("Cannot change ALUA status: %s" % e) + + def _get_alua_access_type(self): + self._check_self() + path = "%s/alua_access_type" % self.path + alua_type = fread(path) + return alua_types.index(alua_type) + + def _set_alua_access_type(self, access_type): + self._check_self() + path = "%s/alua_access_type" % self.path + try: + fwrite(path, str(int(access_type))) + except IOError as e: + raise RTSLibError("Cannot change ALUA access type: %s" % e) + + def _get_preferred(self): + self._check_self() + path = "%s/preferred" % self.path + return int(fread(path)) + + def _set_preferred(self, pref): + self._check_self() + path = "%s/preferred" % self.path + try: + fwrite(path, str(int(pref))) + except IOError as e: + raise RTSLibError("Cannot set preferred: %s" % e) + + def _get_alua_write_metadata(self): + self._check_self() + path = "%s/alua_write_metadata" % self.path + return int(fread(path)) + + def _set_alua_write_metadata(self, pref): + self._check_self() + path = "%s/alua_write_metadata" % self.path + try: + fwrite(path, str(int(pref))) + except IOError as e: + raise RTSLibError("Cannot set alua_write_metadata: %s" % e) + + def _get_alua_support_active_nonoptimized(self): + self._check_self() + path = "%s/alua_support_active_nonoptimized" % self.path + return int(fread(path)) + + def _set_alua_support_active_nonoptimized(self, enabled): + self._check_self() + path = "%s/alua_support_active_nonoptimized" % self.path + try: + fwrite(path, str(int(enabled))) + except IOError as e: + raise RTSLibError("Cannot set alua_support_active_nonoptimized: %s" % e) + + def _get_alua_support_active_optimized(self): + self._check_self() + path = "%s/alua_support_active_optimized" % self.path + return int(fread(path)) + + def _set_alua_support_active_optimized(self, enabled): + self._check_self() + path = "%s/alua_support_active_optimized" % self.path + try: + fwrite(path, str(int(enabled))) + except IOError as e: + raise RTSLibError("Cannot set alua_support_active_optimized: %s" % e) + + def _get_alua_support_offline(self): + self._check_self() + path = "%s/alua_support_offline" % self.path + return int(fread(path)) + + def _set_alua_support_offline(self, enabled): + self._check_self() + path = "%s/alua_support_offline" % self.path + try: + fwrite(path, str(int(enabled))) + except IOError as e: + raise RTSLibError("Cannot set alua_support_offline: %s" % e) + + def _get_alua_support_unavailable(self): + self._check_self() + path = "%s/alua_support_unavailable" % self.path + return int(fread(path)) + + def _set_alua_support_unavailable(self, enabled): + self._check_self() + path = "%s/alua_support_unavailable" % self.path + try: + fwrite(path, str(int(enabled))) + except IOError as e: + raise RTSLibError("Cannot set alua_support_unavailable: %s" % e) + + def _get_alua_support_standby(self): + self._check_self() + path = "%s/alua_support_standby" % self.path + return int(fread(path)) + + def _set_alua_support_standby(self, enabled): + self._check_self() + path = "%s/alua_support_standby" % self.path + try: + fwrite(path, str(int(enabled))) + except IOError as e: + raise RTSLibError("Cannot set alua_support_standby: %s" % e) + + def _get_alua_support_transitioning(self): + self._check_self() + path = "%s/alua_support_transitioning" % self.path + return int(fread(path)) + + def _set_alua_support_transitioning(self, enabled): + self._check_self() + path = "%s/alua_support_transitioning" % self.path + try: + fwrite(path, str(int(enabled))) + except IOError as e: + raise RTSLibError("Cannot set alua_support_transitioning: %s" % e) + + def _get_alua_support_lba_dependent(self): + self._check_self() + path = "%s/alua_support_lba_dependent" % self.path + return int(fread(path)) + + def _get_members(self): + self._check_self() + path = "%s/members" % self.path + return fread(path) + + def _get_tg_pt_gp_id(self): + self._check_self() + path = "%s/tg_pt_gp_id" % self.path + return int(fread(path)) + + def _get_trans_delay_msecs(self): + self._check_self() + path = "%s/trans_delay_msecs" % self.path + return int(fread(path)) + + def _set_trans_delay_msecs(self, secs): + self._check_self() + path = "%s/trans_delay_msecs" % self.path + try: + fwrite(path, str(int(secs))) + except IOError as e: + raise RTSLibError("Cannot set trans_delay_msecs: %s" % e) + + def _get_implicit_trans_secs(self): + self._check_self() + path = "%s/implicit_trans_secs" % self.path + return int(fread(path)) + + def _set_implicit_trans_secs(self, secs): + self._check_self() + path = "%s/implicit_trans_secs" % self.path + try: + fwrite(path, str(int(secs))) + except IOError as e: + raise RTSLibError("Cannot set implicit_trans_secs: %s" % e) + + def _get_nonop_delay_msecs(self): + self._check_self() + path = "%s/nonop_delay_msecs" % self.path + return int(fread(path)) + + def _set_nonop_delay_msecs(self, delay): + self._check_self() + path = "%s/nonop_delay_msecs" % self.path + try: + fwrite(path, str(int(delay))) + except IOError as e: + raise RTSLibError("Cannot set nonop_delay_msecs: %s" % e) + + def dump(self): + d = super(ALUATargetPortGroup, self).dump() + d['name'] = self.name + d['tg_pt_gp_id'] = self.tg_pt_gp_id + for param in alua_rw_params: + d[param] = getattr(self, param, None) + return d + + alua_access_state = property(_get_alua_access_state, _set_alua_access_state, + doc="Get or set ALUA state. " + "0 = Active/optimized, " + "1 = Active/non-optimized, " + "2 = Standby, " + "3 = Unavailable, " + "4 = LBA Dependent, " + "14 = Offline, " + "15 = Transitioning") + + alua_access_type = property(_get_alua_access_type, _set_alua_access_type, + doc="Get or set ALUA access type. " + "1 = Implicit, 2 = Explicit, 3 = Both") + + alua_access_status = property(_get_alua_access_status, + _set_alua_access_status, + doc="Get or set ALUA access status. " + "0 = None, " + "1 = Altered by Explicit STPG, " + "2 = Altered by Implicit ALUA") + + preferred = property(_get_preferred, _set_preferred, + doc="Get or set preferred bit. 1 = Pref, 0 Not-Pre") + + alua_write_metadata = property(_get_alua_write_metadata, + _set_alua_write_metadata, + doc="Get or set alua_write_metadata flag. " + "enable (1) or disable (0)") + + tg_pt_gp_id = property(_get_tg_pt_gp_id, doc="Get ALUA Target Port Group ID") + + members = property(_get_members, doc="Get LUNs in Target Port Group") + + alua_support_active_nonoptimized = property(_get_alua_support_active_nonoptimized, + _set_alua_support_active_nonoptimized, + doc="Enable (1) or disable (0) " + "Active/non-optimized support") + + alua_support_active_optimized = property(_get_alua_support_active_optimized, + _set_alua_support_active_optimized, + doc="Enable (1) or disable (0) " + "Active/optimized support") + + alua_support_offline = property(_get_alua_support_offline, + _set_alua_support_offline, + doc="Enable (1) or disable (0) " + "offline support") + + alua_support_unavailable = property(_get_alua_support_unavailable, + _set_alua_support_unavailable, + doc="enable (1) or disable (0) " + "unavailable support") + + alua_support_standby = property(_get_alua_support_standby, + _set_alua_support_standby, + doc="enable (1) or disable (0) " + "standby support") + + alua_support_lba_dependent = property(_get_alua_support_lba_dependent, + doc="show lba_dependent support " + "enabled (1) or disabled (0)") + + alua_support_transitioning = property(_get_alua_support_transitioning, + _set_alua_support_transitioning, + doc="enable (1) or disable (0) " + "transitioning support") + + trans_delay_msecs = property(_get_trans_delay_msecs, + _set_trans_delay_msecs, + doc="msecs to delay state transition") + + implicit_trans_secs = property(_get_implicit_trans_secs, + _set_implicit_trans_secs, + doc="implicit transition time limit") + + nonop_delay_msecs = property(_get_nonop_delay_msecs, _set_nonop_delay_msecs, + doc="msecs to delay IO when non-optimized") + + @classmethod + def setup(cls, storage_obj, alua_tpg, err_func): + name = alua_tpg['name'] + if name == 'default_tg_pt_gp': + return + + alua_tpg_obj = cls(storage_obj, name, alua_tpg['tg_pt_gp_id']) + for param in alua_rw_params: + setattr(alua_tpg_obj, param, alua_tpg[param]) -- GitLab From 1f3abc3caddc8663abd529ace41e84088f4761c5 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Tue, 3 Jan 2017 13:17:09 -0600 Subject: [PATCH 37/70] Get/Set LUN's ALUA group Add support to Get/Set a Lun's current ALUA group through its alua_tg_pt_gp file. Signed-off-by: Mike Christie --- rtslib/target.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/rtslib/target.py b/rtslib/target.py index c2836db..d8dc655 100644 --- a/rtslib/target.py +++ b/rtslib/target.py @@ -582,6 +582,22 @@ class LUN(CFSNode): if os.path.realpath("%s/%s" % (mlun.path, mlun.alias)) == self.path: yield mlun + def _get_alua_tg_pt_gp_name(self): + self._check_self() + + path = "%s/alua_tg_pt_gp" % self.path + group_name = fread(path).splitlines()[0] + return group_name.split(':')[1].strip() + + def _set_alua_tg_pt_gp_name(self, group_name): + self._check_self() + + path = "%s/alua_tg_pt_gp" % self.path + try: + fwrite(path, group_name) + except IOError as e: + raise RTSLibError("Cannot set ALUA Target Port Group: %s" % e) + # LUN public stuff def delete(self): @@ -615,6 +631,8 @@ class LUN(CFSNode): doc="Get the LUN alias.") mapped_luns = property(_list_mapped_luns, doc="List all MappedLUN objects referencing this LUN.") + alua_tg_pt_gp_name = property(_get_alua_tg_pt_gp_name, _set_alua_tg_pt_gp_name, + doc="Get and Set the LUN's ALUA Target Port Group") @classmethod def setup(cls, tpg_obj, lun, err_func): -- GitLab From b85519f44bd7e940243d89fccf8b59992ef8361a Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Tue, 3 Jan 2017 13:20:21 -0600 Subject: [PATCH 38/70] Add ALUA restore support Add support to save ALUA info and restore it. Signed-off-by: Mike Christie --- rtslib/root.py | 14 +++++++++++++- rtslib/target.py | 9 ++++++++- rtslib/tcm.py | 12 ++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/rtslib/root.py b/rtslib/root.py index b65825f..3cc6fbf 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -28,6 +28,7 @@ from .fabric import FabricModule from .tcm import so_mapping, StorageObject from .utils import RTSLibError, modprobe, mount_configfs from .utils import dict_remove, set_attributes +from .alua import ALUATargetPortGroup default_save_file = "/etc/target/saveconfig.json" @@ -85,6 +86,12 @@ class RTSRoot(CFSNode): for so in StorageObject.all(): yield so + def _list_alua_tpgs(self): + self._check_self() + for so in self.storage_objects: + for a in so.alua_tpgs: + yield a + def _list_tpgs(self): self._check_self() for t in self.targets: @@ -203,7 +210,7 @@ class RTSRoot(CFSNode): err_func("'plugin' not defined or invalid in storageobject %s" % so['name']) continue kwargs = so.copy() - dict_remove(kwargs, ('exists', 'attributes', 'plugin', 'buffered_mode')) + dict_remove(kwargs, ('exists', 'attributes', 'plugin', 'buffered_mode', 'alua_tpgs')) try: so_obj = so_cls(**kwargs) except Exception as e: @@ -216,6 +223,9 @@ class RTSRoot(CFSNode): set_attributes(so_obj, so.get('attributes', {}), so_err_func) + for alua_tpg in so.get('alua_tpgs', {}): + ALUATargetPortGroup.setup(so_obj, alua_tpg, err_func) + # Don't need to create fabric modules for index, fm in enumerate(config.get('fabric_modules', [])): if 'name' not in fm: @@ -299,6 +309,8 @@ class RTSRoot(CFSNode): doc="Get the list of all existing LUN objects.") fabric_modules = property(_list_fabric_modules, doc="Get the list of all FabricModule objects.") + alua_tpgs = property(_list_alua_tpgs, + doc="Get the list of all ALUA TPG objects.") def _test(): '''Run the doctests.''' diff --git a/rtslib/target.py b/rtslib/target.py index d8dc655..4c06cf4 100644 --- a/rtslib/target.py +++ b/rtslib/target.py @@ -655,17 +655,24 @@ class LUN(CFSNode): return try: - cls(tpg_obj, lun['index'], storage_object=match_so, alias=lun.get('alias')) + lun_obj = cls(tpg_obj, lun['index'], storage_object=match_so, alias=lun.get('alias')) except (RTSLibError, KeyError): err_func("Creating TPG %d LUN index %d failed" % (tpg_obj.tag, lun['index'])) + try: + lun_obj.alua_tg_pt_gp_name = lun['alua_tg_pt_gp_name'] + except KeyError: + # alua_tg_pt_gp support not present in older versions + pass + def dump(self): d = super(LUN, self).dump() d['storage_object'] = "/backstores/%s/%s" % \ (self.storage_object.plugin, self.storage_object.name) d['index'] = self.lun d['alias'] = self.alias + d['alua_tg_pt_gp_name'] = self.alua_tg_pt_gp_name return d diff --git a/rtslib/tcm.py b/rtslib/tcm.py index 2ea5d0f..64dd48c 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -25,6 +25,7 @@ import glob import resource from six.moves import range +from .alua import ALUATargetPortGroup from .node import CFSNode from .utils import fread, fwrite, generate_wwn, RTSLibError, RTSLibNotInCFS from .utils import convert_scsi_path_to_hctl, convert_scsi_hctl_to_path @@ -215,6 +216,14 @@ class StorageObject(CFSNode): for lun in self._gen_attached_luns(): yield lun + def _list_alua_tpgs(self): + ''' + Generate all ALUA groups attach to a storage object. + ''' + self._check_self() + for tpg in os.listdir("%s/alua" % self.path): + yield ALUATargetPortGroup(self, tpg) + # StorageObject public stuff def delete(self): @@ -264,11 +273,14 @@ class StorageObject(CFSNode): + "is used by any LUN") attached_luns = property(_list_attached_luns, doc="Get the list of all LUN objects attached.") + alua_tpgs = property(_list_alua_tpgs, + doc="Get list of ALUA Target Port Groups attached.") def dump(self): d = super(StorageObject, self).dump() d['name'] = self.name d['plugin'] = self.plugin + d['alua_tpgs'] = [tpg.dump() for tpg in self.alua_tpgs] return d -- GitLab From 163ee011246971ea3f6a00acd98df5a2df9a8166 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Tue, 3 Jan 2017 13:22:32 -0600 Subject: [PATCH 39/70] Delete ALUA groups with its storage object We must delete the ALUA groups under the storage object when deleting the storage object. Signed-off-by: Mike Christie --- rtslib/tcm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rtslib/tcm.py b/rtslib/tcm.py index 64dd48c..5452f92 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -235,6 +235,10 @@ class StorageObject(CFSNode): ''' self._check_self() + for alua_tpg in self._list_alua_tpgs(): + if alua_tpg.name != 'default_tg_pt_gp': + alua_tpg.delete() + # If we are called after a configure error, we can skip this if self.is_configured(): for lun in self._gen_attached_luns(): -- GitLab From 19efee2bc7e1239ce8a8bd7af599c4903439b2de Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 4 Jan 2017 10:44:31 -0800 Subject: [PATCH 40/70] version 2.1.fb62 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7770032..2e0726c 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.61', + version = '2.1.62', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From 59f3fc46596ed0adfa1e9ed9f56dd71c5efdd47e Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Wed, 11 Jan 2017 16:32:18 -0600 Subject: [PATCH 41/70] Do not set alua_tg_pt_gp if not supported Pass through backends do not support alua_tg_pt_gp and writing to the file will hang the system. Trying to read from them will cause a rtslib crash because they are empty. This patch fixes both issues by detecting the empty file and failing writes. --- rtslib/target.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/rtslib/target.py b/rtslib/target.py index 4c06cf4..816be74 100644 --- a/rtslib/target.py +++ b/rtslib/target.py @@ -586,13 +586,23 @@ class LUN(CFSNode): self._check_self() path = "%s/alua_tg_pt_gp" % self.path - group_name = fread(path).splitlines()[0] - return group_name.split(':')[1].strip() + info = fread(path) + if info: + group_line = info.splitlines()[0] + return group_line.split(':')[1].strip() + return None def _set_alua_tg_pt_gp_name(self, group_name): self._check_self() path = "%s/alua_tg_pt_gp" % self.path + + info = fread(path) + if not info: + # pass through backends will not have setup the default + # ALUA structs in the kernel. + raise RTSLibError("This LUN does not support setting the ALUA Target Port Group") + try: fwrite(path, group_name) except IOError as e: -- GitLab From 6453b0d484f4a5b63439a69de4620b498b5c108a Mon Sep 17 00:00:00 2001 From: Amartey Pearson Date: Sun, 22 Jan 2017 11:48:44 -0600 Subject: [PATCH 42/70] Add ability to invalidate caches For efficiency, rtslib may have caches (today for the data in configfs). A client may need to clear out these caches such as via the refresh option in targetcli. * Add new invalidate_caches method to RTSRoot. While this only has a single cache today, it gives the ability to extend to any future caches. * While bs_cache really should be private, it probably shouldn't be changed given that this is a library and may be used elsewhere. Signed-off-by: Amartey Pearson --- rtslib/root.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rtslib/root.py b/rtslib/root.py index 3cc6fbf..44bb459 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -25,7 +25,7 @@ import json from .node import CFSNode from .target import Target from .fabric import FabricModule -from .tcm import so_mapping, StorageObject +from .tcm import so_mapping, bs_cache, StorageObject from .utils import RTSLibError, modprobe, mount_configfs from .utils import dict_remove, set_attributes from .alua import ALUATargetPortGroup @@ -287,6 +287,12 @@ class RTSRoot(CFSNode): return self.restore(config, clear_existing=clear_existing, abort_on_error=abort_on_error) + def invalidate_caches(self): + ''' + Invalidate any caches used throughout the hierarchy + ''' + bs_cache.clear() + targets = property(_list_targets, doc="Get the list of Target objects.") tpgs = property(_list_tpgs, -- GitLab From 6bd910bd5530c9dfd0a768411beacef568b2f7d4 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Mon, 6 Feb 2017 17:48:19 -0600 Subject: [PATCH 43/70] Add ALUA supported method Passthrough backends like pscsi and user do not support ALUA managment through configfs. You can create groups, but writing to files can cause the kernel crashes. This adds a alua_supported method that backends can override and return False to indicate they do not support ALUA. Signed-off-by: Mike Christie --- rtslib/__init__.py | 1 + rtslib/alua.py | 8 +++++++- rtslib/root.py | 7 +++++-- rtslib/tcm.py | 24 +++++++++++++++++++++++- rtslib/utils.py | 6 ++++++ 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/rtslib/__init__.py b/rtslib/__init__.py index 48aab2a..568619e 100644 --- a/rtslib/__init__.py +++ b/rtslib/__init__.py @@ -23,6 +23,7 @@ if __name__ == "rtslib": from .root import RTSRoot from .utils import RTSLibError, RTSLibBrokenLink, RTSLibNotInCFS +from .utils import RTSLibALUANotSupported from .target import LUN, MappedLUN from .target import NodeACL, NetworkPortal, TPG, Target diff --git a/rtslib/alua.py b/rtslib/alua.py index 86a4dd3..390e74a 100644 --- a/rtslib/alua.py +++ b/rtslib/alua.py @@ -18,7 +18,7 @@ a copy of the License at ''' from .node import CFSNode -from .utils import RTSLibError, fread, fwrite +from .utils import RTSLibError, RTSLibALUANotSupported, fread, fwrite alua_rw_params = ['alua_access_state', 'alua_access_status', 'alua_write_metadata', 'alua_access_type', 'preferred', @@ -46,6 +46,12 @@ class ALUATargetPortGroup(CFSNode): @param tag: target port group id. If not passed in, try to look up existing ALUA TPG with the same name """ + # kernel partially sets up default_tg_pt_gp and will let you partially + # setup ALUA groups for pscsi and user, but writing to some of the + # files will crash the kernel. Just fail to even create groups until + # the kernel is fixed. + if storage_object.alua_supported is False: + raise RTSLibALUANotSupported("Backend does not support ALUA setup") # default_tg_pt_gp takes tag 1 if tag is not None and (tag > 65535 or tag < 1): diff --git a/rtslib/root.py b/rtslib/root.py index 3cc6fbf..6d6b5ee 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -26,7 +26,7 @@ from .node import CFSNode from .target import Target from .fabric import FabricModule from .tcm import so_mapping, StorageObject -from .utils import RTSLibError, modprobe, mount_configfs +from .utils import RTSLibError, RTSLibALUANotSupported, modprobe, mount_configfs from .utils import dict_remove, set_attributes from .alua import ALUATargetPortGroup @@ -224,7 +224,10 @@ class RTSRoot(CFSNode): set_attributes(so_obj, so.get('attributes', {}), so_err_func) for alua_tpg in so.get('alua_tpgs', {}): - ALUATargetPortGroup.setup(so_obj, alua_tpg, err_func) + try: + ALUATargetPortGroup.setup(so_obj, alua_tpg, err_func) + except RTSLibALUANotSupported: + pass # Don't need to create fabric modules for index, fm in enumerate(config.get('fabric_modules', [])): diff --git a/rtslib/tcm.py b/rtslib/tcm.py index 5452f92..aa3530a 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -222,7 +222,15 @@ class StorageObject(CFSNode): ''' self._check_self() for tpg in os.listdir("%s/alua" % self.path): - yield ALUATargetPortGroup(self, tpg) + if self.alua_supported: + yield ALUATargetPortGroup(self, tpg) + + def _get_alua_supported(self): + ''' + Children should override and return false if ALUA setup is not supported. + ''' + self._check_self() + return True # StorageObject public stuff @@ -279,6 +287,8 @@ class StorageObject(CFSNode): doc="Get the list of all LUN objects attached.") alua_tpgs = property(_list_alua_tpgs, doc="Get list of ALUA Target Port Groups attached.") + alua_supported = property(_get_alua_supported, + doc="Returns true if ALUA can be setup. False if not supported.") def dump(self): d = super(StorageObject, self).dump() @@ -408,6 +418,10 @@ class PSCSIStorageObject(StorageObject): self._check_self() return int(self._parse_info('Host ID')) + def _get_alua_supported(self): + self._check_self() + return False + # PSCSIStorageObject public stuff wwn = property(StorageObject._get_wwn, _set_wwn, @@ -427,6 +441,8 @@ class PSCSIStorageObject(StorageObject): doc="Get the SCSI device target id") lun = property(_get_lun, doc="Get the SCSI device LUN") + alua_supported = property(_get_alua_supported, + doc="ALUA cannot be setup with rtslib, so False is returned."); def dump(self): d = super(PSCSIStorageObject, self).dump() @@ -808,10 +824,16 @@ class UserBackedStorageObject(StorageObject): return None return val + def _get_alua_supported(self): + self._check_self() + return False + size = property(_get_size, doc="Get the size in bytes.") config = property(_get_config, doc="Get the TCMU config.") + alua_supported = property(_get_alua_supported, + doc="ALUA cannot be setup with rtslib, so False is returned."); def dump(self): d = super(UserBackedStorageObject, self).dump() diff --git a/rtslib/utils.py b/rtslib/utils.py index 969d2d7..1a5315e 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -37,6 +37,12 @@ class RTSLibError(Exception): ''' pass +class RTSLibALUANotSupported(RTSLibError): + ''' + Backend does not support ALUA. + ''' + pass + class RTSLibBrokenLink(RTSLibError): ''' Broken link in configfs, i.e. missing LUN storage object. -- GitLab From 13dfb1aadbba3d1e2fa64133e9e2392071d58a5a Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Mon, 6 Feb 2017 17:50:44 -0600 Subject: [PATCH 44/70] Don't raise exception when getting/setting a LUNs ALUA group Fix crash due to trying to get/set the ALUA group on passthrough backends. Instead of raising an exception return None/-1 on failure. Signed-off-by: Mike Christie --- rtslib/target.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/rtslib/target.py b/rtslib/target.py index 816be74..d3cdab7 100644 --- a/rtslib/target.py +++ b/rtslib/target.py @@ -582,31 +582,43 @@ class LUN(CFSNode): if os.path.realpath("%s/%s" % (mlun.path, mlun.alias)) == self.path: yield mlun + + # pass through backends will not have setup all the default + # ALUA structs in the kernel. If the kernel has been setup, + # a user created group or default_tg_pt_gp will be returned. + # If the kernel was not properly setup an empty string is + # return in alua_tg_pt_gp. Writing to alua_tg_pt_gp will crash + # older kernels and will return a -Exyz code in newer ones. def _get_alua_tg_pt_gp_name(self): self._check_self() + storage_object = self._get_storage_object() + if storage_object.alua_supported is False: + return None + path = "%s/alua_tg_pt_gp" % self.path - info = fread(path) - if info: + try: + info = fread(path) + if not info: + return None group_line = info.splitlines()[0] return group_line.split(':')[1].strip() - return None + except IOError as e: + return None def _set_alua_tg_pt_gp_name(self, group_name): self._check_self() - path = "%s/alua_tg_pt_gp" % self.path - - info = fread(path) - if not info: - # pass through backends will not have setup the default - # ALUA structs in the kernel. - raise RTSLibError("This LUN does not support setting the ALUA Target Port Group") + if not self._get_alua_tg_pt_gp_name(): + return -1 + path = "%s/alua_tg_pt_gp" % self.path try: fwrite(path, group_name) except IOError as e: - raise RTSLibError("Cannot set ALUA Target Port Group: %s" % e) + return -1 + + return 0 # LUN public stuff -- GitLab From 3637171a68889e340f553943f25d6a5ca20ad1ca Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 1 Mar 2017 13:30:29 -0800 Subject: [PATCH 45/70] update to 2.1.fb63 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2e0726c..11bbb88 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.62', + version = '2.1.63', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From b955af4dab26213b0939ababbcbe900516d3d9df Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Wed, 29 Mar 2017 15:20:28 -0500 Subject: [PATCH 46/70] Support tcmu hw max sectors Support hw_max_sectors setting for tcmu added in this patch: commit 3abaa2bfdb1e6bb33d38a2e82cf3bb82ec0197bf Author: Mike Christie Date: Wed Mar 1 23:14:39 2017 -0600 tcmu: allow hw_max_sectors greater than 128 Signed-off-by: Mike Christie --- rtslib/tcm.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/rtslib/tcm.py b/rtslib/tcm.py index aa3530a..82ef34e 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -772,7 +772,8 @@ class UserBackedStorageObject(StorageObject): An interface to configFS storage objects for userspace-backed backstore. ''' - def __init__(self, name, config=None, size=None, wwn=None): + def __init__(self, name, config=None, size=None, wwn=None, + hw_max_sectors=None): ''' @param name: The name of the UserBackedStorageObject. @type name: string @@ -783,6 +784,8 @@ class UserBackedStorageObject(StorageObject): @type size: int @param wwn: T10 WWN Unit Serial, will generate if None @type wwn: string + @hw_max_sectors: Max sectors per command limit to export to initiators. + @type hw_max_sectors: int @return: A UserBackedStorageObject object. ''' @@ -795,20 +798,22 @@ class UserBackedStorageObject(StorageObject): "from its configuration string") super(UserBackedStorageObject, self).__init__(name, 'create') try: - self._configure(config, size, wwn) + self._configure(config, size, wwn, hw_max_sectors) except: self.delete() raise else: super(UserBackedStorageObject, self).__init__(name, 'lookup') - def _configure(self, config, size, wwn): + def _configure(self, config, size, wwn, hw_max_sectors): self._check_self() if ':' in config: raise RTSLibError("':' not allowed in config string") self._control("dev_config=%s" % config) self._control("dev_size=%d" % size) + if hw_max_sectors is not None: + self._control("hw_max_sectors=%s" % hw_max_sectors) self._enable() super(UserBackedStorageObject, self)._configure(wwn) @@ -817,6 +822,10 @@ class UserBackedStorageObject(StorageObject): self._check_self() return int(self._parse_info('Size')) + def _get_hw_max_sectors(self): + self._check_self() + return int(self._parse_info('HwMaxSectors')) + def _get_config(self): self._check_self() val = self._parse_info('Config') @@ -828,6 +837,8 @@ class UserBackedStorageObject(StorageObject): self._check_self() return False + hw_max_sectors = property(_get_hw_max_sectors, + doc="Get the max sectors per command.") size = property(_get_size, doc="Get the size in bytes.") config = property(_get_config, @@ -840,6 +851,8 @@ class UserBackedStorageObject(StorageObject): d['wwn'] = self.wwn d['size'] = self.size d['config'] = self.config + d['hw_max_sectors'] = self.hw_max_sectors + return d -- GitLab From 58741c11b0ba9c11690808e67e7f6133141cbb21 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Tue, 25 Apr 2017 01:58:51 -0500 Subject: [PATCH 47/70] ALUA: return list of members v2 There is a bug in the kernel where if you map a backend device through multiple targets/tpgts it only returns the first one. The rtslib ALUA members function then returns a string containing whatever the kernel prints out which is just the first lun the backend is mapped to. The others are dropped. When the kernel is fixed this will print out a lun mapping per line with the format: driver/target/tpgt/lun for each line. This patch prepares us for the kernel getting fixed and returns the lun mappings in a format that is easier for the rtslib user. It will return a list of dictionarys with the format: [{'tpgt': 1, 'driver': 'iSCSI', 'target': 'iqn.1999-09.com.tcmu:alua', 'lun': 3}, {'tpgt': 2, 'driver': 'iSCSI', 'target': 'iqn.1999-09.com.tcmu:alua2', 'lun': 1}] -- v2 - remove tpgt/lun prefixes and return ints instead of strings. Signed-off-by: Mike Christie --- rtslib/alua.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rtslib/alua.py b/rtslib/alua.py index 390e74a..70f94b1 100644 --- a/rtslib/alua.py +++ b/rtslib/alua.py @@ -248,7 +248,17 @@ class ALUATargetPortGroup(CFSNode): def _get_members(self): self._check_self() path = "%s/members" % self.path - return fread(path) + + member_list = [] + + for member in fread(path).splitlines(): + lun_path = member.split("/") + if len(lun_path) != 4: + continue + member_list.append({ 'driver': lun_path[0], 'target': lun_path[1], + 'tpgt': int(lun_path[2].split("_", 1)[1]), + 'lun': int(lun_path[3].split("_", 1)[1]) }) + return member_list def _get_tg_pt_gp_id(self): self._check_self() -- GitLab From b49c711b3209764687db9a7620e57f91e45057a2 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 17 May 2017 09:37:59 -0700 Subject: [PATCH 48/70] Fix exception in convert_scsi_hctl_to_path Convert integers to string before using them with .join(). Signed-off-by: Andy Grover --- rtslib/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index 1a5315e..eeb1f6c 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -294,7 +294,7 @@ def convert_scsi_hctl_to_path(host, controller, target, lun): raise RTSLibError( "The host, controller, target and lun parameter must be integers") - hctl = [host, controller, target, lun] + hctl = [str(host), str(controller), str(target), str(lun)] try: scsi_device = pyudev.Device.from_name(_CONTEXT, 'scsi', ':'.join(hctl)) except pyudev.DeviceNotFoundError: -- GitLab From 2a816d67e95ac0edc6d56076595b29eddfeb433b Mon Sep 17 00:00:00 2001 From: "Bryant G. Ly" Date: Thu, 1 Jun 2017 13:11:37 -0500 Subject: [PATCH 49/70] Support Reconfiguration of device path This patch allows users to pass in a string into the attributes. Prior it would throw: Traceback (most recent call last): File "/usr/bin/targetcli", line 121, in main() File "/usr/bin/targetcli", line 117, in main root_node.ui_command_saveconfig() File "/usr/lib/python3/dist-packages/targetcli/ui_root.py", line 98, in ui_command_saveconfig self.rtsroot.save_to_file(savefile) File "/usr/lib/python3/dist-packages/rtslib_fb/root.py", line 270, in save_to_file f.write(json.dumps(self.dump(), sort_keys=True, indent=2)) File "/usr/lib/python3/dist-packages/rtslib_fb/root.py", line 160, in dump d['storage_objects'] = [so.dump() for so in self.storage_objects] File "/usr/lib/python3/dist-packages/rtslib_fb/root.py", line 160, in d['storage_objects'] = [so.dump() for so in self.storage_objects] File "/usr/lib/python3/dist-packages/rtslib_fb/tcm.py", line 850, in dump d = super(UserBackedStorageObject, self).dump() File "/usr/lib/python3/dist-packages/rtslib_fb/tcm.py", line 294, in dump d = super(StorageObject, self).dump() File "/usr/lib/python3/dist-packages/rtslib_fb/node.py", line 217, in dump attrs[item] = int(self.get_attribute(item)) ValueError: invalid literal for int() with base 10: 'file//home/neo/test.raw' This error is due to set attribute dev_path being a non integer. Signed-off-by: Bryant G. Ly --- rtslib/node.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rtslib/node.py b/rtslib/node.py index c319a5b..c0092fc 100644 --- a/rtslib/node.py +++ b/rtslib/node.py @@ -214,7 +214,10 @@ class CFSNode(object): attrs = {} params = {} for item in self.list_attributes(writable=True): - attrs[item] = int(self.get_attribute(item)) + try: + attrs[item] = int(self.get_attribute(item)) + except ValueError: + attrs[item] = self.get_attribute(item) if attrs: d['attributes'] = attrs for item in self.list_parameters(writable=True): -- GitLab From f823033e5c6beced0590a00064e1e7a55e76a995 Mon Sep 17 00:00:00 2001 From: Lee Duncan Date: Tue, 13 Jun 2017 09:45:30 -0700 Subject: [PATCH 50/70] Switch target driver DB root dir to /etc/target This switches the kernel target driver directory from the default of /var/target to /etc/target, if the "dbroot" sysfs attribute is present. If not, the default of /var/target is maintained. This has to be done by rtslib/root.py since this module loads the target_core_mod module, where the dbroot value must be updated before any target_core_* drivers register with target_core_mod. --- rtslib/root.py | 24 ++++++++++++++++++++++++ rtslib/tcm.py | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/rtslib/root.py b/rtslib/root.py index 99a25b7..ee7b103 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -28,6 +28,7 @@ from .fabric import FabricModule from .tcm import so_mapping, bs_cache, StorageObject from .utils import RTSLibError, RTSLibALUANotSupported, modprobe, mount_configfs from .utils import dict_remove, set_attributes +from .utils import fread, fwrite from .alua import ALUATargetPortGroup default_save_file = "/etc/target/saveconfig.json" @@ -57,6 +58,12 @@ class RTSRoot(CFSNode): ''' # RTSRoot private stuff + + # this should match the kernel target driver default db dir + _default_dbroot = "/var/target" + # this is where the target DB is to be located (instead of the default) + _preferred_dbroot = "/etc/target" + def __init__(self): ''' Instantiate an RTSRoot object. Basically checks for configfs setup and @@ -75,6 +82,8 @@ class RTSRoot(CFSNode): modprobe('target_core_mod') self._create_in_cfs_ine('any') + self._set_dbroot_if_needed() + def _list_targets(self): self._check_self() for fabric_module in self.fabric_modules: @@ -148,6 +157,19 @@ class RTSRoot(CFSNode): def __str__(self): return "rtslib" + def _set_dbroot_if_needed(self): + dbroot_path = self.path + "/dbroot" + if not os.path.exists(dbroot_path): + self._dbroot = self._default_dbroot + return + self._dbroot = fread(dbroot_path) + if self._dbroot != self._preferred_dbroot: + fwrite(dbroot_path, self._preferred_dbroot+"\n") + self._dbroot = fread(dbroot_path) + + def _get_dbroot(self): + return self._dbroot + # RTSRoot public stuff def dump(self): @@ -320,6 +342,8 @@ class RTSRoot(CFSNode): doc="Get the list of all FabricModule objects.") alua_tpgs = property(_list_alua_tpgs, doc="Get the list of all ALUA TPG objects.") + dbroot = property(_get_dbroot, + doc="Get the target database root") def _test(): '''Run the doctests.''' diff --git a/rtslib/tcm.py b/rtslib/tcm.py index 82ef34e..73bfbe3 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -78,7 +78,8 @@ class StorageObject(CFSNode): need to read it in and squirt it back into configfs when we configure the storage object. BLEH. """ - aptpl_dir = "/var/target/pr" + from .root import RTSRoot + aptpl_dir = "%s/pr" % RTSRoot().dbroot try: lines = fread("%s/aptpl_%s" % (aptpl_dir, self.wwn)).split() -- GitLab From 3ff084a7bb44d26b26f6f7e4067b5809741503b1 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Tue, 11 Jul 2017 12:42:49 -0500 Subject: [PATCH 51/70] alua: enable alua for pscsi/tcmu if kernel reports support 4.14 (it is currently the 4.13 feature window and I think we will miss it) will add this patch https://git.kernel.org/pub/scm/linux/kernel/git/nab/target-pending.git/commit/?h=for-next&id=c17d5d5f51f72f24e0e17a4450ae5010bf6962d9 commit c17d5d5f51f72f24e0e17a4450ae5010bf6962d9 Author: Mike Christie Date: Mon Jul 10 14:53:31 2017 -0500 target: export lio pgr/alua support as device attr which has the lio device report if it supports lio based ALUA. We can then check this configfs file and know if we are using a modern kernel with support and not worry about possibly crashing the system. --- rtslib/alua.py | 4 ---- rtslib/tcm.py | 23 ++++++++++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/rtslib/alua.py b/rtslib/alua.py index 70f94b1..8a4b30d 100644 --- a/rtslib/alua.py +++ b/rtslib/alua.py @@ -46,10 +46,6 @@ class ALUATargetPortGroup(CFSNode): @param tag: target port group id. If not passed in, try to look up existing ALUA TPG with the same name """ - # kernel partially sets up default_tg_pt_gp and will let you partially - # setup ALUA groups for pscsi and user, but writing to some of the - # files will crash the kernel. Just fail to even create groups until - # the kernel is fixed. if storage_object.alua_supported is False: raise RTSLibALUANotSupported("Backend does not support ALUA setup") diff --git a/rtslib/tcm.py b/rtslib/tcm.py index 73bfbe3..1aeea23 100644 --- a/rtslib/tcm.py +++ b/rtslib/tcm.py @@ -32,6 +32,19 @@ from .utils import convert_scsi_path_to_hctl, convert_scsi_hctl_to_path from .utils import is_dev_in_use, get_blockdev_type from .utils import get_size_for_blk_dev, get_size_for_disk_name +def storage_object_get_alua_support_attr(so): + ''' + Helper function that can be called by passthrough type of backends. + ''' + try: + if int(so.get_attribute("alua_support")) == 1: + return True + except RTSLibError: + pass + # Default to false because older kernels will crash when + # reading/writing to some ALUA files when ALUA was not + # fully supported by pscsi and tcmu. + return False class StorageObject(CFSNode): ''' @@ -228,7 +241,7 @@ class StorageObject(CFSNode): def _get_alua_supported(self): ''' - Children should override and return false if ALUA setup is not supported. + Children should override if the backend did not always support ALUA ''' self._check_self() return True @@ -421,7 +434,7 @@ class PSCSIStorageObject(StorageObject): def _get_alua_supported(self): self._check_self() - return False + return storage_object_get_alua_support_attr(self) # PSCSIStorageObject public stuff @@ -443,7 +456,7 @@ class PSCSIStorageObject(StorageObject): lun = property(_get_lun, doc="Get the SCSI device LUN") alua_supported = property(_get_alua_supported, - doc="ALUA cannot be setup with rtslib, so False is returned."); + doc="Returns true if ALUA can be setup. False if not supported.") def dump(self): d = super(PSCSIStorageObject, self).dump() @@ -836,7 +849,7 @@ class UserBackedStorageObject(StorageObject): def _get_alua_supported(self): self._check_self() - return False + return storage_object_get_alua_support_attr(self) hw_max_sectors = property(_get_hw_max_sectors, doc="Get the max sectors per command.") @@ -845,7 +858,7 @@ class UserBackedStorageObject(StorageObject): config = property(_get_config, doc="Get the TCMU config.") alua_supported = property(_get_alua_supported, - doc="ALUA cannot be setup with rtslib, so False is returned."); + doc="Returns true if ALUA can be setup. False if not supported.") def dump(self): d = super(UserBackedStorageObject, self).dump() -- GitLab From 6bb83d28750d8f3c34cdd2b35fd1267a289a35e4 Mon Sep 17 00:00:00 2001 From: Eric Harney Date: Tue, 18 Jul 2017 12:10:11 -0400 Subject: [PATCH 52/70] Fix python 3.6 escape char warnings in strings In python 3.6, escape sequences that are not recognized in string literals issue DeprecationWarnings. Convert these to raw strings. --- rtslib/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index eeb1f6c..ff1f7a5 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -378,12 +378,12 @@ def normalize_wwn(wwn_types, wwn): wwn_test = { 'free': lambda wwn: True, 'iqn': lambda wwn: \ - re.match("iqn\.[0-9]{4}-[0-1][0-9]\..*\..*", wwn) \ + re.match(r"iqn\.[0-9]{4}-[0-1][0-9]\..*\..*", wwn) \ and not re.search(' ', wwn) \ and not re.search('_', wwn), - 'naa': lambda wwn: re.match("naa\.[125][0-9a-fA-F]{15}$", wwn), - 'eui': lambda wwn: re.match("eui\.[0-9a-f]{16}$", wwn), - 'ib': lambda wwn: re.match("ib\.[0-9a-f]{32}$", wwn), + 'naa': lambda wwn: re.match(r"naa\.[125][0-9a-fA-F]{15}$", wwn), + 'eui': lambda wwn: re.match(r"eui\.[0-9a-f]{16}$", wwn), + 'ib': lambda wwn: re.match(r"ib\.[0-9a-f]{32}$", wwn), 'unit_serial': lambda wwn: \ re.match("[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}$", wwn), } -- GitLab From 737496e24ede21effc807a553cebe88ed5660a2b Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Tue, 15 Aug 2017 14:28:33 -0700 Subject: [PATCH 53/70] version 2.1.fb64 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 11bbb88..eb1114c 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.63', + version = '2.1.64', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From fb653b8c278cf22fa08925a6a6573ea791d9131b Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Sun, 20 Aug 2017 15:03:44 +0200 Subject: [PATCH 54/70] Add missing dependency on six in setup.py Signed-off-by: Christophe Vu-Brugier --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb1114c..08226c3 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,10 @@ setup ( url = 'http://github.com/open-iscsi/rtslib-fb', packages = ['rtslib_fb', 'rtslib'], scripts = ['scripts/targetctl'], - install_requires = ['pyudev >= 0.16.1'], + install_requires = [ + 'pyudev >= 0.16.1', + 'six', + ], classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", -- GitLab From eecb633e0632e5d1783078c04e94335236a6ccc6 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Sun, 20 Aug 2017 17:38:47 +0200 Subject: [PATCH 55/70] Display a more meaningful error when targetcli cannot change "dbroot" The "dbroot" configfs parameter allows to set the directory where persistent information are stored by the kernel, such as persistent reservations or ALUA. Targetcli tries to change the value of the "dbroot" parameter from /var/target/ to /etc/target/. However, if the directory passed to "dbroot" does not exist, the kernel returns an "invalid argument" error: # targetcli [Errno 22] Invalid argument With this patch, a more meaningful message is displayed instead: # targetcli Cannot set dbroot to /etc/target. Please check if this directory exists. This patch fixes issue #103. Signed-off-by: Christophe Vu-Brugier --- rtslib/root.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rtslib/root.py b/rtslib/root.py index ee7b103..e1d5116 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -164,7 +164,11 @@ class RTSRoot(CFSNode): return self._dbroot = fread(dbroot_path) if self._dbroot != self._preferred_dbroot: - fwrite(dbroot_path, self._preferred_dbroot+"\n") + try: + fwrite(dbroot_path, self._preferred_dbroot+"\n") + except: + raise RTSLibError("Cannot set dbroot to {}. Please check if this directory exists." + .format(self._preferred_dbroot)) self._dbroot = fread(dbroot_path) def _get_dbroot(self): -- GitLab From 7b879d4c578ec50cf0a2596e2583846c7b397f52 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Tue, 22 Aug 2017 13:50:53 +0200 Subject: [PATCH 56/70] Raise an error about failing to change the dbroot value only if the directory does not exist This allows targetcli to start when the dbroot value cannot be changed because the target drivers are already registered (issue #105). Signed-off-by: Christophe Vu-Brugier --- rtslib/root.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rtslib/root.py b/rtslib/root.py index e1d5116..0c3c231 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -167,8 +167,9 @@ class RTSRoot(CFSNode): try: fwrite(dbroot_path, self._preferred_dbroot+"\n") except: - raise RTSLibError("Cannot set dbroot to {}. Please check if this directory exists." - .format(self._preferred_dbroot)) + if not os.path.isdir(self._preferred_dbroot): + raise RTSLibError("Cannot set dbroot to {}. Please check if this directory exists." + .format(self._preferred_dbroot)) self._dbroot = fread(dbroot_path) def _get_dbroot(self): -- GitLab From 51993f0d53dd1b75b2c13cabcec778190603e827 Mon Sep 17 00:00:00 2001 From: Ji-Hyeon Gim Date: Sun, 17 Sep 2017 18:23:51 +0900 Subject: [PATCH 57/70] More compatibility syntax for legacy distros Many distros(such as CentOS 6) still uses python 2.6. but in fabric.py, uses new set initialization syntax introduced by python 2.7 We can consider for legacy distros using python 2.6 as we using legacy but not deprecated syntax Signed-off-by: Ji-Hyeon Gim --- rtslib/fabric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rtslib/fabric.py b/rtslib/fabric.py index 2887783..4e9a1ee 100644 --- a/rtslib/fabric.py +++ b/rtslib/fabric.py @@ -118,8 +118,8 @@ from .utils import RTSLibError, modprobe, ignored from .target import Target from .utils import _get_auth_attr, _set_auth_attr -version_attributes = {"lio_version", "version"} -discovery_auth_attributes = {"discovery_auth"} +version_attributes = set(["lio_version", "version"]) +discovery_auth_attributes = set(["discovery_auth"]) target_names_excludes = version_attributes | discovery_auth_attributes -- GitLab From 12e626b649ef4a928aa95e60d8b8c58b20bd3372 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 21 Sep 2017 16:45:30 -0700 Subject: [PATCH 58/70] Remove hba-only directories in clear_existing() rtslib never creates hba directories without a storage object within it, but if under some circumstance these existed then clear_existing() wouldn't remove them, since StorageObject::all()'s glob ignores them. Add code to ensure these are also removed. Signed-off-by: Andy Grover --- rtslib/root.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rtslib/root.py b/rtslib/root.py index 0c3c231..72c0299 100644 --- a/rtslib/root.py +++ b/rtslib/root.py @@ -21,6 +21,7 @@ under the License. import os import stat import json +import glob from .node import CFSNode from .target import Target @@ -206,6 +207,11 @@ class RTSRoot(CFSNode): for so in self.storage_objects: so.delete() + # If somehow some hbas still exist (no storage object within?) clean + # them up too. + for hba_dir in glob.glob("%s/core/*_*" % self.configfs_dir): + os.rmdir(hba_dir) + def restore(self, config, clear_existing=False, abort_on_error=False): ''' Takes a dict generated by dump() and reconfigures the target to match. -- GitLab From 5a5670608b7abab061a59d65568e8ab0a53b6ef0 Mon Sep 17 00:00:00 2001 From: Olaf Hering Date: Wed, 17 Jan 2018 12:27:48 +0100 Subject: [PATCH 59/70] Correct name for Xen pvscsi The name used in released products is xen-pvscsi, not xen_pvscsi. Fixes: 2bcc4cd ("Add support for xen-scsiback") Signed-off-by: Olaf Hering --- rtslib/fabric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rtslib/fabric.py b/rtslib/fabric.py index 4e9a1ee..b529f14 100644 --- a/rtslib/fabric.py +++ b/rtslib/fabric.py @@ -440,7 +440,7 @@ class VhostFabricModule(_BaseFabricModule): class XenPvScsiFabricModule(_BaseFabricModule): def __init__(self): - super(XenPvScsiFabricModule, self).__init__('xen_pvscsi') + super(XenPvScsiFabricModule, self).__init__('xen-pvscsi') self._path = "%s/%s" % (self.configfs_dir, 'xen-pvscsi') self.features = ("nexus", "tpgts") self.wwn_types = ('naa',) @@ -469,7 +469,7 @@ fabric_modules = { "tcm_fc": FCoEFabricModule, # "usb_gadget": USBGadgetFabricModule, # very rare, don't show "vhost": VhostFabricModule, - "xen_pvscsi": XenPvScsiFabricModule, + "xen-pvscsi": XenPvScsiFabricModule, "ibmvscsis": IbmvscsisFabricModule, } -- GitLab From 5b48b49a767183ad7898eb545eb8291b609ca718 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 26 Jan 2018 11:29:46 -0800 Subject: [PATCH 60/70] version 2.1.fb65 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 08226c3..0e157b0 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.64', + version = '2.1.65', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From 4a909fd73063cabe6f387590e388b9e77fa46c75 Mon Sep 17 00:00:00 2001 From: Daniel B Date: Sat, 27 Jan 2018 15:36:19 +0100 Subject: [PATCH 61/70] Fix unqualified reference to pyudev.Device Fixes crash when using a block device with the fileio backstore. --- rtslib/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtslib/utils.py b/rtslib/utils.py index ff1f7a5..3156610 100644 --- a/rtslib/utils.py +++ b/rtslib/utils.py @@ -149,7 +149,7 @@ def get_size_for_blk_dev(path): @raises: DeviceNotFoundError if corresponding device not found @raises: EnvironmentError, ValueError in some situations ''' - device = Device.from_device_file(_CONTEXT, os.path.realpath(str(path))) + device = pyudev.Device.from_device_file(_CONTEXT, os.path.realpath(str(path))) return _get_size_for_dev(device) get_block_size = get_size_for_blk_dev -- GitLab From b068e611c0b8541f2a8453076af31f8fee96dc85 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 31 Jan 2018 13:40:11 -0800 Subject: [PATCH 62/70] version 2.1.fb66 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0e157b0..3fa4eb2 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.65', + version = '2.1.66', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From f06103354394ffa86817696a37edb7ddd351bc10 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Fri, 9 Feb 2018 17:20:26 +0530 Subject: [PATCH 63/70] create: remove stale hba-only dir curretly if there exist any stale hba-only directories, $ ls /sys/kernel/config/target/core/user_0/ hba_info hba_mode Then backend creation fails, $ targetcli /backstores/fileio create block file 1M This _Backstore already exists in configFS We will have to restart target.service or restore the configuration again to clear the stale hba-only dirs. This patch checks for hba directory before creating one, if it is already existing and found to be stale then removes it. Thanks to "Maurizio Lombardi " for debugging along Signed-off-by: Prasanna Kumar Kalever --- rtslib/node.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rtslib/node.py b/rtslib/node.py index c0092fc..1d77cd1 100644 --- a/rtslib/node.py +++ b/rtslib/node.py @@ -52,9 +52,16 @@ class CFSNode(object): ''' if mode not in ['any', 'lookup', 'create']: raise RTSLibError("Invalid mode: %s" % mode) + if self.exists and mode == 'create': - raise RTSLibError("This %s already exists in configFS" - % self.__class__.__name__) + # ensure that self.path is not stale hba-only dir + if os.path.samefile(os.path.dirname(self.path), self.configfs_dir+'/core') \ + and not next(os.walk(self.path))[1]: + os.rmdir(self.path) + else: + raise RTSLibError("This %s already exists in configFS" + % self.__class__.__name__) + elif not self.exists and mode == 'lookup': raise RTSLibNotInCFS("No such %s in configfs: %s" % (self.__class__.__name__, self.path)) -- GitLab From a46e6bf9ea0f83acdff1761a11c502ea5863945f Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 12 Feb 2018 09:34:38 -0800 Subject: [PATCH 64/70] version 2.1.fb67 Signed-off-by: Andy Grover --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3fa4eb2..dd26dc4 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ from setuptools import setup setup ( name = 'rtslib-fb', - version = '2.1.66', + version = '2.1.67', description = 'API for Linux kernel SCSI target (aka LIO)', license = 'Apache 2.0', maintainer = 'Andy Grover', -- GitLab From bc9e47b2af0daac0ef52ce4b6ba804f74b415ec3 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Sat, 17 Feb 2018 19:59:17 +0100 Subject: [PATCH 65/70] debian/watch: download tarballs from the open-iscsi GitHub organization The watch file is slightly modified because `uscan` was not happy with the old watch file: uscan warn: Filename pattern missing version delimiters () without filenamemangle Moreover, the old watch file did not match any release published upstream. Signed-off-by: Christophe Vu-Brugier --- debian/watch | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/debian/watch b/debian/watch index 10dd34f..886293c 100644 --- a/debian/watch +++ b/debian/watch @@ -1,4 +1,3 @@ -version=3 -opts="uversionmangle=s/v//" \ -opts="uversionmangle=s/fb//" \ -https://github.com/agrover/rtslib-fb/tags .*/(\d[\d\.]+)\.tar\.gz +version=4 +opts=uversionmangle=s/fb//,filenamemangle=s/.+\/v?(\d\S+)\.tar\.gz/rtslib-fb-$1\.tar\.gz/ \ + https://github.com/open-iscsi/rtslib-fb/releases .*/v?(\d\S+)\.tar\.gz -- GitLab From e0b6cef98ac54328daa7248bf6d12ee0734c4c8f Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Sat, 17 Feb 2018 20:02:41 +0100 Subject: [PATCH 66/70] debian/control: add dependency on pyudev Signed-off-by: Christophe Vu-Brugier --- debian/control | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/control b/debian/control index 1dd66e7..554786c 100644 --- a/debian/control +++ b/debian/control @@ -24,6 +24,7 @@ Homepage: https://github.com/agrover/rtslib-fb Package: python-rtslib-fb Architecture: all Depends: + python-pyudev, python-six, ${misc:Depends}, ${python:Depends}, @@ -67,6 +68,7 @@ Package: python3-rtslib-fb Architecture: all Depends: python-rtslib-fb, + python3-pyudev, python3-six, ${misc:Depends}, ${python3:Depends}, -- GitLab From 31cee53b91cc3a1edc37fad89b4b648261a9982f Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Sat, 17 Feb 2018 20:06:13 +0100 Subject: [PATCH 67/70] debian: update the URL of the upstream repo to the open-iscsi organization Signed-off-by: Christophe Vu-Brugier --- debian/control | 2 +- debian/copyright | 2 +- debian/rules | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/control b/debian/control index 554786c..2db164a 100644 --- a/debian/control +++ b/debian/control @@ -19,7 +19,7 @@ Build-Depends-Indep: Standards-Version: 4.1.0 Vcs-Browser: https://salsa.debian.org/openstack-team/python/python-rtslib-fb Vcs-Git: https://salsa.debian.org/openstack-team/python/python-rtslib-fb.git -Homepage: https://github.com/agrover/rtslib-fb +Homepage: https://github.com/open-iscsi/rtslib-fb Package: python-rtslib-fb Architecture: all diff --git a/debian/copyright b/debian/copyright index 15dcdd9..cba7065 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,6 +1,6 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: rtslib-fb -Source: https://github.com/agrover/rtslib-fb +Source: https://github.com/open-iscsi/rtslib-fb Files: debian/* Copyright: 2009-2013, Jerome Martin diff --git a/debian/rules b/debian/rules index 6125919..d3bebae 100755 --- a/debian/rules +++ b/debian/rules @@ -6,7 +6,7 @@ PYTHON3S:=$(shell py3versions -vr) #prevent internet access to use PyPi export http_proxy = http://127.0.0.1:9 -UPSTREAM_GIT := https://github.com/agrover/rtslib-fb.git +UPSTREAM_GIT := https://github.com/open-iscsi/rtslib-fb.git -include /usr/share/openstack-pkg-tools/pkgos.make %: -- GitLab From ce6bc42333649efdebd9048325126e91e27e1e42 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Sat, 10 Feb 2018 19:42:11 +0100 Subject: [PATCH 68/70] debian/patches: rediff patches Signed-off-by: Christophe Vu-Brugier --- debian/patches/fix-path-of-etc-saveconfig.json.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/patches/fix-path-of-etc-saveconfig.json.patch b/debian/patches/fix-path-of-etc-saveconfig.json.patch index ccf0092..ca68b3f 100644 --- a/debian/patches/fix-path-of-etc-saveconfig.json.patch +++ b/debian/patches/fix-path-of-etc-saveconfig.json.patch @@ -22,9 +22,9 @@ Index: python-rtslib-fb/rtslib/root.py =================================================================== --- python-rtslib-fb.orig/rtslib/root.py +++ python-rtslib-fb/rtslib/root.py -@@ -29,7 +29,7 @@ from .tcm import so_mapping, StorageObje - from .utils import RTSLibError, modprobe, mount_configfs - from .utils import dict_remove, set_attributes +@@ -32,7 +32,7 @@ from .utils import dict_remove, set_attr + from .utils import fread, fwrite + from .alua import ALUATargetPortGroup -default_save_file = "/etc/target/saveconfig.json" +default_save_file = "/etc/rtslib-fb-target/saveconfig.json" -- GitLab From 2ec88cc993a644478d2142a39a1aa1ff5d448971 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Tue, 13 Feb 2018 19:47:50 +0100 Subject: [PATCH 69/70] debian/patches: patch the value of the preferred dbroot Since the saveconfig.json configuration file is saved in /etc/rtslib-fb-target, the preferred dbroot location where the kernel may store persistent data across reboots (PR or ALUA) has to be updated too. Signed-off-by: Christophe Vu-Brugier --- debian/patches/fix-path-of-etc-saveconfig.json.patch | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/patches/fix-path-of-etc-saveconfig.json.patch b/debian/patches/fix-path-of-etc-saveconfig.json.patch index ca68b3f..2f4884d 100644 --- a/debian/patches/fix-path-of-etc-saveconfig.json.patch +++ b/debian/patches/fix-path-of-etc-saveconfig.json.patch @@ -31,3 +31,12 @@ Index: python-rtslib-fb/rtslib/root.py class RTSRoot(CFSNode): ''' +@@ -63,7 +63,7 @@ class RTSRoot(CFSNode): + # this should match the kernel target driver default db dir + _default_dbroot = "/var/target" + # this is where the target DB is to be located (instead of the default) +- _preferred_dbroot = "/etc/target" ++ _preferred_dbroot = "/etc/rtslib-fb-target" + + def __init__(self): + ''' -- GitLab From 716f1819fd837bb1dc284e92ead12e15bf6d7ea4 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Sat, 17 Feb 2018 21:43:19 +0100 Subject: [PATCH 70/70] debian/changelog: add entry for new upstream version Signed-off-by: Christophe Vu-Brugier --- debian/changelog | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 2025072..d6d7078 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -python-rtslib-fb (2.1.57+debian-4) UNRELEASED; urgency=medium +python-rtslib-fb (2.1.67-1) UNRELEASED; urgency=medium [ Ondřej Nový ] * Fixed VCS URLs (https). @@ -20,7 +20,10 @@ python-rtslib-fb (2.1.57+debian-4) UNRELEASED; urgency=medium [ Ondřej Nový ] * d/control: Set Vcs-* to salsa.debian.org - -- Ondřej Nový Sun, 28 Feb 2016 15:49:37 +0100 + [ Christophe Vu-Brugier ] + * New upstream version. + + -- Christophe Vu-Brugier Sat, 17 Feb 2018 21:43:08 +0100 python-rtslib-fb (2.1.57+debian-3) unstable; urgency=medium -- GitLab