.travis.yml

@@ -16,11 +16,13 @@ env:
     - GDALVERSION="2.1.4"
     - GDALVERSION="2.2.4"
     - GDALVERSION="2.3.3"
-    - GDALVERSION="2.4.0"
+    - GDALVERSION="2.4.1"
+    - GDALVERSION="3.0.0"
     - GDALVERSION="master"
 
 matrix:
   allow_failures:
+    - env: GDALVERSION="3.0.0"
     - env: GDALVERSION="master"
 
 addons:

CHANGES.txt

 Changes
 =======
 
+1.0.24 (2019-06-05)
+-------------------
+
+- By default GDAL reuses existing dataset connections when possible.
+  Multi-threaded programs must avoid this as must programs that re-open
+  datasets in a different context, such as the overview-seeking program shown
+  in issue #1504. The sharing keyword argument of rasterio.open() was supposed
+  to allow users to disable connection reuse on a case-by-case basis, but the
+  implementation was faulty and connection reuse persisted (#1701). This has
+  been fixed in PR #1704.
+
 1.0.23 (2019-05-15)
 -------------------
 

debian/changelog

-rasterio (1.0.23-2) UNRELEASED; urgency=medium
+rasterio (1.0.24-1) unstable; urgency=medium
 
+  * Team upload.
+  * New upstream release.
   * Update dh_install override to use --list-missing.
 
- -- Bas Couwenberg <sebastic@debian.org>  Mon, 27 May 2019 06:51:52 +0200
+ -- Bas Couwenberg <sebastic@debian.org>  Thu, 06 Jun 2019 06:16:12 +0200
 
 rasterio (1.0.23-1) unstable; urgency=medium
 

rasterio/__init__.py

@@ -42,7 +42,7 @@ import rasterio.path
 
 
 __all__ = ['band', 'open', 'pad', 'Env']
-__version__ = "1.0.23"
+__version__ = "1.0.24"
 __gdal_version__ = gdal_version()
 
 # Rasterio attaches NullHandler to the 'rasterio' logger and its
@@ -174,7 +174,7 @@ def open(fp, mode='r', driver=None, width=None, height=None, count=None,
         @contextmanager
         def fp_reader(fp):
             memfile = MemoryFile(fp.read())
-            dataset = memfile.open()
+            dataset = memfile.open(driver=driver, sharing=sharing)
             try:
                 yield dataset
             finally:
@@ -190,7 +190,7 @@ def open(fp, mode='r', driver=None, width=None, height=None, count=None,
             memfile = MemoryFile()
             dataset = memfile.open(driver=driver, width=width, height=height,
                                    count=count, crs=crs, transform=transform,
-                                   dtype=dtype, nodata=nodata, **kwargs)
+                                   dtype=dtype, nodata=nodata, sharing=sharing, **kwargs)
             try:
                 yield dataset
             finally:
@@ -213,15 +213,16 @@ def open(fp, mode='r', driver=None, width=None, height=None, count=None,
         # be taken over by the dataset's context manager if it is not
         # None.
         if mode == 'r':
-            s = DatasetReader(path, driver=driver, **kwargs)
+            s = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
         elif mode == 'r+':
-            s = get_writer_for_path(path)(path, mode, driver=driver, **kwargs)
+            s = get_writer_for_path(path)(path, mode, driver=driver, sharing=sharing, **kwargs)
         elif mode.startswith("w"):
             s = get_writer_for_driver(driver)(path, mode, driver=driver,
                                               width=width, height=height,
                                               count=count, crs=crs,
                                               transform=transform,
                                               dtype=dtype, nodata=nodata,
+                                              sharing=sharing,
                                               **kwargs)
         else:
             raise ValueError(

rasterio/_base.pyx

@@ -197,6 +197,8 @@ cdef class DatasetBase(object):
         cdef int flags = 0
         cdef int sharing_flag = (0x20 if sharing else 0x0)
 
+        log.debug("Sharing flag: %r", sharing_flag)
+
         self._hds = NULL
 
         if path is not None:

rasterio/_env.pyx

@@ -21,6 +21,8 @@ from rasterio._base cimport _safe_osr_release
 from rasterio._err import CPLE_BaseError
 from rasterio._err cimport exc_wrap_ogrerr, exc_wrap_int
 
+from libc.stdio cimport stderr
+
 
 level_map = {
     0: 0,
@@ -400,3 +402,6 @@ cdef class GDALEnv(ConfigEnv):
             result[key] = val
 
         return result
+
+    def _dump_open_datasets(self):
+        GDALDumpOpenDatasets(stderr)

rasterio/env.py

@@ -245,6 +245,13 @@ class Env(object):
         """Return a mapping of registered drivers."""
         return local._env.drivers()
 
+    def _dump_open_datasets(self):
+        """Writes descriptions of open datasets to stderr
+
+        For debugging and testing purposes.
+        """
+        return local._env._dump_open_datasets()
+
     def __enter__(self):
         log.debug("Entering env context: %r", self)
         if local._env is None:

rasterio/gdal.pxi

@@ -280,6 +280,9 @@ cdef extern from "gdal.h" nogil:
     CPLErr GDALSetRasterScale(GDALRasterBandH hBand, double dfNewScale)
     CPLErr GDALSetRasterOffset(GDALRasterBandH hBand, double dfNewOffset)
 
+    int GDALDumpOpenDatasets(FILE *fp)
+
+
 cdef extern from "ogr_api.h" nogil:
 
     ctypedef void * OGRLayerH

rasterio/io.py

@@ -106,7 +106,7 @@ class MemoryFile(MemoryFileBase):
 
     @ensure_env
     def open(self, driver=None, width=None, height=None, count=None, crs=None,
-             transform=None, dtype=None, nodata=None, **kwargs):
+             transform=None, dtype=None, nodata=None, sharing=False, **kwargs):
         """Open the file and return a Rasterio dataset object.
 
         If data has already been written, the file is opened in 'r'
@@ -127,13 +127,13 @@ class MemoryFile(MemoryFileBase):
             raise IOError("I/O operation on closed file.")
         if self.exists():
             log.debug("VSI path: {}".format(vsi_path.path))
-            return DatasetReader(vsi_path, driver=driver, **kwargs)
+            return DatasetReader(vsi_path, driver=driver, sharing=sharing, **kwargs)
         else:
             writer = get_writer_for_driver(driver)
             return writer(vsi_path, 'w+', driver=driver, width=width,
                           height=height, count=count, crs=crs,
                           transform=transform, dtype=dtype,
-                          nodata=nodata, **kwargs)
+                          nodata=nodata, sharing=sharing, **kwargs)
 
     def __enter__(self):
         self._env = env_ctx_if_needed()
@@ -156,7 +156,7 @@ class ZipMemoryFile(MemoryFile):
         super(ZipMemoryFile, self).__init__(file_or_bytes, ext='zip')
 
     @ensure_env
-    def open(self, path, driver=None, **kwargs):
+    def open(self, path, driver=None, sharing=False, **kwargs):
         """Open a dataset within the zipped stream.
 
         Parameters
@@ -165,6 +165,9 @@ class ZipMemoryFile(MemoryFile):
             Path to a dataset in the zip file, relative to the root of the
             archive.
 
+        Other parameters are optional and have the same semantics as the
+        parameters of `rasterio.open()`.
+
         Returns
         -------
         A Rasterio dataset object
@@ -173,7 +176,7 @@ class ZipMemoryFile(MemoryFile):
 
         if self.closed:
             raise IOError("I/O operation on closed file.")
-        return DatasetReader(vsi_path, driver=driver, **kwargs)
+        return DatasetReader(vsi_path, driver=driver, sharing=sharing, **kwargs)
 
 
 def get_writer_for_driver(driver):

tests/conftest.py

@@ -488,6 +488,11 @@ def path_rgb_msk_byte_tif(data_dir):
     return os.path.join(data_dir, 'RGB2.byte.tif')
 
 
+@pytest.fixture(scope='session')
+def path_cogeo_tif(data_dir):
+    return os.path.join(data_dir, 'cogeo.tif')
+
+
 @pytest.fixture(scope='function')
 def _path_multiband_no_colorinterp(tmpdir):
 

tests/test_open_overview_level.py

+"""Tests for fix of #1504"""
+
+import rasterio
+
+from .conftest import requires_gdal2
+
+
+@requires_gdal2
+def test_overview_levels(path_cogeo_tif):
+    """With sharing turned off, problem noted in #1504 vanishes"""
+    olevel = 0
+    with rasterio.open(path_cogeo_tif, overview_level=olevel) as src:
+        assert src.shape == (512, 512)
+
+        olevel = 1
+        with rasterio.open(path_cogeo_tif, sharing=False, overview_level=olevel) as src:
+            assert src.shape == (256, 256)
+
+            olevel = 2
+            with rasterio.open(path_cogeo_tif, sharing=False, overview_level=olevel) as src:
+                assert src.shape == (128, 128)

tests/test_open_sharing.py

+"""Tests of dataset connection sharing"""
+
+import rasterio
+
+from .conftest import requires_gdal2
+
+
+@requires_gdal2
+def test_sharing_on(capfd, path_rgb_byte_tif):
+    """Datasets are shared"""
+    with rasterio.Env() as env:
+
+        # Opens a new file.
+        with rasterio.open(path_rgb_byte_tif, sharing=False) as srcx:
+            env._dump_open_datasets()
+            captured = capfd.readouterr()
+            assert "1 N GTiff" in captured.err
+            assert "1 S GTiff" not in captured.err
+
+            # Does not open a new file.
+            with rasterio.open(path_rgb_byte_tif, sharing=True) as srcy:
+                env._dump_open_datasets()
+                captured = capfd.readouterr()
+                assert "1 N GTiff" in captured.err
+                assert "1 S GTiff" in captured.err
+
+
+@requires_gdal2
+def test_sharing_off(capfd, path_rgb_byte_tif):
+    """Datasets are not shared"""
+    with rasterio.Env() as env:
+
+        # Opens a new file.
+        with rasterio.open(path_rgb_byte_tif, sharing=False) as srcx:
+            env._dump_open_datasets()
+            captured = capfd.readouterr()
+            assert "1 N GTiff" in captured.err
+            assert "1 S GTiff" not in captured.err
+
+            # Does not open a new file.
+            with rasterio.open(path_rgb_byte_tif, sharing=False) as srcy:
+                env._dump_open_datasets()
+                captured = capfd.readouterr()
+                assert captured.err.count("1 N GTiff") == 2
+                assert "1 S GTiff" not in captured.err