.travis.yml

@@ -57,6 +57,6 @@ install:
   - "python -m pip list"
 script:
   - "if [[ $TRAVIS_PYTHON_VERSION == 3.5 && $GDALVERSION == 2.1.0 ]]; then python -m pytest --doctest-ignore-import-errors --doctest-glob='*.rst' docs/*.rst -k 'not index and not quickstart and not switch' ; fi"
-  - python -m pytest -rxXs --cov rasterio --cov-report term-missing
+  - python -m pytest -m "not wheel" -rxXs --cov rasterio --cov-report term-missing
 after_success:
   - coveralls || echo "!! intermittent coveralls failure"

CHANGES.txt

 Changes
 =======
 
+1.0.11 (2019-11-30)
+-------------------
+
+- Prevent needless warning when making boundless reads with a fill value
+  (#1499).
+- The GDAL band cache is flushed at the top of build_overviews to fix #1497.
+- Options --gdal-data and --proj-data have been added to the rio-env command so
+  that users of Rasterio wheels can get paths to set GDAL_DATA and PROJ_LIB
+  environment variables.
+- The attempt to make GDAL and PROJ support file discovery and configuration
+  automatic within methods of the CRS class has been reverted. Users must
+  execute such code inside a `with Env()` block or set the GDAL_DATA and
+  PROJ_LIB environment variables needed by GDAL.
+
 1.0.10 (2019-11-16)
 -------------------
 

ISSUE_TEMPLATE.md

@@ -11,10 +11,14 @@ often omit.
 The primary forum for questions about installation and usage of Rasterio is 
 https://rasterio.groups.io/g/main. The authors and other users will answer 
 questions when they have expertise to share and time to explain. Please take the
-time to craft a clear question and be patient about responses.
-
-Please do not bring these questions to Rasterio's issue tracker, which we want
-to reserve for bug reports and other actionable issues.
+time to craft a clear question and be patient about responses. Please do not
+bring these questions to Rasterio's issue tracker, which we want to reserve for
+bug reports and other actionable issues.
+
+Questions about development of Rasterio, brainstorming, requests for comment,
+and not-yet-actionable proposals are welcome in the project's developers 
+discussion group https://rasterio.groups.io/g/dev. Issues opened in Rasterio's
+GitHub repo which haven't been socialized there may be perfunctorily closed.
 
 Please note: Rasterio contains extension modules and is thus susceptible to
 C library compatibility issues. If you are reporting an installation or module

README.rst

@@ -195,8 +195,7 @@ Linux
 Rasterio distributions are available from UbuntuGIS and Anaconda's conda-forge
 channel.
 
-`Manylinux1 <https://github.com/pypa/manylinux>`__ distributions may be
-available in the future.
+`Manylinux1 <https://github.com/pypa/manylinux>`__ wheels are available on PyPI.```
 
 OS X
 ++++

debian/changelog

+rasterio (1.0.11-1) unstable; urgency=medium
+
+  * Team upload.
+  * New upstream release.
+
+ -- Bas Couwenberg <sebastic@debian.org>  Sat, 01 Dec 2018 08:49:06 +0100
+
 rasterio (1.0.10-1) unstable; urgency=medium
 
   * Team upload.

rasterio/__init__.py

@@ -5,7 +5,6 @@ from __future__ import absolute_import
 from collections import namedtuple
 from contextlib import contextmanager
 import logging
-import warnings
 
 try:
     from pathlib import Path
@@ -43,7 +42,7 @@ import rasterio.path
 
 
 __all__ = ['band', 'open', 'pad', 'Env']
-__version__ = "1.0.10"
+__version__ = "1.0.11"
 __gdal_version__ = gdal_version()
 
 # Rasterio attaches NullHandler to the 'rasterio' logger and its

rasterio/_base.pyx

@@ -23,7 +23,7 @@ from rasterio.coords import BoundingBox
 from rasterio.crs import CRS
 from rasterio.enums import (
     ColorInterp, Compression, Interleaving, MaskFlags, PhotometricInterp)
-from rasterio.env import Env, env_ctx_if_needed
+from rasterio.env import Env
 from rasterio.errors import (
     RasterioIOError, CRSError, DriverRegistrationError, NotGeoreferencedWarning,
     RasterBlockError, BandOverviewError)
@@ -273,7 +273,6 @@ cdef class DatasetBase(object):
         wkt_b = wkt.encode('utf-8')
         cdef const char *wkt_c = wkt_b
 
-        with env_ctx_if_needed():
         try:
 
             osr = exc_wrap_pointer(OSRNewSpatialReference(wkt_c))
@@ -309,12 +308,10 @@ cdef class DatasetBase(object):
         """Return the GDAL dataset's stored CRS"""
         cdef const char *wkt_b = NULL
 
-        with env_ctx_if_needed():
         wkt_b = GDALGetProjectionRef(self._hds)
         if wkt_b == NULL:
             raise ValueError("Unexpected NULL spatial reference")
         wkt = wkt_b
-            log.debug("WKT: %r", wkt)
         return self._handle_crswkt(wkt)
 
     def read_transform(self):

rasterio/_crs.pyx

@@ -33,7 +33,6 @@ class _CRS(UserDict):
         cdef OGRSpatialReferenceH osr_crs = NULL
         cdef int retval
 
-        with env_ctx_if_needed():
         try:
             osr_crs = osr_from_crs(self)
             retval = OSRIsGeographic(osr_crs)
@@ -52,7 +51,6 @@ class _CRS(UserDict):
         cdef OGRSpatialReferenceH osr_crs = NULL
         cdef int retval
 
-        with env_ctx_if_needed():
         try:
             osr_crs = osr_from_crs(self)
             retval = OSRIsProjected(osr_crs)
@@ -95,7 +93,6 @@ class _CRS(UserDict):
         cdef char *srcwkt = NULL
         cdef OGRSpatialReferenceH osr = NULL
 
-        with env_ctx_if_needed():
         try:
             osr = osr_from_crs(self)
             OSRExportToWkt(osr, &srcwkt)
@@ -113,7 +110,6 @@ class _CRS(UserDict):
         """
         cdef OGRSpatialReferenceH osr = NULL
 
-        with env_ctx_if_needed():
         try:
             osr = osr_from_crs(self)
             if OSRAutoIdentifyEPSG(osr) == 0:
@@ -396,6 +392,7 @@ _param_data = """
 +zone      UTM zone
 """
 
+with env_ctx_if_needed():
     _lines = filter(lambda x: len(x) > 1, _param_data.split("\n"))
     all_proj_keys = list(set(line.split()[0].lstrip("+").strip()
                              for line in _lines)) + ['no_mayo']

rasterio/_env.pyx

@@ -187,10 +187,22 @@ cdef class ConfigEnv(object):
 
 
 class GDALDataFinder(object):
-    """Finds GDAL and PROJ data files"""
+    """Finds GDAL data files
+
+    Note: this is not part of the public API in 1.0.x.
+
+    """
 
     def search(self, prefix=None):
-        """Returns GDAL_DATA location"""
+        """Returns GDAL data directory
+
+        Note well that os.environ is not consulted.
+
+        Returns
+        -------
+        str or None
+
+        """
         path = self.search_wheel(prefix or __file__)
         if not path:
             path = self.search_prefix(prefix or sys.prefix)
@@ -207,20 +219,33 @@ class GDALDataFinder(object):
 
     def search_prefix(self, prefix=sys.prefix):
         """Check sys.prefix location"""
-        datadir = os.path.join(prefix, 'share/gdal')
+        datadir = os.path.join(prefix, 'share', 'gdal')
         return datadir if os.path.exists(os.path.join(datadir, 'pcs.csv')) else None
 
     def search_debian(self, prefix=sys.prefix):
         """Check Debian locations"""
         gdal_release_name = GDALVersionInfo("RELEASE_NAME")
-        datadir = os.path.join(prefix, 'share/gdal', '{}.{}'.format(*gdal_release_name.split('.')[:2]))
+        datadir = os.path.join(prefix, 'share', 'gdal', '{}.{}'.format(*gdal_release_name.split('.')[:2]))
         return datadir if os.path.exists(os.path.join(datadir, 'pcs.csv')) else None
 
 
 class PROJDataFinder(object):
+    """Finds PROJ data files
+
+    Note: this is not part of the public API in 1.0.x.
+
+    """
 
     def search(self, prefix=None):
-        """Returns PROJ_LIB location"""
+        """Returns PROJ data directory
+
+        Note well that os.environ is not consulted.
+
+        Returns
+        -------
+        str or None
+
+        """
         path = self.search_wheel(prefix or __file__)
         if not path:
             path = self.search_prefix(prefix or sys.prefix)
@@ -235,7 +260,7 @@ class PROJDataFinder(object):
 
     def search_prefix(self, prefix=sys.prefix):
         """Check sys.prefix location"""
-        datadir = os.path.join(prefix, 'share/proj')
+        datadir = os.path.join(prefix, 'share', 'proj')
         return datadir if os.path.exists(datadir) else None
 
 
@@ -248,7 +273,6 @@ cdef class GDALEnv(ConfigEnv):
 
     def start(self):
         CPLPushErrorHandler(<CPLErrorHandler>logging_error_handler)
-        log.debug("Logging error handler pushed.")
 
         # The outer if statement prevents each thread from acquiring a
         # lock when the environment starts, and the inner avoids a
@@ -259,31 +283,28 @@ cdef class GDALEnv(ConfigEnv):
 
                     GDALAllRegister()
                     OGRRegisterAll()
-                    log.debug("All drivers registered.")
 
-                    if 'GDAL_DATA' not in os.environ:
+                    if 'GDAL_DATA' in os.environ:
+                        self.update_config_options(GDAL_DATA=os.environ['GDAL_DATA'])
+                        log.debug("GDAL_DATA found in environment: %r.", os.environ['GDAL_DATA'])
 
+                    else:
                         path = GDALDataFinder().search()
 
                         if path:
-                            log.debug("GDAL data found in %r", path)
                             self.update_config_options(GDAL_DATA=path)
-
-                    else:
-                        self.update_config_options(GDAL_DATA=os.environ['GDAL_DATA'])
+                            log.debug("GDAL_DATA not found in environment, set to %r.", path)
 
                     if 'PROJ_LIB' not in os.environ:
 
                         path = PROJDataFinder().search()
 
                         if path:
-                            log.debug("PROJ data found in %r", path)
                             os.environ['PROJ_LIB'] = path
-
+                            log.debug("PROJ data not found in environment, set to %r.", path)
 
                     if driver_count() == 0:
                         CPLPopErrorHandler()
-                        log.debug("Error handler popped")
                         raise ValueError("Drivers not registered.")
 
                     # Flag the drivers as registered, otherwise every thread
@@ -298,9 +319,7 @@ cdef class GDALEnv(ConfigEnv):
         # NB: do not restore the CPL error handler to its default
         # state here. If you do, log messages will be written to stderr
         # by GDAL instead of being sent to Python's logging module.
-        log.debug("Stopping GDALEnv %r.", self)
         CPLPopErrorHandler()
-        log.debug("Error handler popped.")
         log.debug("Stopped GDALEnv %r.", self)
 
     def drivers(self):

rasterio/_io.pyx

@@ -15,7 +15,6 @@ import warnings
 import numpy as np
 
 from rasterio._base import tastes_like_gdal, gdal_version
-from rasterio._env import driver_count, GDALEnv
 from rasterio._err import (
     GDALError, CPLE_OpenFailedError, CPLE_IllegalArgError, CPLE_BaseError)
 from rasterio.crs import CRS
@@ -373,7 +372,7 @@ cdef class DatasetReaderBase(DatasetBase):
                 with DatasetWriterBase(
                         bg_path, 'w',
                         driver='GTiff', count=self.count, height=3, width=3,
-                        dtype=dtype, crs=None, transform=None) as bg_dataset:
+                        dtype=dtype, crs=None, transform=Affine.identity() * Affine.translation(1, 1)) as bg_dataset:
                     bg_dataset.write(
                         np.full((self.count, 3, 3), fill_value, dtype=dtype))
                 bg_dataset = DatasetReaderBase(bg_path)
@@ -1644,6 +1643,7 @@ cdef class DatasetWriterBase(DatasetReaderBase):
                 6: 'MODE',
                 7: 'GAUSS'}
             resampling_alg = resampling_map[Resampling(resampling.value)]
+
         except (KeyError, ValueError):
             raise ValueError(
                 "resampling must be one of: {0}".format(", ".join(
@@ -1658,12 +1658,15 @@ cdef class DatasetWriterBase(DatasetReaderBase):
         # Allocate arrays.
         if factors:
             factors_c = <int *>CPLMalloc(len(factors)*sizeof(int))
+
             for i, factor in enumerate(factors):
                 factors_c[i] = factor
+
             try:
                 resampling_b = resampling_alg.encode('utf-8')
                 resampling_c = resampling_b
-                err = exc_wrap_int(
+                GDALFlushCache(self._hds)
+                exc_wrap_int(
                     GDALBuildOverviews(self._hds, resampling_c,
                                        len(factors), factors_c, 0, NULL, NULL,
                                        NULL))

rasterio/enums.py

@@ -6,8 +6,8 @@ from enum import Enum, IntEnum
 class ColorInterp(IntEnum):
     """Raster band color interpretation."""
     undefined = 0
-    grey = 1
     gray = 1
+    grey = 1
     palette = 2
     red = 3
     green = 4

rasterio/plot.py

@@ -155,7 +155,8 @@ def plotting_extent(source, transform=None):
     Parameters
     ----------
     source : array or dataset object opened in 'r' mode
-        input data
+        If array, data in the order rows, columns and optionally bands. If array
+        is band order (bands in the first dimension), use arr[0]
     transform: Affine, required if source is array
         Defines the affine transform if source is an array
 

rasterio/rio/env.py

 """Fetch and edit raster dataset metadata from the command line."""
 
 import json
+import os
 
 import click
 
-import rasterio
+from rasterio._env import GDALDataFinder, PROJDataFinder
 
 
 @click.command(short_help="Print information about the Rasterio environment.")
@@ -12,6 +13,10 @@ import rasterio
               help="Enumerate the available formats.")
 @click.option('--credentials', 'key', flag_value='credentials', default=False,
               help="Print credentials.")
+@click.option('--gdal-data', 'key', flag_value='gdal_data', default=False,
+              help="Print GDAL data path.")
+@click.option('--proj-data', 'key', flag_value='proj_data', default=False,
+              help="Print PROJ data path.")
 @click.pass_context
 def env(ctx, key):
     """Print information about the Rasterio environment."""
@@ -21,3 +26,7 @@ def env(ctx, key):
                 click.echo("{0}: {1}".format(k, v))
         elif key == 'credentials':
             click.echo(json.dumps(env.session.credentials))
+        elif key == 'gdal_data':
+            click.echo(os.environ.get('GDAL_DATA') or GDALDataFinder().search())
+        elif key == 'proj_data':
+            click.echo(os.environ.get('PROJ_LIB') or PROJDataFinder().search())

tests/conftest.py

@@ -463,6 +463,7 @@ def gdalenv(request):
         if rasterio.env.local._env:
             rasterio.env.delenv()
             rasterio.env.local._env = None
+
     request.addfinalizer(fin)
 
 

tests/test__env.py

@@ -11,7 +11,7 @@ from .conftest import gdal_version
 def mock_wheel(tmpdir):
     """A fake rasterio wheel"""
     moduledir = tmpdir.mkdir("rasterio")
-    moduledir.ensure("__init__,py")
+    moduledir.ensure("__init__.py")
     moduledir.ensure("_env.py")
     moduledir.ensure("gdal_data/pcs.csv")
     moduledir.ensure("proj_data/epsg")
@@ -60,7 +60,7 @@ def test_search_prefix_gdal_data_failure(tmpdir):
 def test_search_prefix_gdal_data(mock_fhs):
     """Find GDAL data under prefix"""
     finder = GDALDataFinder()
-    assert finder.search_prefix(str(mock_fhs)) == str(mock_fhs.join("share/gdal"))
+    assert finder.search_prefix(str(mock_fhs)) == str(mock_fhs.join("share").join("gdal"))
 
 
 def test_search_debian_gdal_data_failure(tmpdir):
@@ -72,7 +72,7 @@ def test_search_debian_gdal_data_failure(tmpdir):
 def test_search_debian_gdal_data(mock_debian):
     """Find GDAL data under Debian locations"""
     finder = GDALDataFinder()
-    assert finder.search_debian(str(mock_debian)) == str(mock_debian.join("share/gdal/{}".format(str(gdal_version))))
+    assert finder.search_debian(str(mock_debian)) == str(mock_debian.join("share").join("gdal").join("{}".format(str(gdal_version))))
 
 
 def test_search_gdal_data_wheel(mock_wheel):
@@ -82,13 +82,13 @@ def test_search_gdal_data_wheel(mock_wheel):
 
 def test_search_gdal_data_fhs(mock_fhs):
     finder = GDALDataFinder()
-    assert finder.search(str(mock_fhs)) == str(mock_fhs.join("share/gdal"))
+    assert finder.search(str(mock_fhs)) == str(mock_fhs.join("share").join("gdal"))
 
 
 def test_search_gdal_data_debian(mock_debian):
     """Find GDAL data under Debian locations"""
     finder = GDALDataFinder()
-    assert finder.search(str(mock_debian)) == str(mock_debian.join("share/gdal/{}".format(str(gdal_version))))
+    assert finder.search(str(mock_debian)) == str(mock_debian.join("share").join("gdal").join("{}".format(str(gdal_version))))
 
 
 def test_search_wheel_proj_data_failure(tmpdir):
@@ -112,7 +112,7 @@ def test_search_prefix_proj_data_failure(tmpdir):
 def test_search_prefix_proj_data(mock_fhs):
     """Find GDAL data under prefix"""
     finder = PROJDataFinder()
-    assert finder.search_prefix(str(mock_fhs)) == str(mock_fhs.join("share/proj"))
+    assert finder.search_prefix(str(mock_fhs)) == str(mock_fhs.join("share").join("proj"))
 
 
 def test_search_proj_data_wheel(mock_wheel):
@@ -122,4 +122,4 @@ def test_search_proj_data_wheel(mock_wheel):
 
 def test_search_proj_data_fhs(mock_fhs):
     finder = PROJDataFinder()
-    assert finder.search(str(mock_fhs)) == str(mock_fhs.join("share/proj"))
+    assert finder.search(str(mock_fhs)) == str(mock_fhs.join("share").join("proj"))

tests/test_blocks.py

@@ -112,6 +112,7 @@ class WindowWriteTest(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.tempdir)
 
+    @pytest.mark.gdalbin
     def test_write_window(self):
         name = os.path.join(self.tempdir, "test_write_window.tif")
         a = np.ones((50, 50), dtype=rasterio.ubyte) * 127

tests/test_crs.py

@@ -7,6 +7,7 @@ import pytest
 import rasterio
 from rasterio._base import _can_create_osr
 from rasterio.crs import CRS
+from rasterio.env import env_ctx_if_needed
 from rasterio.errors import CRSError
 
 from .conftest import requires_gdal21, requires_gdal22
@@ -40,6 +41,7 @@ def test_read_esri_wkt(tmpdir):
         }
 
 
+@pytest.mark.gdalbin
 def test_read_epsg3857(tmpdir):
     tiffname = str(tmpdir.join('lol.tif'))
     subprocess.call([
@@ -50,6 +52,7 @@ def test_read_epsg3857(tmpdir):
 
 
 # Ensure that CRS sticks when we write a file.
+@pytest.mark.gdalbin
 def test_write_3857(tmpdir):
     src_path = str(tmpdir.join('lol.tif'))
     subprocess.call([
@@ -327,3 +330,12 @@ def test_compound_crs():
 def test_dataset_compound_crs():
     with rasterio.open("tests/data/compdcs.vrt") as dataset:
         assert dataset.crs.wkt.startswith('GEOGCS["WGS 84"')
+
+
+@pytest.mark.wheel
+def test_environ_patch(gdalenv, monkeypatch):
+    """GDAL_DATA is patched as when rasterio._crs is imported"""
+    monkeypatch.delenv('GDAL_DATA', raising=False)
+    monkeypatch.delenv('PROJ_LIB', raising=False)
+    with env_ctx_if_needed():
+        assert CRS.from_epsg(4326) != CRS(units='m', proj='aeqd', ellps='WGS84', datum='WGS84', lat_0=-17.0, lon_0=-44.0)

tests/test_data_paths.py

+"""Tests of GDAL and PROJ data finding"""
+
+import os.path
+
+from click.testing import CliRunner
+import pytest
+
+import rasterio
+from rasterio._env import GDALDataFinder, PROJDataFinder
+from rasterio.rio.main import main_group
+
+
+@pytest.mark.wheel
+def test_gdal_data():
+    """Get GDAL data path from a wheel"""
+    assert GDALDataFinder().search() == os.path.join(os.path.dirname(rasterio.__file__), 'gdal_data')
+
+
+@pytest.mark.wheel
+def test_proj_data():
+    """Get GDAL data path from a wheel"""
+    assert PROJDataFinder().search() == os.path.join(os.path.dirname(rasterio.__file__), 'proj_data')
+
+
+@pytest.mark.wheel
+def test_env_gdal_data():
+    runner = CliRunner()
+    result = runner.invoke(main_group, ['env', '--gdal-data'])
+    assert result.exit_code == 0
+    assert result.output.strip() == os.path.join(os.path.dirname(rasterio.__file__), 'gdal_data')
+
+
+@pytest.mark.wheel
+def test_env_proj_data():
+    runner = CliRunner()
+    result = runner.invoke(main_group, ['env', '--proj-data'])
+    assert result.exit_code == 0
+    assert result.output.strip() == os.path.join(os.path.dirname(rasterio.__file__), 'proj_data')

tests/test_enums.py

+"""Enum tests"""
+
+from rasterio import enums
+
+
+def test_grey_gray():
+    """Name of ColorInterp.grey is 'gray'"""
+    assert enums.ColorInterp.grey.name == "gray"
+
+
+def test_gray_gray():
+    """Name of ColorInterp.gray is 'gray'"""
+    assert enums.ColorInterp.gray.name == "gray"

tests/test_env.py

@@ -134,7 +134,7 @@ def test_ensure_env_decorator_sets_gdal_data_prefix(gdalenv, monkeypatch, tmpdir
     monkeypatch.delenv('GDAL_DATA', raising=False)
     monkeypatch.setattr(sys, 'prefix', str(tmpdir))
 
-    assert f() == str(tmpdir.join("share/gdal"))
+    assert f() == str(tmpdir.join("share").join("gdal"))
 
 
 def test_ensure_env_decorator_sets_gdal_data_wheel(gdalenv, monkeypatch, tmpdir):