Commit 652d6524 authored by Bas Couwenberg's avatar Bas Couwenberg

New upstream version 1.8.1

parent d2bec597
......@@ -3,6 +3,25 @@ Changes
All issue numbers are relative to https://github.com/Toblerity/Fiona/issues.
1.8.1 (2018-11-15)
------------------
Bug fixes:
- Add checks around OSRGetAuthorityName and OSRGetAuthorityCode calls that will
log problems with looking up these items.
- Opened data sources are now released before we raise exceptions in
WritingSession.start (#676). This fixes an issue with locked files on
Windows.
- We now ensure that an Env instance exists when getting the crs or crs_wkt
properties of a Collection (#673, #690). Otherwise, required GDAL and PROJ
data files included in Fiona wheels can not be found.
- GDAL and PROJ data search has been refactored to improve testability (#678).
- In the project's Cython code, void* pointers have been replaced with proper
GDAL types (#672).
- Pervasive warning level log messages about ENCODING creation options (#668)
have been eliminated.
1.8.0 (2018-10-31)
------------------
......@@ -74,8 +93,8 @@ Refactoring:
Deprecations:
- The ``rasterio.drivers()`` context manager is officially deprecated. All
users should switch to ``rasterio.Env()``, which registers format drivers and
- The ``fiona.drivers()`` context manager is officially deprecated. All
users should switch to ``fiona.Env()``, which registers format drivers and
manages GDAL configuration in a reversible manner.
Bug fixes:
......
......@@ -85,12 +85,12 @@ from fiona.drvsupport import supported_drivers
from fiona.env import ensure_env_with_credentials, Env
from fiona.errors import FionaDeprecationWarning
from fiona._env import driver_count
from fiona._env import (
calc_gdal_version_num, get_gdal_version_num, get_gdal_release_name,
get_gdal_version_tuple)
from fiona.compat import OrderedDict
from fiona.io import MemoryFile
from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, _remove_layer
from fiona.ogrext import (
calc_gdal_version_num, get_gdal_version_num, get_gdal_release_name,
get_gdal_version_tuple)
from fiona.path import ParsedPath, parse_path, vsi_path
from fiona.vfs import parse_paths as vfs_parse_paths
......@@ -101,12 +101,13 @@ import uuid
__all__ = ['bounds', 'listlayers', 'open', 'prop_type', 'prop_width']
__version__ = "1.8.0"
__version__ = "1.8.1"
__gdal_version__ = get_gdal_release_name()
gdal_version = get_gdal_version_tuple()
log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())
@ensure_env_with_credentials
......
......@@ -18,7 +18,8 @@ cdef extern from "cpl_conv.h":
cdef extern from "cpl_error.h":
void CPLSetErrorHandler (void *handler)
ctypedef void (*CPLErrorHandler)(int, int, const char*);
void CPLSetErrorHandler (CPLErrorHandler handler)
cdef extern from "gdal.h":
......@@ -105,7 +106,7 @@ cdef class GDALEnv(object):
GDALAllRegister()
if OGRGetDriverCount() == 0:
OGRRegisterAll()
CPLSetErrorHandler(<void *>errorHandler)
CPLSetErrorHandler(<CPLErrorHandler>errorHandler)
if OGRGetDriverCount() == 0:
raise ValueError("Drivers not registered")
......
......@@ -10,6 +10,7 @@ option is set to a new value inside the thread.
include "gdal.pxi"
from collections import namedtuple
import logging
import os
import os.path
......@@ -53,14 +54,61 @@ log = logging.getLogger(__name__)
cdef bint is_64bit = sys.maxsize > 2 ** 32
def calc_gdal_version_num(maj, min, rev):
"""Calculates the internal gdal version number based on major, minor and revision
GDAL Version Information macro changed with GDAL version 1.10.0 (April 2013)
"""
if (maj, min, rev) >= (1, 10, 0):
return int(maj * 1000000 + min * 10000 + rev * 100)
else:
return int(maj * 1000 + min * 100 + rev * 10)
def get_gdal_version_num():
"""Return current internal version number of gdal"""
return int(GDALVersionInfo("VERSION_NUM"))
def get_gdal_release_name():
"""Return release name of gdal"""
cdef const char *name_c = NULL
name_c = GDALVersionInfo("RELEASE_NAME")
name = name_c
return name
GDALVersion = namedtuple("GDALVersion", ["major", "minor", "revision"])
def get_gdal_version_tuple():
"""
Calculates gdal version tuple from gdal's internal version number.
GDAL Version Information macro changed with GDAL version 1.10.0 (April 2013)
"""
gdal_version_num = get_gdal_version_num()
if gdal_version_num >= calc_gdal_version_num(1, 10, 0):
major = gdal_version_num // 1000000
minor = (gdal_version_num - (major * 1000000)) // 10000
revision = (gdal_version_num - (major * 1000000) - (minor * 10000)) // 100
return GDALVersion(major, minor, revision)
else:
major = gdal_version_num // 1000
minor = (gdal_version_num - (major * 1000)) // 100
revision = (gdal_version_num - (major * 1000) - (minor * 100)) // 10
return GDALVersion(major, minor, revision)
cdef void log_error(CPLErr err_class, int err_no, const char* msg) with gil:
"""Send CPL debug messages and warnings to Python's logger."""
log = logging.getLogger(__name__)
if err_class < 3:
if err_no in code_map:
log.log(level_map[err_class], "%s in %s", code_map[err_no], msg)
else:
log.info("Unknown error number %r", err_no)
if err_no in code_map:
log.log(level_map[err_class], "%s", msg)
else:
log.info("Unknown error number %r", err_no)
# Definition of GDAL callback functions, one for Windows and one for
......@@ -186,6 +234,59 @@ cdef class ConfigEnv(object):
return {k: get_gdal_config(k) for k in self.options}
class GDALDataFinder(object):
"""Finds GDAL and PROJ data files"""
def search(self, prefix=None):
"""Returns GDAL_DATA location"""
path = self.search_wheel(prefix or __file__)
if not path:
path = self.search_prefix(prefix or sys.prefix)
if not path:
path = self.search_debian(prefix or sys.prefix)
return path
def search_wheel(self, prefix=None):
"""Check wheel location"""
if prefix is None:
prefix = __file__
datadir = os.path.abspath(os.path.join(os.path.dirname(prefix), "gdal_data"))
return datadir if os.path.exists(os.path.join(datadir, 'pcs.csv')) else None
def search_prefix(self, prefix=sys.prefix):
"""Check sys.prefix location"""
datadir = os.path.join(prefix, 'share', 'gdal')
return datadir if os.path.exists(os.path.join(datadir, 'pcs.csv')) else None
def search_debian(self, prefix=sys.prefix):
"""Check Debian locations"""
gdal_version = get_gdal_version_tuple()
datadir = os.path.join(prefix, 'share', 'gdal', '{}.{}'.format(gdal_version.major, gdal_version.minor))
return datadir if os.path.exists(os.path.join(datadir, 'pcs.csv')) else None
class PROJDataFinder(object):
def search(self, prefix=None):
"""Returns PROJ_LIB location"""
path = self.search_wheel(prefix or __file__)
if not path:
path = self.search_prefix(prefix or sys.prefix)
return path
def search_wheel(self, prefix=None):
"""Check wheel location"""
if prefix is None:
prefix = __file__
datadir = os.path.abspath(os.path.join(os.path.dirname(prefix), "proj_data"))
return datadir if os.path.exists(datadir) else None
def search_prefix(self, prefix=sys.prefix):
"""Check sys.prefix location"""
datadir = os.path.join(prefix, 'share', 'proj')
return datadir if os.path.exists(datadir) else None
cdef class GDALEnv(ConfigEnv):
"""Configuration and driver management"""
......@@ -210,40 +311,22 @@ cdef class GDALEnv(ConfigEnv):
if 'GDAL_DATA' not in os.environ:
# We will try a few well-known paths, starting with the
# official wheel path.
whl_datadir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "gdal_data"))
fhs_share_datadir = os.path.join(sys.prefix, 'share/gdal')
# Debian supports multiple GDAL installs.
gdal_release_name = GDALVersionInfo("RELEASE_NAME")
deb_share_datadir = os.path.join(
fhs_share_datadir,
"{}.{}".format(*gdal_release_name.split('.')[:2]))
path = GDALDataFinder().search()
# If we find GDAL data at the well-known paths, we will
# add a GDAL_DATA key to the config options dict.
if os.path.exists(os.path.join(whl_datadir, 'pcs.csv')):
self.update_config_options(GDAL_DATA=whl_datadir)
if path:
log.debug("GDAL data found in %r", path)
self.update_config_options(GDAL_DATA=path)
elif os.path.exists(os.path.join(deb_share_datadir, 'pcs.csv')):
self.update_config_options(GDAL_DATA=deb_share_datadir)
elif os.path.exists(os.path.join(fhs_share_datadir, 'pcs.csv')):
self.update_config_options(GDAL_DATA=fhs_share_datadir)
else:
self.update_config_options(GDAL_DATA=os.environ['GDAL_DATA'])
if 'PROJ_LIB' not in os.environ:
whl_datadir = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'proj_data'))
share_datadir = os.path.join(sys.prefix, 'share/proj')
if os.path.exists(whl_datadir):
os.environ['PROJ_LIB'] = whl_datadir
path = PROJDataFinder().search()
elif os.path.exists(share_datadir):
os.environ['PROJ_LIB'] = share_datadir
if path:
log.debug("PROJ data found in %r", path)
os.environ['PROJ_LIB'] = path
if driver_count() == 0:
CPLPopErrorHandler()
......@@ -274,8 +357,8 @@ cdef class GDALEnv(ConfigEnv):
result = {}
for i in range(OGRGetDriverCount()):
drv = OGRGetDriver(i)
key = OGR_Dr_GetName(drv)
val = OGR_Dr_GetName(drv)
key = <char *>OGR_Dr_GetName(drv)
val = <char *>OGR_Dr_GetName(drv)
result[key] = val
return result
......@@ -66,7 +66,7 @@ class CPLE_BaseError(Exception):
return self.__unicode__()
def __unicode__(self):
return "{}".format(self.errmsg)
return u"{}".format(self.errmsg)
@property
def args(self):
......@@ -203,7 +203,7 @@ cdef class GDALErrCtxManager:
cdef inline object exc_check():
"""Checks GDAL error stack for fatal or non-fatal errors
Returns
-------
An Exception, SystemExit, or None
......
......@@ -3,6 +3,81 @@
ctypedef int OGRErr
cdef extern from "ogr_core.h":
ctypedef enum OGRwkbGeometryType:
wkbUnknown
wkbPoint
wkbLineString
wkbPolygon
wkbMultiPoint
wkbMultiLineString
wkbMultiPolygon
wkbGeometryCollection
wkbCircularString
wkbCompoundCurve
wkbCurvePolygon
wkbMultiCurve
wkbMultiSurface
wkbCurve
wkbSurface
wkbPolyhedralSurface
wkbTIN
wkbTriangle
wkbNone
wkbLinearRing
wkbCircularStringZ
wkbCompoundCurveZ
wkbCurvePolygonZ
wkbMultiCurveZ
wkbMultiSurfaceZ
wkbCurveZ
wkbSurfaceZ
wkbPolyhedralSurfaceZ
wkbTINZ
wkbTriangleZ
wkbPointM
wkbLineStringM
wkbPolygonM
wkbMultiPointM
wkbMultiLineStringM
wkbMultiPolygonM
wkbGeometryCollectionM
wkbCircularStringM
wkbCompoundCurveM
wkbCurvePolygonM
wkbMultiCurveM
wkbMultiSurfaceM
wkbCurveM
wkbSurfaceM
wkbPolyhedralSurfaceM
wkbTINM
wkbTriangleM
wkbPointZM
wkbLineStringZM
wkbPolygonZM
wkbMultiPointZM
wkbMultiLineStringZM
wkbMultiPolygonZM
wkbGeometryCollectionZM
wkbCircularStringZM
wkbCompoundCurveZM
wkbCurvePolygonZM
wkbMultiCurveZM
wkbMultiSurfaceZM
wkbCurveZM
wkbSurfaceZM
wkbPolyhedralSurfaceZM
wkbTINZM
wkbTriangleZM
wkbPoint25D
wkbLineString25D
wkbPolygon25D
wkbMultiPoint25D
wkbMultiLineString25D
wkbMultiPolygon25D
wkbGeometryCollection25D
ctypedef struct OGREnvelope:
double MinX
double MaxX
......@@ -15,7 +90,7 @@ cdef extern from "ogr_api.h":
void OGR_G_AddPoint (void *geometry, double x, double y, double z)
void OGR_G_AddPoint_2D (void *geometry, double x, double y)
void OGR_G_CloseRings (void *geometry)
void * OGR_G_CreateGeometry (int wkbtypecode)
void * OGR_G_CreateGeometry (OGRwkbGeometryType wkbtypecode)
void OGR_G_DestroyGeometry (void *geometry)
unsigned char * OGR_G_ExportToJson (void *geometry)
void OGR_G_ExportToWkb (void *geometry, int endianness, char *buffer)
......
......@@ -85,7 +85,7 @@ cdef void * _createOgrGeomFromWKB(object wkb) except NULL:
"""Make an OGR geometry from a WKB string"""
wkbtype = bytearray(wkb)[1]
cdef unsigned char *buffer = wkb
cdef void *cogr_geometry = OGR_G_CreateGeometry(wkbtype)
cdef void *cogr_geometry = OGR_G_CreateGeometry(<OGRwkbGeometryType>wkbtype)
if cogr_geometry is not NULL:
OGR_G_ImportFromWkb(cogr_geometry, buffer, len(wkb))
return cogr_geometry
......@@ -191,7 +191,7 @@ cdef class OGRGeomBuilder:
"""Builds OGR geometries from Fiona geometries.
"""
cdef void * _createOgrGeometry(self, int geom_type) except NULL:
cdef void *cogr_geometry = OGR_G_CreateGeometry(geom_type)
cdef void *cogr_geometry = OGR_G_CreateGeometry(<OGRwkbGeometryType>geom_type)
if cogr_geometry == NULL:
raise Exception("Could not create OGR Geometry of type: %i" % geom_type)
return cogr_geometry
......@@ -286,11 +286,6 @@ cdef class OGRGeomBuilder:
raise ValueError("Unsupported geometry type %s" % typename)
cdef geometry(void *geom):
"""Factory for Fiona geometries"""
return GeomBuilder().build(geom)
def geometryRT(geometry):
# For testing purposes only, leaks the JSON data
cdef void *cogr_geometry = OGRGeomBuilder().build(geometry)
......
......@@ -10,7 +10,7 @@ ctypedef enum OGRFieldSubType:
cdef bint is_field_null(void *feature, int n)
cdef void set_field_null(void *feature, int n)
cdef void gdal_flush_cache(void *cogr_ds)
cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL
cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) except NULL
cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NULL
cdef OGRErr gdal_start_transaction(void *cogr_ds, int force)
cdef OGRErr gdal_commit_transaction(void *cogr_ds)
......
......@@ -3,7 +3,7 @@ include "ogrext2.pxd"
cdef bint is_field_null(void *feature, int n)
cdef void set_field_null(void *feature, int n)
cdef void gdal_flush_cache(void *cogr_ds)
cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL
cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) except NULL
cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NULL
cdef OGRErr gdal_start_transaction(void *cogr_ds, int force)
cdef OGRErr gdal_commit_transaction(void *cogr_ds)
......
......@@ -25,7 +25,7 @@ cdef void gdal_flush_cache(void *cogr_ds):
GDALFlushCache(cogr_ds)
cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL:
cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) except NULL:
cdef void* cogr_ds = NULL
cdef char **drvs = NULL
cdef char **open_opts = NULL
......
......@@ -3,7 +3,7 @@ include "ogrext2.pxd"
cdef bint is_field_null(void *feature, int n)
cdef void set_field_null(void *feature, int n)
cdef void gdal_flush_cache(void *cogr_ds)
cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL
cdef void* gdal_open_vector(const char *path_c, int mode, drivers, options) except NULL
cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NULL
cdef OGRErr gdal_start_transaction(void *cogr_ds, int force)
cdef OGRErr gdal_commit_transaction(void *cogr_ds)
......
......@@ -30,7 +30,7 @@ cdef void gdal_flush_cache(void *cogr_ds):
GDALFlushCache(cogr_ds)
cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL:
cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) except NULL:
cdef void* cogr_ds = NULL
cdef char **drvs = NULL
cdef void* drv = NULL
......@@ -64,7 +64,7 @@ cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NUL
try:
cogr_ds = exc_wrap_pointer(
GDALOpenEx(path_c, flags, <const char *const *>drvs, open_opts, NULL)
GDALOpenEx(path_c, flags, <const char *const *>drvs, <const char *const *>open_opts, NULL)
)
return cogr_ds
except FionaNullPointerError:
......
......@@ -8,12 +8,12 @@ import warnings
from fiona import compat, vfs
from fiona.ogrext import Iterator, ItemsIterator, KeysIterator
from fiona.ogrext import Session, WritingSession
from fiona.ogrext import (
calc_gdal_version_num, get_gdal_version_num, get_gdal_release_name)
from fiona.ogrext import buffer_to_virtual_file, remove_virtual_file, GEOMETRY_TYPES
from fiona.errors import (DriverError, SchemaError, CRSError, UnsupportedGeometryTypeError, DriverSupportError)
from fiona.logutils import FieldSkipLogFilter
from fiona._env import driver_count
from fiona._env import (
calc_gdal_version_num, get_gdal_version_num, get_gdal_release_name)
from fiona.env import Env
from fiona.errors import FionaDeprecationWarning
from fiona.drvsupport import supported_drivers
......
"""Fiona's GDAL/AWS environment"""
from contextlib import contextmanager
from functools import wraps, total_ordering
import logging
import re
......@@ -9,7 +10,8 @@ import attr
from six import string_types
from fiona._env import (
GDALEnv, get_gdal_config, set_gdal_config)
GDALEnv, calc_gdal_version_num, get_gdal_version_num, get_gdal_config,
set_gdal_config, get_gdal_release_name)
from fiona.compat import getargspec
from fiona.errors import EnvError, GDALVersionError
from fiona.session import Session, DummySession
......@@ -160,15 +162,16 @@ class Env(object):
self.context_options = {}
@classmethod
def from_defaults(cls, *args, **kwargs):
def from_defaults(cls, session=None, **options):
"""Create an environment with default config options
Parameters
----------
args : optional
Positional arguments for Env()
kwargs : optional
Keyword arguments for Env()
session : optional
A Session object.
**options : optional
A mapping of GDAL configuration options, e.g.,
`CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.
Returns
-------
......@@ -179,9 +182,9 @@ class Env(object):
The items in kwargs will be overlaid on the default values.
"""
options = Env.default_options()
options.update(**kwargs)
return Env(*args, **options)
opts = Env.default_options()
opts.update(**options)
return Env(session=session, **opts)
@property
def is_credentialized(self):
......@@ -313,16 +316,55 @@ def delenv():
local._env = None
class NullContextManager(object):
def __init__(self):
pass
def __enter__(self):
return self
def __exit__(self, *args):
pass
def env_ctx_if_needed():
"""Return an Env if one does not exist
Returns
-------
Env or a do-nothing context manager
"""
if local._env:
return NullContextManager()
else:
return Env.from_defaults()
def ensure_env(f):
"""A decorator that ensures an env exists before a function
calls any GDAL C functions."""
calls any GDAL C functions.
Parameters
----------
f : function
A function.
Returns
-------
A function wrapper.
Notes
-----
If there is already an existing environment, the wrapper does
nothing and immediately calls f with the given arguments.
"""
@wraps(f)
def wrapper(*args, **kwds):
def wrapper(*args, **kwargs):
if local._env:
return f(*args, **kwds)
return f(*args, **kwargs)
else:
with Env.from_defaults():
return f(*args, **kwds)
return f(*args, **kwargs)
return wrapper
......@@ -344,25 +386,23 @@ def ensure_env_with_credentials(f):
credentializes the environment if the first argument is a URI with
scheme "s3".
If there is already an existing environment, the wrapper does
nothing and immediately calls f with the given arguments.
"""
@wraps(f)
def wrapper(*args, **kwds):
def wrapper(*args, **kwargs):
if local._env:
env_ctor = Env
else:
env_ctor = Env.from_defaults
if hascreds():
session = DummySession()
elif isinstance(args[0], str):
session = Session.from_path(args[0])
return f(*args, **kwargs)
else:
session = Session.from_path(None)
with env_ctor(session=session):
log.debug("Credentialized: {!r}".format(getenv()))
return f(*args, **kwds)
if isinstance(args[0], str):
session = Session.from_path(args[0])
else:
session = Session.from_path(None)
with Env.from_defaults(session=session):
log.debug("Credentialized: {!r}".format(getenv()))
return f(*args, **kwargs)
return wrapper
......@@ -425,7 +465,6 @@ class GDALVersion(object):
@classmethod
def runtime(cls):
"""Return GDALVersion of current GDAL runtime"""
from fiona.ogrext import get_gdal_release_name # to avoid circular import
return cls.parse(get_gdal_release_name())
def at_least(self, other):
......
This diff is collapsed.
......@@ -52,6 +52,94 @@ ctypedef struct OGREnvelope:
double MaxY
cdef extern from "ogr_core.h":
ctypedef enum OGRwkbGeometryType:
wkbUnknown
wkbPoint
wkbLineString
wkbPolygon
wkbMultiPoint
wkbMultiLineString
wkbMultiPolygon
wkbGeometryCollection
wkbCircularString
wkbCompoundCurve
wkbCurvePolygon
wkbMultiCurve
wkbMultiSurface
wkbCurve
wkbSurface
wkbPolyhedralSurface
wkbTIN
wkbTriangle
wkbNone
wkbLinearRing
wkbCircularStringZ
wkbCompoundCurveZ
wkbCurvePolygonZ
wkbMultiCurveZ
wkbMultiSurfaceZ
wkbCurveZ
wkbSurfaceZ
wkbPolyhedralSurfaceZ
wkbTINZ
wkbTriangleZ
wkbPointM
wkbLineStringM
wkbPolygonM
wkbMultiPointM
wkbMultiLineStringM
wkbMultiPolygonM
wkbGeometryCollectionM
wkbCircularStringM
wkbCompoundCurveM
wkbCurvePolygonM
wkbMultiCurveM
wkbMultiSurfaceM
wkbCurveM
wkbSurfaceM
wkbPolyhedralSurfaceM
wkbTINM
wkbTriangleM
wkbPointZM
wkbLineStringZM
wkbPolygonZM
wkbMultiPointZM
wkbMultiLineStringZM
wkbMultiPolygonZM
wkbGeometryCollectionZM
wkbCircularStringZM
wkbCompoundCurveZM
wkbCurvePolygonZM
wkbMultiCurveZM
wkbMultiSurfaceZM
wkbCurveZM
wkbSurfaceZM
wkbPolyhedralSurfaceZM
wkbTINZM
wkbTriangleZM
wkbPoint25D
wkbLineString25D
wkbPolygon25D
wkbMultiPoint25D
wkbMultiLineString25D
wkbMultiPolygon25D
wkbGeometryCollection25D
ctypedef enum OGRFieldType:
OFTInteger
OFTIntegerList
OFTReal
OFTRealList
OFTString
OFTStringList
OFTWideString
OFTWideStringList
OFTBinary
OFTDate
OFTTime
OFTDateTime