CHANGES.txt

@@ -3,6 +3,32 @@ Changes
 
 All issue numbers are relative to https://github.com/Toblerity/Fiona/issues.
 
+1.8.5 (2019-03-15)
+------------------
+
+- GDAL seems to work best if GDAL_DATA is set as early as possible. Ideally it
+  is set when building the library or in the environment before importing
+  Fiona, but for wheels we patch GDAL_DATA into os.environ when fiona.env
+  is imported. This resolves #731.
+- A combination of bugs which allowed .cpg files to be overlooked has been
+  fixed (#726).
+- On entering a collection context (Collection.__enter__) a new anonymous GDAL
+  environment is created if needed and entered. This makes `with
+  fiona.open(...) as collection:` roughly equivalent to `with fiona.open(...)
+  as collection, Env():`. This helps prevent bugs when Collections are created
+  and then used later or in different scopes.
+- Missing GDAL support for TopoJSON, GeoJSONSeq, and ESRIJSON has been enabled
+  (#721).
+- A regression in handling of polygons with M values (#724) has been fixed.
+- Per-feature debug logging calls in OGRFeatureBuilder methods have been
+  eliminated to improve feature writing performance (#718).
+- Native support for datasets in Google Cloud Storage identified by "gs"
+  resource names has been added (#709).
+- Support has been added for triangle, polyhedral surface, and TIN geometry
+  types (#679).
+- Notes about using the MemoryFile and ZipMemoryFile classes has been added to
+  the manual (#674).
+
 1.8.4 (2018-12-10)
 ------------------
 

README.rst

@@ -24,7 +24,7 @@ Fiona is supported only on CPython versions 2.7 and 3.4+.
 For more details, see:
 
 * Fiona `home page <https://github.com/Toblerity/Fiona>`__
-* Fiona `docs and manual <http://toblerity.github.com/fiona/>`__
+* Fiona `docs and manual <https://fiona.readthedocs.io/en/stable/>`__
 * Fiona `examples <https://github.com/Toblerity/Fiona/tree/master/examples>`__
 
 Usage
@@ -54,7 +54,7 @@ file, change their geometry attributes, and write them to a new data file.
         # collection's ``meta`` property and then modify them as
         # desired.
 
-        meta = source.meta
+        meta = src.meta
         meta['schema']['geometry'] = 'Point'
 
         # Open an output file, using the same format driver and
@@ -64,7 +64,7 @@ file, change their geometry attributes, and write them to a new data file.
         with fiona.open('test_write.shp', 'w', **meta) as dst:
 
             # Process only the records intersecting a box.
-            for f in source.filter(bbox=(-107.0, 37.0, -105.0, 39.0)):
+            for f in src.filter(bbox=(-107.0, 37.0, -105.0, 39.0)):
 
                 # Get a point on the boundary of the record's
                 # geometry.
@@ -214,7 +214,7 @@ info`` pretty prints information about a data file.
 Installation
 ============
 
-Fiona requires Python 2.6, 2.7, 3.3, or 3.4 and GDAL/OGR 1.8+. To build from
+Fiona requires Python 2.7 or 3.4+ and GDAL/OGR 1.8+. To build from
 a source distribution you will need a C compiler and GDAL and Python
 development headers and libraries (libgdal1-dev for Debian/Ubuntu, gdal-dev for
 CentOS/Fedora).

appveyor.yml

@@ -45,8 +45,8 @@ environment:
           PYTHON_VERSION: "3.6.4"
           PYTHON_ARCH: "64"
           GDAL_VERSION: "2.4.0"
-          GIS_INTERNALS: "release-1911-x64-gdal-mapserver.zip"
-          GIS_INTERNALS_LIBS: "release-1911-x64-gdal-mapserver-libs.zip"
+          GIS_INTERNALS: "release-1911-x64-gdal-2-4-0-mapserver-7-2-2.zip"
+          GIS_INTERNALS_LIBS: "release-1911-x64-gdal-2-4-0-mapserver-7-2-2-libs.zip"
 
 install:
 
@@ -125,7 +125,7 @@ build_script:
 
   # install the wheel
   - ps: python -m pip install --upgrade pip
-  - ps: pip install --force-reinstall --ignore-installed (gci dist\*.whl | % { "$_" })
+  - ps: python -m pip install --force-reinstall --ignore-installed (gci dist\*.whl | % { "$_" })
   - ps: move fiona fiona.build
 
 
@@ -140,7 +140,7 @@ test_script:
 
 matrix:
   allow_failures:
-    - GDAL_VERSION: 2.4.0
+    - PYTHON_VERSION: "2.7.14"
 
 artifacts:
   - path: dist\*.whl

debian/changelog

-fiona (1.8.4-2) UNRELEASED; urgency=medium
+fiona (1.8.5-1~exp1) experimental; urgency=medium
 
+  * Team upload.
+  * New upstream release.
   * Bump Standards-Version to 4.3.0, no changes.
+  * Refresh patches.
+  * Add python{,3}-mock to build dependencies.
 
- -- Bas Couwenberg <sebastic@debian.org>  Tue, 25 Dec 2018 22:07:36 +0100
+ -- Bas Couwenberg <sebastic@debian.org>  Sat, 16 Mar 2019 08:34:02 +0100
 
 fiona (1.8.4-1) unstable; urgency=medium
 

debian/control

@@ -22,6 +22,8 @@ Build-Depends: debhelper (>= 9),
                python-cligj,
                python3-cligj,
                python-enum34,
+               python-mock,
+               python3-mock,
                python-munch,
                python3-munch,
                python-pytest,

debian/patches/0001-Rename-fio-command-to-fiona-to-avoid-name-clash.patch

@@ -9,7 +9,7 @@ There is already another package providing a binary "fio" (fio).
 
 --- a/setup.py
 +++ b/setup.py
-@@ -306,7 +306,7 @@ setup_args = dict(
+@@ -309,7 +309,7 @@ setup_args = dict(
      packages=['fiona', 'fiona.fio'],
      entry_points='''
          [console_scripts]

debian/patches/0006-Remove-unknown-distribution-options.patch

@@ -6,7 +6,7 @@ Author: Bas Couwenberg <sebastic@debian.org>
 
 --- a/setup.py
 +++ b/setup.py
-@@ -288,11 +288,8 @@ extras_require['all'] = list(set(it.chai
+@@ -291,11 +291,8 @@ extras_require['all'] = list(set(it.chai
  
  setup_args = dict(
      cmdclass={'sdist': sdist_multi_gdal},

fiona/__init__.py

@@ -101,7 +101,7 @@ import uuid
 
 
 __all__ = ['bounds', 'listlayers', 'open', 'prop_type', 'prop_width']
-__version__ = "1.8.4"
+__version__ = "1.8.5"
 __gdal_version__ = get_gdal_release_name()
 
 gdal_version = get_gdal_version_tuple()

fiona/_env.pyx

@@ -17,6 +17,9 @@ import os.path
 import sys
 import threading
 
+from fiona._err cimport exc_wrap_int, exc_wrap_ogrerr
+from fiona._err import CPLE_BaseError
+
 
 level_map = {
     0: 0,
@@ -54,6 +57,13 @@ log = logging.getLogger(__name__)
 cdef bint is_64bit = sys.maxsize > 2 ** 32
 
 
+cdef _safe_osr_release(OGRSpatialReferenceH srs):
+    """Wrapper to handle OSR release when NULL."""
+    if srs != NULL:
+        OSRRelease(srs)
+    srs = NULL
+
+
 def calc_gdal_version_num(maj, min, rev):
     """Calculates the internal gdal version number based on major, minor and revision
 
@@ -237,12 +247,41 @@ cdef class ConfigEnv(object):
 class GDALDataFinder(object):
     """Finds GDAL data files
 
-    Note: this class is private in 1.8.x and not in the public API.
+    Note: this is not part of the 1.8.x public API.
+
+    """
+    def find_file(self, basename):
+        """Returns path of a GDAL data file or None
+
+        Parameters
+        ----------
+        basename : str
+            Basename of a data file such as "header.dxf"
+
+        Returns
+        -------
+        str (on success) or None (on failure)
 
         """
+        cdef const char *path_c = NULL
+        basename_b = basename.encode('utf-8')
+        path_c = CPLFindFile("gdal", <const char *>basename_b)
+        if path_c == NULL:
+            return None
+        else:
+            path = path_c
+            return path
 
     def search(self, prefix=None):
-        """Returns GDAL_DATA location"""
+        """Returns GDAL data directory
+
+        Note well that os.environ is not consulted.
+
+        Returns
+        -------
+        str or None
+
+        """
         path = self.search_wheel(prefix or __file__)
         if not path:
             path = self.search_prefix(prefix or sys.prefix)
@@ -264,20 +303,47 @@ class GDALDataFinder(object):
 
     def search_debian(self, prefix=sys.prefix):
         """Check Debian locations"""
-        gdal_version = get_gdal_version_tuple()
-        datadir = os.path.join(prefix, 'share', 'gdal', '{}.{}'.format(gdal_version.major, gdal_version.minor))
+        gdal_release_name = GDALVersionInfo("RELEASE_NAME")
+        datadir = os.path.join(prefix, 'share', 'gdal', '{}.{}'.format(*gdal_release_name.split('.')[:2]))
         return datadir if os.path.exists(os.path.join(datadir, 'pcs.csv')) else None
 
 
 class PROJDataFinder(object):
     """Finds PROJ data files
 
-    Note: this class is private in 1.8.x and not in the public API.
+    Note: this is not part of the public 1.8.x API.
 
     """
+    def has_data(self):
+        """Returns True if PROJ's data files can be found
+
+        Returns
+        -------
+        bool
+
+        """
+        cdef OGRSpatialReferenceH osr = OSRNewSpatialReference(NULL)
+
+        try:
+            exc_wrap_ogrerr(exc_wrap_int(OSRImportFromProj4(osr, "+init=epsg:4326")))
+        except CPLE_BaseError:
+            return False
+        else:
+            return True
+        finally:
+            _safe_osr_release(osr)
+
 
     def search(self, prefix=None):
-        """Returns PROJ_LIB location"""
+        """Returns PROJ data directory
+
+        Note well that os.environ is not consulted.
+
+        Returns
+        -------
+        str or None
+
+        """
         path = self.search_wheel(prefix or __file__)
         if not path:
             path = self.search_prefix(prefix or sys.prefix)
@@ -322,6 +388,10 @@ cdef class GDALEnv(ConfigEnv):
                         self.update_config_options(GDAL_DATA=os.environ['GDAL_DATA'])
                         log.debug("GDAL_DATA found in environment: %r.", os.environ['GDAL_DATA'])
 
+                    # See https://github.com/mapbox/rasterio/issues/1631.
+                    elif GDALDataFinder().find_file("header.dxf"):
+                        log.debug("GDAL data files are available at built-in paths")
+
                     else:
                         path = GDALDataFinder().search()
 
@@ -329,8 +399,13 @@ cdef class GDALEnv(ConfigEnv):
                             self.update_config_options(GDAL_DATA=path)
                             log.debug("GDAL_DATA not found in environment, set to %r.", path)
 
-                    if 'PROJ_LIB' not in os.environ:
+                    if 'PROJ_LIB' in os.environ:
+                        log.debug("PROJ_LIB found in environment: %r.", os.environ['PROJ_LIB'])
+
+                    elif PROJDataFinder().has_data():
+                        log.debug("PROJ data files are available at built-in paths")
 
+                    else:
                         path = PROJDataFinder().search()
 
                         if path:

fiona/_err.pxd

@@ -4,7 +4,12 @@ cdef extern from "cpl_vsi.h":
 
     ctypedef FILE VSILFILE
 
+cdef extern from "ogr_core.h":
+
+    ctypedef int OGRErr
+
 
 cdef int exc_wrap_int(int retval) except -1
+cdef OGRErr exc_wrap_ogrerr(OGRErr retval) except -1
 cdef void *exc_wrap_pointer(void *ptr) except NULL
 cdef VSILFILE *exc_wrap_vsilfile(VSILFILE *f) except NULL

fiona/_err.pyx

@@ -249,6 +249,17 @@ cdef int exc_wrap_int(int err) except -1:
     return err
 
 
+cdef OGRErr exc_wrap_ogrerr(OGRErr err) except -1:
+    """Wrap a function that returns OGRErr but does not use the
+    CPL error stack.
+
+    """
+    if err == 0:
+        return err
+    else:
+        raise CPLE_BaseError(3, err, "OGR Error code {}".format(err))
+
+
 cdef void *exc_wrap_pointer(void *ptr) except NULL:
     """Wrap a GDAL/OGR function that returns GDALDatasetH etc (void *)
     Raises a Rasterio exception if a non-fatal error has be set.

fiona/_geometry.pxd

@@ -142,3 +142,5 @@ cdef class OGRGeomBuilder:
 
 cdef unsigned int geometry_type_code(object name) except? 9999
 cdef object normalize_geometry_type_code(unsigned int code)
+cdef unsigned int base_geometry_type_code(unsigned int code)
+

fiona/_geometry.pyx

@@ -80,6 +80,17 @@ cdef object normalize_geometry_type_code(unsigned int code):
     return code
 
 
+cdef inline unsigned int base_geometry_type_code(unsigned int code):
+    """ Returns base geometry code without Z, M and ZM types """
+    # Remove 2.5D flag.
+    code = code & (~0x80000000)
+
+    # Normalize Z, M, and ZM types. Fiona 1.x does not support M
+    # and doesn't treat OGC 'Z' variants as special types of their
+    # own.
+    return code % 1000
+
+
 # Geometry related functions and classes follow.
 cdef void * _createOgrGeomFromWKB(object wkb) except NULL:
     """Make an OGR geometry from a WKB string"""
@@ -162,13 +173,7 @@ cdef class GeomBuilder:
 
         cdef unsigned int etype = OGR_G_GetGeometryType(geom)
 
-        # Remove 2.5D flag.
-        self.code = etype & (~0x80000000)
-
-        # Normalize Z, M, and ZM types. Fiona 1.x does not support M
-        # and doesn't treat OGC 'Z' variants as special types of their
-        # own.
-        self.code = self.code % 1000
+        self.code = base_geometry_type_code(etype)
 
         if self.code not in GEOMETRY_TYPES:
             raise UnsupportedGeometryTypeError(self.code)

fiona/_shim1.pyx

+"""Shims on top of ogrext for GDAL versions < 2"""
+
 import logging
 
 from fiona.ogrext1 cimport *
@@ -44,14 +46,11 @@ cdef void* gdal_open_vector(const char *path_c, int mode, drivers, options) exce
         for name in drivers:
             name_b = name.encode()
             name_c = name_b
-            #log.debug("Trying driver: %s", name)
             drv = OGRGetDriverByName(name_c)
             if drv != NULL:
                 ds = OGR_Dr_Open(drv, path_c, mode)
             if ds != NULL:
                 cogr_ds = ds
-                # TODO
-                #collection._driver = name
                 break
     else:
         cogr_ds = OGROpen(path_c, mode, NULL)
@@ -95,22 +94,32 @@ cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NU
     finally:
         CSLDestroy(opts)
 
+
 # transactions are not supported in GDAL 1.x
 cdef OGRErr gdal_start_transaction(void* cogr_ds, int force):
     return OGRERR_NONE
+
+
 cdef OGRErr gdal_commit_transaction(void* cogr_ds):
     return OGRERR_NONE
+
+
 cdef OGRErr gdal_rollback_transaction(void* cogr_ds):
     return OGRERR_NONE
 
+
 # field subtypes are not supported in GDAL 1.x
 cdef OGRFieldSubType get_field_subtype(void *fielddefn):
     return OFSTNone
+
+
 cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype):
     pass
 
+
 cdef bint check_capability_create_layer(void *cogr_ds):
     return OGR_DS_TestCapability(cogr_ds, ODsCCreateLayer)
 
+
 cdef void *get_linear_geometry(void *geom):
     return geom

fiona/_shim2.pyx

+"""Shims on top of ogrext for GDAL versions > 2"""
+
 from fiona.ogrext2 cimport *
 from fiona._err cimport exc_wrap_pointer
 from fiona._err import cpl_errs, CPLE_BaseError, FionaNullPointerError
@@ -46,6 +48,10 @@ cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) exce
                 drvs = CSLAddString(drvs, name_c)
 
     for k, v in options.items():
+
+        if v is None:
+            continue
+
         k = k.upper().encode('utf-8')
         if isinstance(v, bool):
             v = ('ON' if v else 'OFF').encode('utf-8')
@@ -95,20 +101,27 @@ cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NU
 cdef OGRErr gdal_start_transaction(void* cogr_ds, int force):
     return GDALDatasetStartTransaction(cogr_ds, force)
 
+
 cdef OGRErr gdal_commit_transaction(void* cogr_ds):
     return GDALDatasetCommitTransaction(cogr_ds)
 
+
 cdef OGRErr gdal_rollback_transaction(void* cogr_ds):
     return GDALDatasetRollbackTransaction(cogr_ds)
 
+
 cdef OGRFieldSubType get_field_subtype(void *fielddefn):
     return OGR_Fld_GetSubType(fielddefn)
 
+
 cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype):
     OGR_Fld_SetSubType(fielddefn, subtype)
 
+
 cdef bint check_capability_create_layer(void *cogr_ds):
     return GDALDatasetTestCapability(cogr_ds, ODsCCreateLayer)
 
+
 cdef void *get_linear_geometry(void *geom):
     return OGR_G_GetLinearGeometry(geom, 0.0, NULL)
+

fiona/_shim22.pyx

+"""Shims on top of ogrext for GDAL versions >= 2.2"""
+
 cdef extern from "ogr_api.h":
+
     int OGR_F_IsFieldNull(void *feature, int n)
 
+
 from fiona.ogrext2 cimport *
 from fiona._err cimport exc_wrap_pointer
 from fiona._err import cpl_errs, CPLE_BaseError, FionaNullPointerError
@@ -30,7 +34,7 @@ cdef void gdal_flush_cache(void *cogr_ds):
         GDALFlushCache(cogr_ds)
 
 
-cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) except NULL:
+cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL:
     cdef void* cogr_ds = NULL
     cdef char **drvs = NULL
     cdef void* drv = NULL
@@ -46,12 +50,15 @@ cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) exce
         for name in drivers:
             name_b = name.encode()
             name_c = name_b
-            #log.debug("Trying driver: %s", name)
             drv = GDALGetDriverByName(name_c)
             if drv != NULL:
                 drvs = CSLAddString(drvs, name_c)
 
     for k, v in options.items():
+
+        if v is None:
+            continue
+
         k = k.upper().encode('utf-8')
         if isinstance(v, bool):
             v = ('ON' if v else 'OFF').encode('utf-8')
@@ -64,7 +71,7 @@ cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) exce
 
     try:
         cogr_ds = exc_wrap_pointer(
-            GDALOpenEx(path_c, flags, <const char *const *>drvs, <const char *const *>open_opts, NULL)
+            GDALOpenEx(path_c, flags, <const char *const *>drvs, open_opts, NULL)
         )
         return cogr_ds
     except FionaNullPointerError:
@@ -102,20 +109,26 @@ cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NU
 cdef OGRErr gdal_start_transaction(void* cogr_ds, int force):
     return GDALDatasetStartTransaction(cogr_ds, force)
 
+
 cdef OGRErr gdal_commit_transaction(void* cogr_ds):
     return GDALDatasetCommitTransaction(cogr_ds)
 
+
 cdef OGRErr gdal_rollback_transaction(void* cogr_ds):
     return GDALDatasetRollbackTransaction(cogr_ds)
 
+
 cdef OGRFieldSubType get_field_subtype(void *fielddefn):
     return OGR_Fld_GetSubType(fielddefn)
 
+
 cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype):
     OGR_Fld_SetSubType(fielddefn, subtype)
 
+
 cdef bint check_capability_create_layer(void *cogr_ds):
     return GDALDatasetTestCapability(cogr_ds, ODsCCreateLayer)
 
+
 cdef void *get_linear_geometry(void *geom):
     return OGR_G_GetLinearGeometry(geom, 0.0, NULL)

fiona/collection.py

@@ -11,11 +11,11 @@ from fiona.ogrext import Session, WritingSession
 from fiona.ogrext import buffer_to_virtual_file, remove_virtual_file, GEOMETRY_TYPES
 from fiona.errors import (DriverError, SchemaError, CRSError, UnsupportedGeometryTypeError, DriverSupportError)
 from fiona.logutils import FieldSkipLogFilter
-from fiona._env import driver_count, get_gdal_release_name, get_gdal_version_tuple
-from fiona.env import Env
+from fiona._env import get_gdal_release_name, get_gdal_version_tuple
+from fiona.env import env_ctx_if_needed
 from fiona.errors import FionaDeprecationWarning
 from fiona.drvsupport import supported_drivers
-from fiona.path import Path, UnparsedPath, vsi_path, parse_path
+from fiona.path import Path, vsi_path, parse_path
 from six import string_types, binary_type
 
 
@@ -150,7 +150,7 @@ class Collection(object):
                     raise CRSError("crs lacks init or proj parameter")
 
         self._driver = driver
-        kwargs.update(encoding=encoding or '')
+        kwargs.update(encoding=encoding)
         self.encoding = encoding
 
         try:
@@ -166,8 +166,6 @@ class Collection(object):
 
         if self.session is not None:
             self.guard_driver_mode()
-            if not self.encoding:
-                self.encoding = self.session.get_fileencoding().lower()
 
         if self.mode in ("a", "w"):
             self._valid_geom_types = _get_valid_geom_types(self.schema, self.driver)
@@ -459,10 +457,13 @@ class Collection(object):
 
     def __enter__(self):
         logging.getLogger('fiona.ogrext').addFilter(self.field_skip_log_filter)
+        self._env = env_ctx_if_needed()
+        self._env.__enter__()
         return self
 
     def __exit__(self, type, value, traceback):
         logging.getLogger('fiona.ogrext').removeFilter(self.field_skip_log_filter)
+        self._env.__exit__()
         self.close()
 
     def __del__(self):

fiona/env.py

@@ -3,6 +3,7 @@
 from contextlib import contextmanager
 from functools import wraps, total_ordering
 import logging
+import os
 import re
 import threading
 
@@ -11,7 +12,7 @@ from six import string_types
 
 from fiona._env import (
     GDALEnv, calc_gdal_version_num, get_gdal_version_num, get_gdal_config,
-    set_gdal_config, get_gdal_release_name)
+    set_gdal_config, get_gdal_release_name, GDALDataFinder, PROJDataFinder)
 from fiona.compat import getargspec
 from fiona.errors import EnvError, GDALVersionError
 from fiona.session import Session, DummySession
@@ -581,3 +582,32 @@ def require_gdal_version(version, param=None, values=None, is_max_version=False,
         return wrapper
 
     return decorator
+
+
+# Patch the environment if needed, such as in the installed wheel case.
+
+if 'GDAL_DATA' not in os.environ:
+
+    # See https://github.com/mapbox/rasterio/issues/1631.
+    if GDALDataFinder().find_file("header.dxf"):
+        log.debug("GDAL data files are available at built-in paths")
+
+    else:
+        path = GDALDataFinder().search()
+
+        if path:
+            os.environ['GDAL_DATA'] = path
+            log.debug("GDAL_DATA not found in environment, set to %r.", path)
+
+if 'PROJ_LIB' not in os.environ:
+
+    # See https://github.com/mapbox/rasterio/issues/1631.
+    if PROJDataFinder().has_data():
+        log.debug("PROJ data files are available at built-in paths")
+
+    else:
+        path = PROJDataFinder().search()
+
+        if path:
+            os.environ['PROJ_LIB'] = path
+            log.debug("PROJ data not found in environment, set to %r.", path)

fiona/gdal.pxi

@@ -10,6 +10,7 @@ cdef extern from "cpl_conv.h" nogil:
     void CPLSetThreadLocalConfigOption(const char* key, const char* val)
     void CPLSetConfigOption(const char* key, const char* val)
     const char* CPLGetConfigOption(const char* key, const char* default)
+    const char *CPLFindFile(const char *pszClass, const char *pszBasename)
 
 
 cdef extern from "cpl_error.h" nogil:

fiona/ogrext.pyx

@@ -18,7 +18,7 @@ from fiona._shim cimport *
 
 from fiona._geometry cimport (
     GeomBuilder, OGRGeomBuilder, geometry_type_code,
-    normalize_geometry_type_code)
+    normalize_geometry_type_code, base_geometry_type_code)
 from fiona._err cimport exc_wrap_int, exc_wrap_pointer, exc_wrap_vsilfile
 
 import fiona
@@ -42,6 +42,7 @@ from libc.stdlib cimport malloc, free
 from libc.string cimport strcmp
 from cpython cimport PyBytes_FromStringAndSize, PyBytes_AsString
 
+
 cdef extern from "ogr_api.h" nogil:
 
     ctypedef void * OGRLayerH
@@ -125,8 +126,7 @@ cdef class FeatureBuilder:
     argument is not destroyed.
     """
 
-    cdef build(self, void *feature, encoding='utf-8', bbox=False, driver=None,
-               ignore_fields=None, ignore_geometry=False):
+    cdef build(self, void *feature, encoding='utf-8', bbox=False, driver=None, ignore_fields=None, ignore_geometry=False):
         """Build a Fiona feature object from an OGR feature
 
         Parameters
@@ -262,24 +262,40 @@ cdef class FeatureBuilder:
                 props[key] = None
 
         cdef void *cogr_geometry = NULL
+        cdef void *org_geometry = NULL
 
         if not ignore_geometry:
             cogr_geometry = OGR_F_GetGeometryRef(feature)
 
             if cogr_geometry is not NULL:
-                code = OGR_G_GetGeometryType(cogr_geometry)
 
-                if 7 < code < 100:  # Curves.
+                code = base_geometry_type_code(OGR_G_GetGeometryType(cogr_geometry))
+
+                if 8 <= code <= 14:  # Curves.
                     cogr_geometry = get_linear_geometry(cogr_geometry)
                     geom = GeomBuilder().build(cogr_geometry)
                     OGR_G_DestroyGeometry(cogr_geometry)
 
+                elif 15 <= code <= 17:
+                    # We steal the geometry: the geometry of the in-memory feature is now null
+                    # and we are responsible for cogr_geometry.
+                    org_geometry = OGR_F_StealGeometry(feature)
+
+                    if code in (15, 16):
+                        cogr_geometry = OGR_G_ForceToMultiPolygon(org_geometry)
+                    elif code == 17:
+                        cogr_geometry = OGR_G_ForceToPolygon(org_geometry)
+
+                    geom = GeomBuilder().build(cogr_geometry)
+                    OGR_G_DestroyGeometry(cogr_geometry)
+
                 else:
                     geom = GeomBuilder().build(cogr_geometry)
 
                 fiona_feature["geometry"] = geom
 
             else:
+
                 fiona_feature["geometry"] = None
 
         return fiona_feature
@@ -313,9 +329,8 @@ cdef class OGRFeatureBuilder:
                                 feature['geometry'])
         OGR_F_SetGeometryDirectly(cogr_feature, cogr_geometry)
 
-        # OGR_F_SetFieldString takes UTF-8 encoded strings ('bytes' in
-        # Python 3).
-        encoding = session.get_internalencoding()
+        # OGR_F_SetFieldString takes encoded strings ('bytes' in Python 3).
+        encoding = session._get_internal_encoding()
 
         for key, value in feature['properties'].items():
             log.debug(
@@ -412,12 +427,10 @@ def featureRT(feature, collection):
     cdef void *cogr_geometry = OGR_F_GetGeometryRef(cogr_feature)
     if cogr_geometry == NULL:
         raise ValueError("Null geometry")
-    log.debug("Geometry: %s" % OGR_G_ExportToJson(cogr_geometry))
-    encoding = collection.encoding or 'utf-8'
     result = FeatureBuilder().build(
         cogr_feature,
+        encoding='utf-8',
         bbox=False,
-        encoding=encoding,
         driver=collection.driver
     )
     _deleteOgrFeature(cogr_feature)
@@ -453,7 +466,7 @@ cdef class Session:
         path_b = collection.path.encode('utf-8')
         path_c = path_b
 
-        userencoding = kwargs.get('encoding')
+        self._fileencoding = kwargs.get('encoding') or collection.encoding
 
         # We have two ways of specifying drivers to try. Resolve the
         # values into a single set of driver short names.
@@ -464,16 +477,18 @@ cdef class Session:
         else:
             drivers = None
 
+        encoding = kwargs.pop('encoding', None)
+        if encoding:
+            kwargs['encoding'] = encoding.upper()
+
         self.cogr_ds = gdal_open_vector(path_c, 0, drivers, kwargs)
 
         if isinstance(collection.name, string_types):
             name_b = collection.name.encode('utf-8')
             name_c = name_b
-            self.cogr_layer = GDALDatasetGetLayerByName(
-                                self.cogr_ds, name_c)
+            self.cogr_layer = GDALDatasetGetLayerByName(self.cogr_ds, name_c)
         elif isinstance(collection.name, int):
-            self.cogr_layer = GDALDatasetGetLayer(
-                                self.cogr_ds, collection.name)
+            self.cogr_layer = GDALDatasetGetLayer(self.cogr_ds, collection.name)
             name_c = OGR_L_GetName(self.cogr_layer)
             name_b = name_c
             collection.name = name_b.decode('utf-8')
@@ -481,21 +496,16 @@ cdef class Session:
         if self.cogr_layer == NULL:
             raise ValueError("Null layer: " + repr(collection.name))
 
-        self._fileencoding = userencoding or (
-            OGR_L_TestCapability(
-                self.cogr_layer, OLC_STRINGSASUTF8) and
-            'utf-8') or (
-            "Shapefile" in self.get_driver() and
-            'ISO-8859-1') or locale.getpreferredencoding().upper()
+        encoding = self._get_internal_encoding()
 
         if collection.ignore_fields:
             try:
                 for name in collection.ignore_fields:
                     try:
-                        name = name.encode(self._fileencoding)
+                        name_b = name.encode(encoding)
                     except AttributeError:
                         raise TypeError("Ignored field \"{}\" has type \"{}\", expected string".format(name, name.__class__.__name__))
-                    ignore_fields = CSLAddString(ignore_fields, <const char *>name)
+                    ignore_fields = CSLAddString(ignore_fields, <const char *>name_b)
                 OGR_L_SetIgnoredFields(self.cogr_layer, <const char**>ignore_fields)
             finally:
                 CSLDestroy(ignore_fields)
@@ -509,16 +519,54 @@ cdef class Session:
         self.cogr_ds = NULL
 
     def get_fileencoding(self):
+        """DEPRECATED"""
+        warnings.warn("get_fileencoding is deprecated and will be removed in a future version.", FionaDeprecationWarning)
         return self._fileencoding
 
-    def get_internalencoding(self):
-        if not self._encoding:
-            fileencoding = self.get_fileencoding()
-            self._encoding = (
-                OGR_L_TestCapability(
-                    self.cogr_layer, OLC_STRINGSASUTF8) and
-                'utf-8') or fileencoding
-        return self._encoding
+    def _get_fallback_encoding(self):
+        """Determine a format-specific fallback encoding to use when using OGR_F functions
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        str
+
+        """
+        if "Shapefile" in self.get_driver():
+            return 'iso-8859-1'
+        else:
+            return locale.getpreferredencoding()
+
+
+    def _get_internal_encoding(self):
+        """Determine the encoding to use when use OGR_F functions
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        str
+
+        Notes
+        -----
+        If the layer implements RFC 23 support for UTF-8, the return
+        value will be 'utf-8' and callers can be certain that this is
+        correct.  If the layer does not have the OLC_STRINGSASUTF8
+        capability marker, it is not possible to know exactly what the
+        internal encoding is and this method returns best guesses. That
+        means ISO-8859-1 for shapefiles and the locale's preferred
+        encoding for other formats such as CSV files.
+
+        """
+        if OGR_L_TestCapability(self.cogr_layer, OLC_STRINGSASUTF8):
+            return 'utf-8'
+        else:
+            return self._fileencoding or self._get_fallback_encoding()
 
     def get_length(self):
         if self.cogr_layer == NULL:
@@ -553,6 +601,8 @@ cdef class Session:
         if cogr_featuredefn == NULL:
             raise ValueError("Null feature definition")
 
+        encoding = self._get_internal_encoding()
+
         n = OGR_FD_GetFieldCount(cogr_featuredefn)
 
         for i from 0 <= i < n:
@@ -566,7 +616,7 @@ cdef class Session:
             if not bool(key_b):
                 raise ValueError("Invalid field name ref: %s" % key)
 
-            key = key_b.decode(self.get_internalencoding())
+            key = key_b.decode(encoding)
 
             if key in ignore_fields:
                 log.debug("By request, ignoring field %r", key)
@@ -771,8 +821,8 @@ cdef class Session:
         if cogr_feature != NULL:
             feature = FeatureBuilder().build(
                 cogr_feature,
+                encoding=self._get_internal_encoding(),
                 bbox=False,
-                encoding=self.get_internalencoding(),
                 driver=self.collection.driver,
                 ignore_fields=self.collection.ignore_fields,
                 ignore_geometry=self.collection.ignore_geometry,
@@ -806,8 +856,8 @@ cdef class Session:
                 return None
             feature = FeatureBuilder().build(
                 cogr_feature,
+                encoding=self._get_internal_encoding(),
                 bbox=False,
-                encoding=self.get_internalencoding(),
                 driver=self.collection.driver,
                 ignore_fields=self.collection.ignore_fields,
                 ignore_geometry=self.collection.ignore_geometry,
@@ -856,7 +906,7 @@ cdef class WritingSession(Session):
                 self.cogr_ds = gdal_open_vector(path_c, 1, None, kwargs)
 
                 if isinstance(collection.name, string_types):
-                    name_b = collection.name.encode()
+                    name_b = collection.name.encode('utf-8')
                     name_c = name_b
                     self.cogr_layer = exc_wrap_pointer(GDALDatasetGetLayerByName(self.cogr_ds, name_c))
 
@@ -870,11 +920,7 @@ cdef class WritingSession(Session):
                 raise DriverError(u"{}".format(exc))
 
             else:
-                self._fileencoding = (userencoding or (
-                    OGR_L_TestCapability(self.cogr_layer, OLC_STRINGSASUTF8) and
-                    'utf-8') or (
-                    self.get_driver() == "ESRI Shapefile" and
-                    'ISO-8859-1') or locale.getpreferredencoding()).upper()
+                self._fileencoding = userencoding or self._get_fallback_encoding()
 
         elif collection.mode == 'w':
 
@@ -975,19 +1021,16 @@ cdef class WritingSession(Session):
                 self.cogr_layer = NULL
                 raise CRSError(u"{}".format(exc))
 
-
-            # Figure out what encoding to use. The encoding parameter given
-            # to the collection constructor takes highest precedence, then
-            # 'iso-8859-1', then the system's default encoding as last resort.
+            # Determine which encoding to use. The encoding parameter given to
+            # the collection constructor takes highest precedence, then
+            # 'iso-8859-1' (for shapefiles), then the system's default encoding
+            # as last resort.
             sysencoding = locale.getpreferredencoding()
-            self._fileencoding = (userencoding or (
-                "Shapefile" in collection.driver and
-                'ISO-8859-1') or sysencoding).upper()
+            self._fileencoding = userencoding or ("Shapefile" in collection.driver and 'iso-8859-1') or sysencoding
 
             if "Shapefile" in collection.driver:
-                fileencoding = self.get_fileencoding()
-                if fileencoding:
-                    fileencoding_b = fileencoding.encode('utf-8')
+                if self._fileencoding:
+                    fileencoding_b = self._fileencoding.upper().encode('utf-8')
                     fileencoding_c = fileencoding_b
                     options = CSLSetNameValue(options, "ENCODING", fileencoding_c)
 
@@ -1017,6 +1060,9 @@ cdef class WritingSession(Session):
 
             for k, v in kwargs.items():
 
+                if v is None:
+                    continue
+
                 # We need to remove encoding from the layer creation
                 # options if we're not creating a shapefile.
                 if k == 'encoding' and "Shapefile" not in collection.driver:
@@ -1068,6 +1114,9 @@ cdef class WritingSession(Session):
 
             # Next, make a layer definition from the given schema properties,
             # which are an ordered dict since Fiona 1.0.1.
+
+            encoding = self._get_internal_encoding()
+
             for key, value in collection.schema['properties'].items():
 
                 log.debug("Begin creating field: %r value: %r", key, value)
@@ -1104,7 +1153,7 @@ cdef class WritingSession(Session):
                             value = 'int32'
 
                 field_type = FIELD_TYPES.index(value)
-                encoding = self.get_internalencoding()
+
                 try:
                     key_bytes = key.encode(encoding)
                     cogr_fielddefn = exc_wrap_pointer(OGR_Fld_Create(key_bytes, <OGRFieldType>field_type))
@@ -1116,6 +1165,7 @@ cdef class WritingSession(Session):
                         # subtypes are new in GDAL 2.x, ignored in 1.x
                         set_field_subtype(cogr_fielddefn, field_subtype)
                     exc_wrap_int(OGR_L_CreateField(self.cogr_layer, cogr_fielddefn, 1))
+
                 except (UnicodeEncodeError, CPLE_BaseError) as exc:
                     OGRReleaseDataSource(self.cogr_ds)
                     self.cogr_ds = NULL
@@ -1250,9 +1300,9 @@ cdef class Iterator:
             OGR_G_DestroyGeometry(cogr_geometry)
 
         else:
-            OGR_L_SetSpatialFilter(
-                cogr_layer, NULL)
-        self.encoding = session.get_internalencoding()
+            OGR_L_SetSpatialFilter(cogr_layer, NULL)
+
+        self.encoding = session._get_internal_encoding()
 
         self.fastindex = OGR_L_TestCapability(
             session.cogr_layer, OLC_FASTSETNEXTBYINDEX)
@@ -1352,8 +1402,8 @@ cdef class Iterator:
         try:
             return FeatureBuilder().build(
                 cogr_feature,
+                encoding=self.collection.session._get_internal_encoding(),
                 bbox=False,
-                encoding=self.encoding,
                 driver=self.collection.driver,
                 ignore_fields=self.collection.ignore_fields,
                 ignore_geometry=self.collection.ignore_geometry,
@@ -1379,12 +1429,11 @@ cdef class ItemsIterator(Iterator):
         if cogr_feature == NULL:
             raise StopIteration
 
-
         fid = OGR_F_GetFID(cogr_feature)
         feature = FeatureBuilder().build(
             cogr_feature,
+            encoding=self.collection.session._get_internal_encoding(),
             bbox=False,
-            encoding=self.encoding,
             driver=self.collection.driver,
             ignore_fields=self.collection.ignore_fields,
             ignore_geometry=self.collection.ignore_geometry,