.travis.yml

@@ -11,10 +11,9 @@ env:
     - GDALINST=$HOME/gdalinstall
     - GDALBUILD=$HOME/gdalbuild
   matrix:
-    - GDALVERSION="1.9.2"
     - GDALVERSION="1.11.5"
     - GDALVERSION="2.0.3"
-    - GDALVERSION="2.1.1"
+    - GDALVERSION="2.1.4"
 addons:
   apt:
     packages:

CHANGES.txt

@@ -3,6 +3,24 @@ Changes
 
 All issue numbers are relative to https://github.com/Toblerity/Fiona/issues.
 
+1.7.11 (2017-12-14)
+-------------------
+
+- The ``encoding`` keyword argument for ``fiona.open()``, which is intended
+  to allow a caller to override a data source's own and possibly erroneous
+  encoding, has not been working (#510, #512). The problem is that we weren't
+  always setting GDAL open or config options before opening the data sources.
+  This bug is resolved by a number of commits in the maint-1.7 branch and
+  the fix is demonstrated in tests/test_encoding.py.
+- An ``--encoding`` option has been added to fio-load to enable creation of
+  encoded shapefiles with an accompanying .cpg file (#499, #517).
+
+1.7.10.post1 (2017-10-30)
+-------------------------
+
+- A post-release has been made to fix a problem with macosx wheels uploaded
+  to PyPI.
+
 1.7.10 (2017-10-26)
 -------------------
 

CREDITS.txt

@@ -33,6 +33,7 @@ Fiona is written by:
 - dimlev <dimlev@gmail.com>
 - wilsaj <wilson.andrew.j+github@gmail.com>
 - Jesse Crocker <jesse@gaiagps.com>
+- Arne Schlüter
 
 Fiona would not be possible without the great work of Frank Warmerdam and other
 GDAL/OGR developers.

debian/changelog

-fiona (1.7.10-2) UNRELEASED; urgency=medium
+fiona (1.7.11-1) unstable; urgency=medium
 
   * Team upload.
+  * New upstream release.
   * Strip trailing whitespace from changelog, control & rules files.
   * Update copyright-format URL to use HTTPS.
   * Update Vcs-* URLs for Salsa.
   * Bump Standards-Version to 4.1.4, no changes.
   * Add module import tests to autopkgtest configuration.
+  * Add upstream metadata.
 
- -- Bas Couwenberg <sebastic@debian.org>  Wed, 15 Nov 2017 21:48:42 +0100
+ -- Bas Couwenberg <sebastic@debian.org>  Sat, 05 May 2018 20:58:47 +0200
 
 fiona (1.7.10-1) unstable; urgency=medium
 

debian/upstream/metadata

+---
+Bug-Database: https://github.com/Toblerity/Fiona/issues
+Bug-Submit: https://github.com/Toblerity/Fiona/issues/new
+Name: Fiona
+Repository: https://github.com/Toblerity/Fiona.git
+Repository-Browse: https://github.com/Toblerity/Fiona

fiona/__init__.py

@@ -81,7 +81,7 @@ import uuid
 
 
 __all__ = ['bounds', 'listlayers', 'open', 'prop_type', 'prop_width']
-__version__ = "1.7.10"
+__version__ = "1.7.11"
 __gdal_version__ = get_gdal_release_name().decode('utf-8')
 
 log = logging.getLogger(__name__)

fiona/fio/load.py

@@ -30,8 +30,11 @@ FIELD_TYPES_MAP_REV = dict([(v, k) for k, v in fiona.FIELD_TYPES_MAP.items()])
 @click.option('--layer', metavar="INDEX|NAME", callback=options.cb_layer,
               help="Load features into specified layer.  Layers use "
                    "zero-based numbering when accessed by index.")
+@click.option('--encoding',
+              help="Output encoding. Defaults to latin1 for "
+                   "backward compatibility.")
 @click.pass_context
-def load(ctx, output, driver, src_crs, dst_crs, sequence, layer):
+def load(ctx, output, driver, src_crs, dst_crs, sequence, layer, encoding):
     """Load features from JSON to a file in another format.
 
     The input is a GeoJSON feature collection or optionally a sequence of
@@ -98,7 +101,8 @@ def load(ctx, output, driver, src_crs, dst_crs, sequence, layer):
                     driver=driver,
                     crs=dst_crs,
                     schema=schema,
-                    layer=layer) as dst:
+                    layer=layer,
+                    encoding=encoding) as dst:
                 dst.write(first)
                 dst.writerecords(source)
 

fiona/ogrext1.pyx

@@ -396,6 +396,15 @@ cdef class Session:
             path_b = path
         path_c = path_b
 
+        userencoding = collection.encoding
+        if userencoding:
+            self._fileencoding = userencoding.upper()
+            val = self._fileencoding.encode('utf-8')
+            ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', val)
+            log.debug("SHAPE_ENCODING set to %r", val)
+        else:
+            ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', "")
+
         with cpl_errs:
             drivers = []
             if collection._driver:
@@ -438,13 +447,7 @@ cdef class Session:
         if self.cogr_layer == NULL:
             raise ValueError("Null layer: " + repr(collection.name))
 
-        self.collection = collection
-        
-        userencoding = self.collection.encoding
-        if userencoding:
-            ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', '')
-            self._fileencoding = userencoding.upper()
-        else:
+        if not userencoding:
             self._fileencoding = (
                 ogrext1.OGR_L_TestCapability(
                     self.cogr_layer, OLC_STRINGSASUTF8) and
@@ -452,6 +455,8 @@ cdef class Session:
                 self.get_driver() == "ESRI Shapefile" and
                 'ISO-8859-1') or locale.getpreferredencoding().upper()
 
+        self.collection = collection
+
     def stop(self):
         self.cogr_layer = NULL
         if self.cogr_ds != NULL:
@@ -697,19 +702,27 @@ cdef class WritingSession(Session):
     cdef object _schema_mapping
 
     def start(self, collection):
-        cdef void *cogr_fielddefn
-        cdef void *cogr_driver
+        cdef void *cogr_fielddefn = NULL
+        cdef void *cogr_driver = NULL
         cdef void *cogr_ds = NULL
         cdef void *cogr_layer = NULL
         cdef void *cogr_srs = NULL
         cdef char **options = NULL
-        self.collection = collection
-        cdef char *path_c
-        cdef char *driver_c
-        cdef char *name_c
-        cdef char *proj_c
-        cdef char *fileencoding_c
+        cdef char *path_c = NULL
+        cdef char *driver_c = NULL
+        cdef char *name_c = NULL
+        cdef char *proj_c = NULL
+        cdef char *fileencoding_c = NULL
+
         path = collection.path
+        self.collection = collection
+
+        userencoding = collection.encoding
+        if userencoding:
+            self._fileencoding = userencoding.upper()
+            val = self._fileencoding.encode('utf-8')
+            ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', val)
+            log.debug("SHAPE_ENCODING set to %r", val)
 
         if collection.mode == 'a':
             if os.path.exists(path):

fiona/ogrext2.pxd

@@ -60,11 +60,13 @@ cdef extern from "cpl_conv.h":
     void *  CPLMalloc (size_t)
     void    CPLFree (void *ptr)
     void    CPLSetThreadLocalConfigOption (char *key, char *val)
+    void    CPLSetConfigOption (char *key, char *val)
     const char *CPLGetConfigOption (char *, char *)
 
 
 cdef extern from "cpl_string.h":
     char ** CSLSetNameValue (char **list, const char *name, const char *value)
+    char ** CSLAddNameValue (char **list, const char *name, const char *value)
     void CSLDestroy (char **list)
     char ** CSLAddString(char **list, const char *string)
 

fiona/ogrext2.pyx

@@ -378,6 +378,7 @@ cdef class Session:
         cdef void *drv = NULL
         cdef void *ds = NULL
         cdef char **drvs = NULL
+        cdef char **open_opts = NULL
 
         if collection.path == '-':
             path = '/vsistdin/'
@@ -390,6 +391,15 @@ cdef class Session:
             path_b = path
         path_c = path_b
 
+        userencoding = collection.encoding
+        log.debug("Userencoding: %r", userencoding)
+
+        if userencoding:
+            self._fileencoding = userencoding.upper()
+            val = self._fileencoding.encode('utf-8')
+            open_opts = ogrext2.CSLAddNameValue(open_opts, "ENCODING", val)
+            log.debug("ENCODING open option set to %r", val)
+
         # TODO: eliminate this context manager in 2.0 as we have done
         # in Rasterio 1.0.
         with cpl_errs:
@@ -414,12 +424,15 @@ cdef class Session:
                     if drv != NULL:
                         drvs = ogrext2.CSLAddString(drvs, name_c)
 
+
             flags = ogrext2.GDAL_OF_VECTOR | ogrext2.GDAL_OF_READONLY
             try:
                 self.cogr_ds = ogrext2.GDALOpenEx(
-                    path_c, flags, <const char *const *>drvs, NULL, NULL)
+                    path_c, flags, <const char *const *>drvs, open_opts, NULL)
             finally:
                 ogrext2.CSLDestroy(drvs)
+                ogrext2.CSLDestroy(open_opts)
+                open_opts = NULL
 
         if self.cogr_ds == NULL:
             raise FionaValueError(
@@ -442,13 +455,7 @@ cdef class Session:
         if self.cogr_layer == NULL:
             raise ValueError("Null layer: " + repr(collection.name))
 
-        self.collection = collection
-        
-        userencoding = self.collection.encoding
-        if userencoding:
-            ogrext2.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', '')
-            self._fileencoding = userencoding.upper()
-        else:
+        if not userencoding:
             self._fileencoding = (
                 ogrext2.OGR_L_TestCapability(
                     self.cogr_layer, OLC_STRINGSASUTF8) and
@@ -456,6 +463,8 @@ cdef class Session:
                 self.get_driver() == "ESRI Shapefile" and
                 'ISO-8859-1') or locale.getpreferredencoding().upper()
 
+        self.collection = collection
+
     def stop(self):
         self.cogr_layer = NULL
         if self.cogr_ds != NULL:
@@ -709,13 +718,24 @@ cdef class WritingSession(Session):
         cdef void *cogr_layer = NULL
         cdef void *cogr_srs = NULL
         cdef char **options = NULL
-        self.collection = collection
         cdef const char *path_c = NULL
         cdef const char *driver_c = NULL
         cdef const char *name_c = NULL
         cdef const char *proj_c = NULL
         cdef const char *fileencoding_c = NULL
+        cdef char **open_opts = NULL
+
         path = collection.path
+        self.collection = collection
+
+        userencoding = collection.encoding
+        log.debug("Userencoding: %r", userencoding)
+
+        if userencoding:
+            self._fileencoding = userencoding.upper()
+            val = self._fileencoding.encode('utf-8')
+            open_opts = ogrext2.CSLAddNameValue(open_opts, "ENCODING", val)
+            log.debug("ENCODING open option set to %r", val)
 
         if collection.mode == 'a':
             if os.path.exists(path):
@@ -726,10 +746,11 @@ cdef class WritingSession(Session):
                 path_c = path_b
                 self.cogr_ds = ogrext2.GDALOpenEx(path_c,
                         ogrext2.GDAL_OF_VECTOR | ogrext2.GDAL_OF_UPDATE,
-                        NULL, NULL, NULL)
+                        NULL, open_opts, NULL)
 
                 cogr_driver = ogrext2.GDALGetDatasetDriver(self.cogr_ds)
                 if cogr_driver == NULL:
+                    ogrext2.CSLDestroy(open_opts)
                     raise ValueError("Null driver")
 
                 if isinstance(collection.name, string_types):
@@ -742,9 +763,11 @@ cdef class WritingSession(Session):
                                         self.cogr_ds, collection.name)
 
                 if self.cogr_layer == NULL:
+                    ogrext2.CSLDestroy(open_opts)
                     raise RuntimeError(
                         "Failed to get layer %s" % collection.name)
             else:
+                ogrext2.CSLDestroy(open_opts)
                 raise OSError("No such file or directory %s" % path)
 
             userencoding = self.collection.encoding
@@ -766,6 +789,7 @@ cdef class WritingSession(Session):
 
             cogr_driver = ogrext2.GDALGetDriverByName(driver_c)
             if cogr_driver == NULL:
+                ogrext2.CSLDestroy(open_opts)
                 raise ValueError("Null driver")
 
             # Our most common use case is the creation of a new data
@@ -790,7 +814,7 @@ cdef class WritingSession(Session):
                 cogr_ds = ogrext2.GDALOpenEx(path_c,
                                  ogrext2.GDAL_OF_VECTOR | ogrext2.GDAL_OF_UPDATE,
                                  NULL,
-                                 NULL,
+                                 open_opts,
                                  NULL)
 
                 # TODO: use exc_wrap_pointer()
@@ -820,6 +844,10 @@ cdef class WritingSession(Session):
                 else:
                     pass
 
+                if open_opts != NULL:
+                    ogrext2.CSLDestroy(open_opts)
+                    open_opts = NULL
+
             if cogr_ds == NULL:
                 raise RuntimeError("Failed to open %s" % path)
             else:

tests/data/gre.cpg

+UTF-8

tests/data/gre.prj

+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file

tests/test_encoding.py

+# coding=utf-8
+"""Encoding tests"""
+
+from glob import glob
+import logging
+import os
+import shutil
+import tempfile
+import unittest
+
+import pytest
+
+import fiona
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+# Uncomment when merged to master, where we're using pytest.
+# We're still holding our noses in maint-1.7.
+# @pytest.fixture(scope='function')
+# def gre_shp_cp1252():
+#     """A tempdir containing copies of gre.* files, .cpg set to cp1252
+#
+#     The shapefile attributes are in fact utf-8 encoded.
+#     """
+#     test_files = glob(os.path.join(os.path.dirname(__file__), 'data/gre.*'))
+#     tmpdir = pytest.ensuretemp('tests/data')
+#     for filename in test_files:
+#         shutil.copy(filename, str(tmpdir))
+#     tmpdir.join('gre.cpg').write('cp1252')
+#     return tmpdir
+
+class BadCodePointTest(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp(prefix='badcodepointtest')
+        test_files = glob(os.path.join(os.path.dirname(__file__), 'data/gre.*'))
+        for filename in test_files:
+            shutil.copy(filename, self.tempdir)
+        with open(os.path.join(self.tempdir, 'gre.cpg'), 'w') as cpg:
+            cpg.write('cp1252')
+        self.shapefile = os.path.join(self.tempdir, 'gre.shp')
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test_broken_encoding(self):
+        """Reading as cp1252 mis-encodes a Russian name"""
+        with fiona.open(self.shapefile) as src:
+            self.assertNotEqual(next(iter(src))['properties']['name_ru'], u'Гренада')
+
+    def test_override_encoding(self):
+        """utf-8 override succeeds"""
+        with fiona.open(self.shapefile, encoding='utf-8') as src:
+            self.assertEqual(next(iter(src))['properties']['name_ru'], u'Гренада')

tests/test_fio_ls.py

@@ -23,7 +23,7 @@ def test_fio_ls_single_layer():
         'tests/data/'])
     assert result.exit_code == 0
     assert len(result.output.splitlines()) == 1
-    assert json.loads(result.output) == ['coutwildrnp']
+    assert json.loads(result.output) == ['coutwildrnp', 'gre']
 
 
 @unittest.skipIf(FIXME_WINDOWS, 

tests/test_geopackage.py

@@ -33,7 +33,7 @@ class ReadingTest(unittest.TestCase):
             self.assertRaises(DriverError, fiona.open, 'tests/data/coutwildrnp.gpkg', 'r', driver="GPKG")
         else:
             with fiona.open('tests/data/coutwildrnp.gpkg', 'r', driver="GPKG") as c:
-                self.assertEquals(len(c), 48)
+                self.assertEquals(len(c), 67)
 
 
 class WritingTest(unittest.TestCase):

tests/test_listing.py

@@ -19,12 +19,12 @@ def test_single_file():
     assert fiona.listlayers('tests/data/coutwildrnp.shp') == ['coutwildrnp']
 
 def test_directory():
-    assert fiona.listlayers('tests/data') == ['coutwildrnp']
+    assert fiona.listlayers('tests/data') == ['coutwildrnp', 'gre']
 
 @unittest.skipIf(FIXME_WINDOWS,
                  reason="FIXME on Windows. ValueError raised. Please look into why this test isn't working.")
 def test_directory_trailing_slash():
-    assert fiona.listlayers('tests/data/') == ['coutwildrnp']
+    assert fiona.listlayers('tests/data/') == ['coutwildrnp', 'gre']
 
 def test_zip_path():
     assert fiona.listlayers('zip://tests/data/coutwildrnp.zip') == ['coutwildrnp']