.travis.yml

@@ -5,6 +5,7 @@ python:
   - "3.3"
   - "3.4"
   - "3.5"
+  - "3.6"
 
 services:
   - couchdb
@@ -32,19 +33,31 @@ env:
   global:
     - MAPPROXY_TEST_COUCHDB=http://127.0.0.1:5984
     - MAPPROXY_TEST_REDIS=127.0.0.1:6379
+    - MAPPROXY_TEST_RIAK_HTTP=http://localhost:8098
+    - MAPPROXY_TEST_RIAK_PBC=pbc://localhost:8087
 
     # do not load /etc/boto.cfg with Python 3 incompatible plugin
     # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
     - BOTO_CONFIG=/doesnotexist
 
+matrix:
+  # Test 2.7 and 3.6 also with latest Pillow version
+  include:
+    - python: "2.7"
+      env: USE_LATEST_PILLOW=1
+    - python: "3.6"
+      env: USE_LATEST_PILLOW=1
+
 cache:
   directories:
     - $HOME/.cache/pip
 
+before_install:
+    - echo -n "ulimit -n 4096" | sudo tee /etc/default/riak && sudo service riak restart	# default open file limit is too low for riak
+
 install:
-    # riak packages are not compatible with Python 3
-    - "if [[ $TRAVIS_PYTHON_VERSION = '2.7' ]]; then pip install protobuf>=2.4.1 riak==2.2 riak_pb>=2.0; export MAPPROXY_TEST_RIAK_PBC=pbc://localhost:8087; fi"
     - "pip install -r requirements-tests.txt"
+    - "if [[ $USE_LATEST_PILLOW = '1' ]]; then pip install -U Pillow; fi"
     - "pip freeze"
 
 script: nosetests mapproxy

AUTHORS.txt

@@ -25,7 +25,7 @@ Patches and Suggestions
 - Matt Walker
 - Miloslav Kmeť
 - Paul Norman
-- Ramūnas
+- Ramūnas Dronga
 - Richard Duivenvoorde
 - Stephan Holl
 - Steven D. Lander

CHANGES.txt

+1.11.0 2017-11-xx
+~~~~~~~~~~~~~~~~~
+
+Improvements:
+
+- Improve reprojection performance and accuracy.
+- ArcGIS compact cache: Support for version 2.
+- ArcGIS compact cache: Improve performance for version 1.
+- ArcGIS compact cache: Add ``mapproxy-util defrag`` to reduce bundle size
+  after tiles were removed/updated.
+- ArcGIS REST source: Support opts.map and seed_only.
+- Use systems CA certs by default and fix ssl_no_cert_checks
+  for Python >=2.7.9 and >=3.4
+- WMS: Improve Bounding Boxes in Capabilities.
+- Mapserver: Find mapserv binary in PATH environment.
+
+Fixes:
+
+- Seed: Always show last log line (100%).
+- Fix saving transparent PNGs for some versions of Pillow
+  (workaround for Pillow bug #2633)
+- SQLite: Fix possible errors on first request after start.
+- Demo: Fix demo client with `use_grid_names`.
+- serve-develop: Fix header encoding for Python 3.
+- Seed: Fix --interactive for Python 3.
+- Support tagged layers for sources with colons in name.
+- Support # character in Basis Authentication password.
+- Fix import error with shapely>=1.6
+- Fix duplicate level caches when using WMTS KVP with MBtile/SQLite/CouchDB.
+
+Other:
+
+- Remove support for Python 2.6
+
 1.10.4 2017-08-17
 ~~~~~~~~~~~~~~~~~
 

debian/changelog

+mapproxy (1.11.0-1~bpo9+1) stretch-backports; urgency=medium
+
+  * Rebuild for stretch-backports.
+
+ -- Bas Couwenberg <sebastic@debian.org>  Sun, 07 Jan 2018 11:28:12 +0100
+
+mapproxy (1.11.0-1) unstable; urgency=medium
+
+  * New upstream release.
+  * Drop libproj-dev from build dependencies, not directly required and
+    already pulled in via libgdal-dev.
+  * Refresh patches.
+  * Update copyright years for Omniscale.
+  * Bump Standards-Version to 4.1.1, no changes.
+
+ -- Bas Couwenberg <sebastic@debian.org>  Mon, 20 Nov 2017 17:07:46 +0100
+
 mapproxy (1.10.4-1~bpo9+1) stretch-backports; urgency=medium
 
   * Rebuild for stretch-backports.

debian/control

@@ -6,7 +6,6 @@ Priority: optional
 Build-Depends: debhelper (>= 9),
                dh-python,
                libgdal-dev,
-               libproj-dev,
                python-all,
                python-lxml,
                python-nose,
@@ -37,7 +36,7 @@ Build-Depends: debhelper (>= 9),
                docbook-xsl,
                docbook-xml,
                xsltproc
-Standards-Version: 4.0.0
+Standards-Version: 4.1.1
 Vcs-Browser: https://anonscm.debian.org/cgit/pkg-grass/mapproxy.git
 Vcs-Git: https://anonscm.debian.org/git/pkg-grass/mapproxy.git -b stretch-backports
 Homepage: http://mapproxy.org/

debian/copyright

@@ -4,7 +4,7 @@ Upstream-Contact: MapProxy Developers <mapproxy@lists.osgeo.org>
 Source: https://github.com/mapproxy/mapproxy
 
 Files: *
-Copyright: 2010-2016, Omniscale <http://omniscale.de>
+Copyright: 2010-2017, Omniscale <http://omniscale.de>
 License: Apache-2.0
 
 Files: mapproxy/image/fonts/*.ttf

debian/patches/offline-tests.patch

@@ -4,7 +4,7 @@ Forwarded: not-needed
 
 --- a/mapproxy/test/system/test_util_wms_capabilities.py
 +++ b/mapproxy/test/system/test_util_wms_capabilities.py
-@@ -17,6 +17,7 @@ from __future__ import with_statement
+@@ -16,6 +16,7 @@
  import os
  
  from nose.tools import assert_raises
@@ -12,7 +12,7 @@ Forwarded: not-needed
  
  from mapproxy.client.http import HTTPClient
  from mapproxy.script.wms_capabilities import wms_capabilities_command
-@@ -36,6 +37,9 @@ class TestUtilWMSCapabilities(object):
+@@ -35,6 +36,9 @@ class TestUtilWMSCapabilities(object):
          self.args = ['command_dummy', '--host', TESTSERVER_URL + '/service']
  
      def test_http_error(self):
@@ -22,7 +22,7 @@ Forwarded: not-needed
          self.args = ['command_dummy', '--host', 'http://foo.doesnotexist']
          with capture() as (out,err):
              assert_raises(SystemExit, wms_capabilities_command, self.args)
-@@ -47,6 +51,9 @@ class TestUtilWMSCapabilities(object):
+@@ -46,6 +50,9 @@ class TestUtilWMSCapabilities(object):
          assert err.getvalue().startswith("ERROR:")
  
      def test_request_not_parsable(self):
@@ -32,7 +32,7 @@ Forwarded: not-needed
          with mock_httpd(TESTSERVER_ADDRESS, [({'path': '/service?request=GetCapabilities&version=1.1.1&service=WMS', 'method': 'GET'},
                                                {'status': '200', 'body': ''})]):
              with capture() as (out,err):
-@@ -55,6 +62,9 @@ class TestUtilWMSCapabilities(object):
+@@ -54,6 +61,9 @@ class TestUtilWMSCapabilities(object):
              assert error_msg.startswith('Could not parse the document')
  
      def test_service_exception(self):
@@ -42,7 +42,7 @@ Forwarded: not-needed
          self.args = ['command_dummy', '--host', TESTSERVER_URL + '/service?request=GetCapabilities']
          with open(SERVICE_EXCEPTION_FILE, 'rb') as fp:
              capabilities_doc = fp.read()
-@@ -66,6 +76,9 @@ class TestUtilWMSCapabilities(object):
+@@ -65,6 +75,9 @@ class TestUtilWMSCapabilities(object):
                  assert 'Not a capabilities document' in error_msg
  
      def test_parse_capabilities(self):
@@ -52,7 +52,7 @@ Forwarded: not-needed
          self.args = ['command_dummy', '--host', TESTSERVER_URL + '/service?request=GetCapabilities', '--version', '1.1.1']
          with open(CAPABILITIES111_FILE, 'rb') as fp:
              capabilities_doc = fp.read()
-@@ -77,6 +90,9 @@ class TestUtilWMSCapabilities(object):
+@@ -76,6 +89,9 @@ class TestUtilWMSCapabilities(object):
                  assert lines[1].startswith('Capabilities Document Version 1.1.1')
  
      def test_parse_130capabilities(self):
@@ -62,7 +62,7 @@ Forwarded: not-needed
          self.args = ['command_dummy', '--host', TESTSERVER_URL + '/service?request=GetCapabilities', '--version', '1.3.0']
          with open(CAPABILITIES130_FILE, 'rb') as fp:
              capabilities_doc = fp.read()
-@@ -88,6 +104,9 @@ class TestUtilWMSCapabilities(object):
+@@ -87,6 +103,9 @@ class TestUtilWMSCapabilities(object):
                  assert lines[1].startswith('Capabilities Document Version 1.3.0')
  
      def test_key_error(self):
@@ -74,7 +74,7 @@ Forwarded: not-needed
              capabilities_doc = fp.read()
 --- a/mapproxy/test/unit/test_cache.py
 +++ b/mapproxy/test/unit/test_cache.py
-@@ -56,6 +56,7 @@ from mapproxy.test.http import assert_qu
+@@ -55,6 +55,7 @@ from mapproxy.test.http import assert_qu
  from collections import defaultdict
  
  from nose.tools import eq_, raises, assert_not_equal, assert_raises
@@ -82,7 +82,7 @@ Forwarded: not-needed
  
  TEST_SERVER_ADDRESS = ('127.0.0.1', 56413)
  GLOBAL_GEOGRAPHIC_EXTENT = MapExtent((-180, -90, 180, 90), SRS(4326))
-@@ -744,6 +745,9 @@ class TestWMSSourceWithClient(object):
+@@ -743,6 +744,9 @@ class TestWMSSourceWithClient(object):
          self.source = WMSSource(self.client)
  
      def test_get_map(self):
@@ -92,7 +92,7 @@ Forwarded: not-needed
          with tmp_image((512, 512)) as img:
              expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                                       '&REQUEST=GetMap&HEIGHT=512&SRS=EPSG%3A4326&styles='
-@@ -757,6 +761,9 @@ class TestWMSSourceWithClient(object):
+@@ -756,6 +760,9 @@ class TestWMSSourceWithClient(object):
                  assert is_png(result.as_buffer(seekable=True))
                  eq_(result.as_image().size, (512, 512))
      def test_get_map_non_image_content_type(self):
@@ -102,7 +102,7 @@ Forwarded: not-needed
          with tmp_image((512, 512)) as img:
              expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                                       '&REQUEST=GetMap&HEIGHT=512&SRS=EPSG%3A4326&styles='
-@@ -771,6 +778,9 @@ class TestWMSSourceWithClient(object):
+@@ -770,6 +777,9 @@ class TestWMSSourceWithClient(object):
                  else:
                      assert False, 'no SourceError raised'
      def test_basic_auth(self):
@@ -112,7 +112,7 @@ Forwarded: not-needed
          http_client = HTTPClient(self.req_template.url, username='foo', password='bar@')
          self.client.http_client = http_client
          def assert_auth(req_handler):
-@@ -937,4 +947,4 @@ class TestNeastedConditionalLayers(objec
+@@ -935,4 +945,4 @@ class TestNeastedConditionalLayers(objec
          assert self.l4326.requested
      def test_resolution_low_projected(self):
          self.layer.get_map(MapQuery((0, 0, 10000, 10000), (100, 100), SRS(31467)))
@@ -121,7 +121,7 @@ Forwarded: not-needed
 +        assert self.l900913.requested
 --- a/mapproxy/test/unit/test_client.py
 +++ b/mapproxy/test/unit/test_client.py
-@@ -43,11 +43,17 @@ class TestHTTPClient(object):
+@@ -42,11 +42,17 @@ class TestHTTPClient(object):
          self.client = HTTPClient()
  
      def test_post(self):
@@ -139,47 +139,96 @@ Forwarded: not-needed
          try:
              with mock_httpd(TESTSERVER_ADDRESS, [({'path': '/'},
                                                    {'status': '500', 'body': b''})]):
-@@ -87,6 +93,9 @@ class TestHTTPClient(object):
+@@ -56,6 +62,9 @@ class TestHTTPClient(object):
+         else:
+             assert False, 'expected HTTPClientError'
+     def test_invalid_url_type(self):
++        if 'OFFLINE_TESTS' in os.environ:
++            raise SkipTest
++
+         try:
+             self.client.open('htp://example.org')
+         except HTTPClientError as e:
+@@ -63,6 +72,9 @@ class TestHTTPClient(object):
+         else:
+             assert False, 'expected HTTPClientError'
+     def test_invalid_url(self):
++        if 'OFFLINE_TESTS' in os.environ:
++            raise SkipTest
++
+         try:
+             self.client.open('this is not a url')
+         except HTTPClientError as e:
+@@ -70,6 +82,9 @@ class TestHTTPClient(object):
+         else:
+             assert False, 'expected HTTPClientError'
+     def test_unknown_host(self):
++        if 'OFFLINE_TESTS' in os.environ:
++            raise SkipTest
++
+         try:
+             self.client.open('http://thishostshouldnotexist000136really42.org')
+         except HTTPClientError as e:
+@@ -77,6 +92,9 @@ class TestHTTPClient(object):
+         else:
+             assert False, 'expected HTTPClientError'
+     def test_no_connect(self):
++        if 'OFFLINE_TESTS' in os.environ:
++            raise SkipTest
++
+         try:
+             self.client.open('http://localhost:53871')
+         except HTTPClientError as e:
+@@ -86,6 +104,9 @@ class TestHTTPClient(object):
  
      @attr('online')
-     def test_https_no_ssl_module_error(self):
+     def test_https_untrusted_root(self):
 +        if 'OFFLINE_TESTS' in os.environ:
 +            raise SkipTest
 +
-         from mapproxy.client import http
-         old_ssl = http.ssl
-         try:
-@@ -102,6 +111,9 @@ class TestHTTPClient(object):
+         if not supports_ssl_default_context:
+             # old python versions require ssl_ca_certs
+             raise SkipTest()
+@@ -97,12 +118,18 @@ class TestHTTPClient(object):
  
      @attr('online')
-     def test_https_no_ssl_module_insecure(self):
+     def test_https_insecure(self):
 +        if 'OFFLINE_TESTS' in os.environ:
 +            raise SkipTest
 +
-         from mapproxy.client import http
-         old_ssl = http.ssl
-         try:
-@@ -113,6 +125,9 @@ class TestHTTPClient(object):
+         self.client = HTTPClient(
+             'https://untrusted-root.badssl.com/', insecure=True)
+         self.client.open('https://untrusted-root.badssl.com/')
  
      @attr('online')
-     def test_https_valid_cert(self):
+     def test_https_valid_ca_cert_file(self):
 +        if 'OFFLINE_TESTS' in os.environ:
 +            raise SkipTest
 +
-         try:
-             import ssl; ssl
-         except ImportError:
-@@ -131,6 +146,9 @@ class TestHTTPClient(object):
+         # verify with fixed ca_certs file
+         cert_file = '/etc/ssl/certs/ca-certificates.crt'
+         if os.path.exists(cert_file):
+@@ -117,6 +144,9 @@ class TestHTTPClient(object):
+ 
+     @attr('online')
+     def test_https_valid_default_cert(self):
++        if 'OFFLINE_TESTS' in os.environ:
++            raise SkipTest
++
+         # modern python should verify by default
+         if not supports_ssl_default_context:
+             raise SkipTest()
+@@ -125,6 +155,9 @@ class TestHTTPClient(object):
  
      @attr('online')
      def test_https_invalid_cert(self):
 +        if 'OFFLINE_TESTS' in os.environ:
 +            raise SkipTest
 +
-         try:
-             import ssl; ssl
-         except ImportError:
-@@ -144,6 +162,9 @@ class TestHTTPClient(object):
+         # load 'wrong' root cert
+         with TempFile() as tmp:
+             with open(tmp, 'wb') as f:
+@@ -137,6 +170,9 @@ class TestHTTPClient(object):
                  assert_re(e.args[0], r'Could not verify connection to URL')
  
      def test_timeouts(self):
@@ -189,7 +238,7 @@ Forwarded: not-needed
          test_req = ({'path': '/', 'req_assert_function': lambda x: time.sleep(0.9) or True},
                      {'body': b'nothing'})
  
-@@ -209,6 +230,9 @@ class TestTMSClient(object):
+@@ -240,6 +276,9 @@ class TestTMSClient(object):
      def setup(self):
          self.client = TMSClient(TESTSERVER_URL)
      def test_get_tile(self):
@@ -199,7 +248,7 @@ Forwarded: not-needed
          with mock_httpd(TESTSERVER_ADDRESS, [({'path': '/9/5/13.png'},
                                                  {'body': b'tile', 'headers': {'content-type': 'image/png'}})]):
              resp = self.client.get_tile((5, 13, 9)).source.read()
-@@ -216,6 +240,9 @@ class TestTMSClient(object):
+@@ -247,6 +286,9 @@ class TestTMSClient(object):
  
  class TestTileClient(object):
      def test_tc_path(self):
@@ -209,7 +258,7 @@ Forwarded: not-needed
          template = TileURLTemplate(TESTSERVER_URL + '/%(tc_path)s.png')
          client = TileClient(template)
          with mock_httpd(TESTSERVER_ADDRESS, [({'path': '/09/000/000/005/000/000/013.png'},
-@@ -225,6 +252,9 @@ class TestTileClient(object):
+@@ -256,6 +298,9 @@ class TestTileClient(object):
              eq_(resp, b'tile')
  
      def test_quadkey(self):
@@ -219,7 +268,7 @@ Forwarded: not-needed
          template = TileURLTemplate(TESTSERVER_URL + '/key=%(quadkey)s&format=%(format)s')
          client = TileClient(template)
          with mock_httpd(TESTSERVER_ADDRESS, [({'path': '/key=000002303&format=png'},
-@@ -233,6 +263,9 @@ class TestTileClient(object):
+@@ -264,6 +309,9 @@ class TestTileClient(object):
              resp = client.get_tile((5, 13, 9)).source.read()
              eq_(resp, b'tile')
      def test_xyz(self):
@@ -229,7 +278,7 @@ Forwarded: not-needed
          template = TileURLTemplate(TESTSERVER_URL + '/x=%(x)s&y=%(y)s&z=%(z)s&format=%(format)s')
          client = TileClient(template)
          with mock_httpd(TESTSERVER_ADDRESS, [({'path': '/x=5&y=13&z=9&format=png'},
-@@ -242,6 +275,9 @@ class TestTileClient(object):
+@@ -273,6 +321,9 @@ class TestTileClient(object):
              eq_(resp, b'tile')
  
      def test_arcgiscache_path(self):
@@ -239,7 +288,7 @@ Forwarded: not-needed
          template = TileURLTemplate(TESTSERVER_URL + '/%(arcgiscache_path)s.png')
          client = TileClient(template)
          with mock_httpd(TESTSERVER_ADDRESS, [({'path': '/L09/R0000000d/C00000005.png'},
-@@ -251,6 +287,9 @@ class TestTileClient(object):
+@@ -282,6 +333,9 @@ class TestTileClient(object):
              eq_(resp, b'tile')
  
      def test_bbox(self):
@@ -251,16 +300,16 @@ Forwarded: not-needed
          client = TileClient(template, grid=grid)
 --- a/mapproxy/test/unit/test_tiled_source.py
 +++ b/mapproxy/test/unit/test_tiled_source.py
-@@ -15,6 +15,8 @@
+@@ -14,6 +14,8 @@
+ # limitations under the License.
  
- from __future__ import with_statement
  
 +import os
 +
  from mapproxy.client.tile import TMSClient
  from mapproxy.grid import TileGrid
  from mapproxy.srs import SRS
-@@ -24,6 +26,7 @@ from mapproxy.layer import MapQuery
+@@ -23,6 +25,7 @@ from mapproxy.layer import MapQuery
  
  from mapproxy.test.http import mock_httpd
  from nose.tools import eq_
@@ -268,7 +317,7 @@ Forwarded: not-needed
  
  TEST_SERVER_ADDRESS = ('127.0.0.1', 56413)
  TESTSERVER_URL = 'http://%s:%d' % TEST_SERVER_ADDRESS
-@@ -34,6 +37,9 @@ class TestTileClientOnError(object):
+@@ -33,6 +36,9 @@ class TestTileClientOnError(object):
          self.client = TMSClient(TESTSERVER_URL)
  
      def test_cacheable_response(self):
@@ -278,7 +327,7 @@ Forwarded: not-needed
          error_handler = HTTPSourceErrorHandler()
          error_handler.add_handler(500, (255, 0, 0), cacheable=True)
          self.source = TiledSource(self.grid, self.client, error_handler=error_handler)
-@@ -45,6 +51,9 @@ class TestTileClientOnError(object):
+@@ -44,6 +50,9 @@ class TestTileClientOnError(object):
              eq_(resp.as_image().getcolors(), [((256*256), (255, 0, 0))])
  
      def test_image_response(self):
@@ -288,7 +337,7 @@ Forwarded: not-needed
          error_handler = HTTPSourceErrorHandler()
          error_handler.add_handler(500, (255, 0, 0), cacheable=False)
          self.source = TiledSource(self.grid, self.client, error_handler=error_handler)
-@@ -56,6 +65,9 @@ class TestTileClientOnError(object):
+@@ -55,6 +64,9 @@ class TestTileClientOnError(object):
              eq_(resp.as_image().getcolors(), [((256*256), (255, 0, 0))])
  
      def test_multiple_image_responses(self):
@@ -300,7 +349,7 @@ Forwarded: not-needed
          error_handler.add_handler(204, (255, 0, 127, 200), cacheable=True)
 --- a/mapproxy/test/system/test_seed.py
 +++ b/mapproxy/test/system/test_seed.py
-@@ -32,6 +32,7 @@ from mapproxy.test.http import mock_http
+@@ -31,6 +31,7 @@ from mapproxy.test.http import mock_http
  from mapproxy.test.image import tmp_image, create_tmp_image_buf, create_tmp_image
  
  from nose.tools import eq_
@@ -308,7 +357,7 @@ Forwarded: not-needed
  
  FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
  
-@@ -357,6 +358,9 @@ class TestConcurrentRequestsSeed(SeedTes
+@@ -356,6 +357,9 @@ class TestConcurrentRequestsSeed(SeedTes
      empty_ogrdata = 'empty_ogrdata.geojson'
  
      def test_timeout(self):

doc/caches.rst

@@ -286,7 +286,7 @@ This backend is good for very large caches which can be distributed over many no
 Requirements
 ------------
 
-You will need the `Python Riak client <https://pypi.python.org/pypi/riak>`_ version 2.0 or newer. You can install it in the usual way, for example with ``pip install riak``. Environments with older version must be upgraded with ``pip install -U riak``.
+You will need the `Python Riak client <https://pypi.python.org/pypi/riak>`_ version 2.4.2 or older. You can install it in the usual way, for example with ``pip install riak==2.4.2``. Environments with older version must be upgraded with ``pip install -U riak==2.4.2``. Python library depends on packages `python-dev`, `libffi-dev` and `libssl-dev`.
 
 Configuration
 -------------
@@ -294,13 +294,13 @@ Configuration
 Available options:
 
 ``nodes``:
-    A list of riak nodes. Each node needs a ``host`` and optionally a ``pb_port`` and an ``http_port`` if the ports differ from the default. A single localhost node is used if you don't configure any nodes.
+    A list of riak nodes. Each node needs a ``host`` and optionally a ``pb_port`` and an ``http_port`` if the ports differ from the default. Defaults to single localhost node.
 
 ``protocol``:
     Communication protocol. Allowed options is ``http``, ``https`` and ``pbc``. Defaults to ``pbc``.
 
 ``bucket``:
-    The name of the bucket MapProxy uses for this cache. The bucket is the namespace for the tiles and needs to be unique for each cache. Defaults to cache name suffixed with grid name (e.g. ``mycache_webmercator``).
+    The name of the bucket MapProxy uses for this cache. The bucket is the namespace for the tiles and must be unique for each cache. Defaults to cache name suffixed with grid name (e.g. ``mycache_webmercator``).
 
 ``default_ports``:
     Default ``pb`` and ``http`` ports for ``pbc`` and ``http`` protocols. Will be used as the default for each defined node.
@@ -316,6 +316,7 @@ Example
   myriakcache:
     sources: [mywms]
     grids: [mygrid]
+    cache:
       type: riak
       nodes:
         - host: 1.example.org
@@ -488,10 +489,16 @@ Example
 ===========
 
 .. versionadded:: 1.10.0
+  Support for format version 1
+
+.. versionadded:: 1.11.0
+  Support for format version 2
 
-Store tiles in ArcGIS compatible compact cache files. A single compact cache ``.bundle`` file stores up to about 16,000 tiles. There is one additional ``.bundlx`` index file for each ``.bundle`` data file.
+Store tiles in ArcGIS compatible compact cache files. A single compact cache ``.bundle`` file stores up to about 16,000 tiles.
+
+Version 1 of the compact cache format is compatible with ArcGIS 10.0 and the default version of ArcGIS 10.0-10.2. Version 2 is supported by ArcGIS 10.3 or higher.
+Version 1 stores is one additional ``.bundlx`` index file for each ``.bundle`` data file.
 
-Only version 1 of the compact cache format (ArcGIS 10.0-10.2) is supported. Version 2 (ArcGIS 10.3 or higher) is not supported at the moment.
 
 Available options:
 
@@ -499,7 +506,7 @@ Available options:
   Directory where MapProxy should store the level directories. This will not add the cache name or grid name to the path. You can use this option to point MapProxy to an existing compact cache.
 
 ``version``:
-  The version of the ArcGIS compact cache format. This option is required.
+  The version of the ArcGIS compact cache format. This option is required. Either ``1`` or ``2``.
 
 
 You can set the ``sources`` to an empty list, if you use an existing compact cache files and do not have a source.
@@ -515,9 +522,14 @@ The following configuration will load tiles from ``/path/to/cache/L00/R0000C0000
       grids: [webmercator]
       cache:
         type: compact
-        version: 1
+        version: 2
         directory: /path/to/cache
 
+.. note::
+
+  MapProxy does not support reading and writiting of the ``conf.cdi`` and ``conf.xml`` files. You need to configure a compatible MapProxy grid when you want to reuse exsting ArcGIS compact caches in MapProxy. You need to create or modify existing ``conf.cdi`` and ``conf.xml`` files when you want to use compact caches created with MapProxy in ArcGIS.
+
+
 .. note::
 
   The compact cache format does not include any timestamps for each tile and the seeding function is limited therefore. If you include any ``refresh_before`` time in a seed task, all tiles will be recreated regardless of the value. The cleanup process does not support any ``remove_before`` times for compact caches and it always removes all tiles.
@@ -526,5 +538,6 @@ The following configuration will load tiles from ``/path/to/cache/L00/R0000C0000
 
 .. note::
 
-  The compact cache format is append-only to allow parallel read and write operations. Removing or refreshing tiles with ``mapproxy-seed`` does not reduce the size of the cache files. Therefore, this format is not suitable for caches that require frequent updates.
-
+  The compact cache format is append-only to allow parallel read and write operations.
+  Removing or refreshing tiles with ``mapproxy-seed`` does not reduce the size of the cache files.
+  You can use the :ref:`defrag-compact-cache <mapproxy_defrag_compact_cache>` util to reduce the file size of existing bundle files.

doc/conf.py

@@ -49,9 +49,9 @@ copyright = u'Oliver Tonnhofer, Omniscale'
 # built documents.
 #
 # The short X.Y version.
-version = '1.10'
+version = '1.11'
 # The full version, including alpha/beta/rc tags.
-release = '1.10.4'
+release = '1.11.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/configuration.rst

@@ -127,7 +127,7 @@ The old syntax to configure each layer as a dictionary with the key as the name
   layers:
     mylayer:
       title: My Layer
-      source: [mysoruce]
+      source: [mysource]
 
 should become
 
@@ -136,7 +136,7 @@ should become
   layers:
     - name: mylayer
       title: My Layer
-      source: [mysoruce]
+      source: [mysource]
 
 The mixed format where the layers are a list (``-``) but each layer is still a dictionary is no longer supported (e.g. ``- mylayer:`` becomes ``- name: mylayer``).
 
@@ -614,7 +614,7 @@ Requests with 1500, 1000 or 701m/px resolution will use the first level, request
 
 The extent of your grid. You can use either a list or a string with the lower left and upper right coordinates. You can set the SRS of the coordinates with the ``bbox_srs`` option. If that option is not set the ``srs`` of the grid will be used.
 
-MapProxy always expects your BBOX coordinates order to be east, south, west, north, regardless of your SRS :ref:`axis order <axis_order>`.
+MapProxy always expects your BBOX coordinates order to be west, south, east, north regardless of your SRS :ref:`axis order <axis_order>`.
 
 ::
 
@@ -900,14 +900,13 @@ The following options define how tiles are created and stored. Most options can
 
 HTTP related options.
 
-Secure HTTPS Connections (HTTPS)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. note:: You need Python 2.6 or the `SSL module <http://pypi.python.org/pypi/ssl>`_ for this feature.
+Secure HTTP Connections (HTTPS)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 MapProxy supports access to HTTPS servers. Just use ``https`` instead of ``http`` when
-defining the URL of a source. MapProxy needs a file that contains the root and CA
-certificates. If the server certificate is signed by a "standard" root certificate (i.e. your browser does not warn you), then you can use a cert file that is distributed with your system. On Debian based systems you can use ``/etc/ssl/certs/ca-certificates.crt``.
+defining the URL of a source.
+
+MapProxy verifies the SSL/TLS connections against your systems "certification authority" (CA) certificates. You can provide your own set of root certificates with the ``ssl_ca_certs`` option.
 See the `Python SSL documentation <http://docs.python.org/dev/library/ssl.html#ssl-certificates>`_ for more information about the format.
 
 ::
@@ -915,11 +914,26 @@ See the `Python SSL documentation <http://docs.python.org/dev/library/ssl.html#s
   http:
     ssl_ca_certs: /etc/ssl/certs/ca-certificates.crt
 
-If you want to use SSL but do not need certificate verification, then you can disable it with the ``ssl_no_cert_checks`` option. You can also disable this check on a source level, see :ref:`WMS source options <wms_source_ssl_no_cert_checks>`.
+
+.. versionadded:: 1.11.0
+
+  MapProxy uses the systems CA files by default, if you use Python >=2.7.9 or >=3.4.
+
+
+.. note::
+
+  You need to supply a CA file that includes the root certificates if you use older MapProxy or older Python versions. Otherwise MapProxy will fail to establish the connection. You can set the ``http.ssl_no_cert_checks`` options to ``true`` to disable this verification.
+
+
+``ssl_no_cert_checks``
+
+If you want to use SSL/TLS but do not need certificate verification, then you can disable it with the ``ssl_no_cert_checks`` option. You can also disable this check on a source level.
+
 ::
 
   http:
-    ssl_no_cert_checks: True
+    ssl_no_cert_checks: true
+
 
 ``client_timeout``
 ^^^^^^^^^^^^^^^^^^

doc/deployment.rst

@@ -145,7 +145,7 @@ You need a server script that creates the MapProxy application (see :ref:`above
 To start MapProxy with the Gunicorn web server with four processes, the eventlet worker and our server script (without ``.py``)::
 
   cd /path/of/config.py/
-  gunicorn -k eventlet -w 4 -b :8080 config:application
+  gunicorn -k eventlet -w 4 -b :8080 config:application --no-sendfile
 
 
 An example upstart script (``/etc/init/mapproxy.conf``) might look like::
@@ -160,7 +160,9 @@ An example upstart script (``/etc/init/mapproxy.conf``) might look like::
 
     chdir /etc/opt/mapproxy
 
-    exec /opt/mapproxy/bin/gunicorn -k eventlet -w 8 -b :8080 application \
+    exec /opt/mapproxy/bin/gunicorn -k eventlet -w 8 -b :8080 \
+        --no-sendfile \
+        application \
         >>/var/log/mapproxy/gunicorn.log 2>&1
 
 

doc/install_osgeo4w.rst

@@ -18,22 +18,19 @@ Please refer to the `OSGeo4W installer FAQ <http://trac.osgeo.org/osgeo4w/wiki/F
 
 At this point, you should see an OSGeo4W shell icon on your desktop and/or start menu. Right-click that, and *run as administrator*.
 
-As happens with the standard Windows installation, you need to `install the distribute package <http://pypi.python.org/pypi/distribute#distribute-setup-py>`_ to get the ``easy_install`` command. Run this in your administrator OSGeo4W shell, e.g.::
+In the OSGeo4W window, run::
 
- C:\OSGeo4W> python C:\Users\MyUsername\Downloads\distribute-setup.py
-
-Once ``easy_install`` is working within the OSGeo4W python environment, run::
-
- C:\OSGeo4W> easy_install mapproxy
+ C:\OSGeo4W> pip install mapproxy
 
 and
 
 ::
 
- C:\OSGeo4W> easy_install pyproj
+ C:\OSGeo4W> pip install pyproj
 
-If these three last commands didn't print out any errors, your installation of MapProxy is successful. You can now close the OSGeo4W shell with administrator privileges, as it is no longer needed.
+If these last two commands didn't print out any errors, your installation of MapProxy is successful. You can now close the OSGeo4W shell with administrator privileges, as it is no longer needed.
 
+In older versions of OSGeo4W  ``pip`` may not recognized. In such a case, please follow the instructions for `installing pip with get-pip.py <https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py>`_ and rerty the above ``pip install`` commands.
 
 Check installation
 ------------------

doc/mapproxy_util.rst

@@ -31,6 +31,7 @@ The current sub-commands are:
 - :ref:`mapproxy_util_wms_capabilities`
 - :ref:`mapproxy_util_grids`
 - :ref:`mapproxy_util_export`
+- :ref:`mapproxy_defrag_compact_cache`
 - ``autoconfig`` (see :ref:`mapproxy_util_autoconfig`)
 
 
@@ -54,7 +55,7 @@ This sub-command creates example configurations for you. There are templates for
 
 .. cmdoption:: -f <mapproxy.yaml>, --mapproxy-conf <mapproxy.yaml>
 
-  The path to the MapProxy configuration. Required for some templates.
+  The path of the MapProxy configuration. Required for some templates.
 
 .. cmdoption:: --force
 
@@ -457,7 +458,7 @@ Required arguments:
 
 .. cmdoption:: -f, --mapproxy-conf
 
-  The path to the MapProxy configuration of the source cache.
+  The path of the MapProxy configuration of the source cache.
 
 .. cmdoption:: --source
 
@@ -547,3 +548,49 @@ Export tiles into an MBTiles file using a custom grid definition.
         --grid "srs='EPSG:4326' bbox=[5,50,10,60] tile_size=[512,512]" \
         --source osm_cache --dest osm.mbtiles --type mbtile \
 
+
+
+.. _mapproxy_defrag_compact_cache:
+
+``defrag-compact-cache``
+========================
+
+
+The ArcGIS compact cache format version 1 and 2 are append only. Updating existing tiles will increase the file size. Bundle files become larger and fragmented with time. The ``defrag-compact-cache`` sub-command compacts existing bundle files by rewriting and reorganizing each bundle file.
+
+
+.. program:: mapproxy-util defrag-compact-cache
+
+
+Required arguments:
+
+.. cmdoption:: -f, --mapproxy-conf
+
+  The path of the MapProxy configuration with the configured compact caches.
+
+Optional arguments:
+
+.. cmdoption:: --caches
+
+  Comma separated list of caches to defragment. By default all configured compact caches will be defragmented.
+
+.. cmdoption:: --min-percent, --min-mb
+
+  Bundle files with only a minmal fragmentation are skipped. You can define this threshold with ``--min-percent`` as the required minimal percentage of unused space and ``--min-mb`` as the minimal required unused space in megabytes. Both thresholds must be exceeded. Defaults to 10% and 1MB.
+
+.. option:: -n, --dry-run
+
+  This will simulate the defragmentation process.
+
+
+Examples
+--------
+
+Defragment bundle files from ``map1_cache`` and ``map2_cache`` when they have more than 20% and 5MB of unused space. E.g. a 20 MB bundle file only gets rewritten if it becomes smaller then 15MB after defragmentation; a 500MB bundle file only gets rewritten if it becomes smaller then 400MB after defragmentation.
+
+::
+
+  mapproxy-util defrag-compact-cache -f mapproxy.yaml \
+    --min-percent 20 \
+    --min-mb 5 \
+    --caches map1_cache,map2_cache

doc/sources.rst

@@ -178,16 +178,10 @@ You can configure the following HTTP related options for this source:
 - ``headers``
 - ``client_timeout``
 - ``ssl_ca_certs``
-- ``ssl_no_cert_checks`` (see below)
+- ``ssl_no_cert_checks``
 
 See :ref:`HTTP Options <http_ssl>` for detailed documentation.
 
-.. _wms_source_ssl_no_cert_checks:
-
-``ssl_no_cert_checks``
-
-  MapProxy checks the SSL server certificates for any ``req.url`` that use HTTPS. You need to supply a file (see) that includes that certificate, otherwise MapProxy will fail to establish the connection. You can set the ``http.ssl_no_cert_checks`` options to ``true`` to disable this verification.
-
 .. _tagged_source_names:
 
 Tagged source names
@@ -274,7 +268,10 @@ This describes the ArcGIS source. The only required option is ``url``. You need
 ``opts``
 ^^^^^^^^
 
-.. versionadded: 1.10.0
+.. versionadded:: 1.10.0
+.. versionadded:: 1.11.0
+  ``map`` option
+
 
 This option affects what request MapProxy sends to the source ArcGIS server.
 
@@ -287,6 +284,19 @@ This option affects what request MapProxy sends to the source ArcGIS server.
 ``featureinfo_tolerance``
   Tolerance in pixel within the ArcGIS server should identify features.
 
+
+``map``
+  If this is set to ``false``, MapProxy will not request images from this source. You can use this option in combination with ``featureinfo: true`` to create a source that is only used for feature info requests.
+
+
+
+``seed_only``
+^^^^^^^^^^^^^
+
+.. versionadded:: 1.11.0
+
+See :ref:`seed_only <wms_seed_only>`
+
 Example configuration
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -375,7 +385,7 @@ You can configure the following HTTP related options for this source:
 - ``headers``
 - ``client_timeout``
 - ``ssl_ca_certs``
-- ``ssl_no_cert_checks`` (:ref:`see above <wms_source_ssl_no_cert_checks>`)
+- ``ssl_no_cert_checks``
 
 See :ref:`HTTP Options <http_ssl>` for detailed documentation.
 
@@ -469,6 +479,9 @@ You can also set these options in the :ref:`globals-conf-label` section.
   Path where the Mapserver should be executed from. It should be the directory where any relative paths in your mapfile are based on.
 
 
+.. versionadded:: 1.11.0
+  The ``mapserv`` binary is searched in all directories of the ``PATH`` environment, if ``binary`` is not set.
+
 Example configuration
 ^^^^^^^^^^^^^^^^^^^^^
 

mapproxy/__init__.py

-__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file

mapproxy/cache/couchdb.py

@@ -13,10 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import with_statement
 
 import codecs
 import datetime
+import json
 import socket
 import time
 import hashlib
@@ -37,13 +37,6 @@ try:
 except ImportError:
     requests = None
 
-try:
-    import simplejson as json
-except ImportError:
-    try:
-        import json
-    except ImportError:
-        json = None
 
 import logging
 log = logging.getLogger(__name__)
@@ -59,9 +52,6 @@ class CouchDBCache(TileCacheBase):
         if requests is None:
             raise ImportError("CouchDB backend requires 'requests' package.")
 
-        if json is None:
-            raise ImportError("CouchDB backend requires 'simplejson' package or Python 2.6+.")
-
         self.lock_cache_id = 'couchdb-' + hashlib.md5((url + db_name).encode('utf-8')).hexdigest()
         self.file_ext = file_ext
         self.tile_grid = tile_grid

mapproxy/cache/file.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import with_statement
 import os
 import errno
 import hashlib
@@ -164,11 +163,8 @@ class FileCache(TileCacheBase):
         if os.path.exists(tile_loc) or os.path.islink(tile_loc):
             os.unlink(tile_loc)
 
-        # Use relative path for the symlink if os.path.relpath is available
-        # (only supported with >= Python 2.6)
-        if hasattr(os.path, 'relpath'):
-            real_tile_loc = os.path.relpath(real_tile_loc,
-                                            os.path.dirname(tile_loc))
+        # Use relative path for the symlink
+        real_tile_loc = os.path.relpath(real_tile_loc, os.path.dirname(tile_loc))
 
         try:
             os.symlink(real_tile_loc, tile_loc)

mapproxy/cache/geopackage.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import with_statement
 
 import hashlib
 import logging
@@ -274,9 +273,10 @@ class GeopackageCache(TileCacheBase):
                         """
 PROJCS["WGS 84 / Pseudo-Mercator",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,\
 AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],\
-UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","9122"]]AUTHORITY["EPSG","4326"]],\
+UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],\
 PROJECTION["Mercator_1SP"],PARAMETER["central_meridian",0],PARAMETER["scale_factor",1],PARAMETER["false_easting",0],\
-PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH]\
+PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH],\
+AUTHORITY["EPSG","3857"]]\
                         """
                         ),
                        (4326, 'epsg', 4326, 'WGS 84',

mapproxy/cache/legend.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import with_statement
 
 import os
 import hashlib