.travis.yml

@@ -13,6 +13,7 @@ addons:
     - libproj-dev
     - lua5.2
     - liblua5.2-dev
+    - libluajit-5.1-dev
     - libboost1.55-dev
     - libboost-system1.55-dev
     - libboost-filesystem1.55-dev
@@ -20,13 +21,19 @@ matrix:
   include:
     - os: linux
       compiler: clang
-      env: CXXFLAGS="-pedantic -Werror"
+      env: CXXFLAGS="-pedantic -Werror" LUAJIT_OPTION="OFF"
     - os: linux
       compiler: gcc
-      env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror -fsanitize=address"
+      env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror -fsanitize=address" LUAJIT_OPTION="OFF"
+    - os: linux
+      compiler: clang
+      env: CXXFLAGS="-pedantic -Werror" LUAJIT_OPTION="ON"
+    - os: linux
+      compiler: gcc
+      env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror -fsanitize=address" LUAJIT_OPTION="ON"
     - os: osx
       compiler: clang
-      env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror -fsanitize=address"
+      env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror -fsanitize=address" LUAJIT_OPTION="OFF"
 before_install:
   - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
       brew install lua;
@@ -46,7 +53,7 @@ before_script:
   - lua -v
 script:
   - mkdir build && cd build
-  - cmake .. -DBUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Debug
+  - cmake .. -DBUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DWITH_LUAJIT=$LUAJIT_OPTION
   - make -j2
   - echo "Running tests that does not require PostgreSQL server"
   - if [[ $RUNTEST ]]; then ctest -VV $RUNTEST; fi

CMakeLists.txt

 set(PACKAGE osm2pgsql)
 set(PACKAGE_NAME osm2pgsql)
-set(PACKAGE_VERSION 0.94.0)
+set(PACKAGE_VERSION 0.96.0)
 
 cmake_minimum_required(VERSION 2.8.7)
 
@@ -8,10 +8,15 @@ project(osm2pgsql)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
-set(DATA_DIR \"${CMAKE_INSTALL_PREFIX}/share/osm2pgsql\")
+if (WIN32)
+    set(DEFAULT_STYLE "default.style" CACHE STRING "Default style used unless one is given on the command line")
+else()
+    set(DEFAULT_STYLE "${CMAKE_INSTALL_PREFIX}/share/osm2pgsql/default.style" CACHE STRING "Default style used unless one is given on the command line")
+endif()
 
 option(BUILD_TESTS "Build test suite" OFF)
-option(WITH_LUA    "Build with lua support" ON)
+option(WITH_LUA    "Build with Lua support" ON)
+option(WITH_LUAJIT "Build with LuaJIT support" OFF)
 
 if (NOT TESTING_TIMEOUT)
   set(TESTING_TIMEOUT 1200)
@@ -51,6 +56,7 @@ else()
 endif()
 
 option(EXTERNAL_LIBOSMIUM "Do not use the bundled libosmium" OFF)
+option(EXTERNAL_PROTOZERO "Do not use the bundled protozero" OFF)
 
 #############################################################
 # Detect available headers and set global compiler options
@@ -60,7 +66,7 @@ include (CheckIncludeFiles)
 include (CheckFunctionExists)
 include (CheckTypeSize)
 
-add_definitions( -DOSM2PGSQL_DATADIR=${DATA_DIR} )
+add_definitions( -DDEFAULT_STYLE=\"${DEFAULT_STYLE}\" )
 add_definitions( -DFIXED_POINT )
 
 CHECK_INCLUDE_FILES (termios.h HAVE_TERMIOS_H)
@@ -79,14 +85,24 @@ if (NOT EXTERNAL_LIBOSMIUM)
   set(OSMIUM_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/contrib/libosmium")
 endif()
 
+if (NOT EXTERNAL_PROTOZERO)
+  set(PROTOZERO_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/contrib/protozero/include")
+endif()
+
 include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
-find_package(Osmium REQUIRED COMPONENTS io proj)
-include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS})
+find_package(Osmium 2.14 REQUIRED COMPONENTS io proj)
+include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS} ${PROTOZERO_INCLUDE_DIR})
 
 if (WITH_LUA)
+  if (WITH_LUAJIT)
+    find_package(LuaJIT REQUIRED)
+    include_directories(${LUAJIT_INCLUDE_DIR})
+    set(HAVE_LUAJIT 1)
+  else()
     find_package(Lua REQUIRED)
     include_directories(${LUA_INCLUDE_DIR})
+  endif()
   set(HAVE_LUA 1)
 endif()
 
@@ -108,7 +124,9 @@ find_package(Threads)
 
 set (LIBS ${Boost_LIBRARIES} ${PostgreSQL_LIBRARY} ${OSMIUM_LIBRARIES})
 
-if (LUA_FOUND)
+if (LUAJIT_FOUND)
+  list(APPEND LIBS ${LUAJIT_LIBRARIES})
+elseif(LUA_FOUND)
   list(APPEND LIBS ${LUA_LIBRARIES})
 endif()
 
@@ -212,7 +230,7 @@ set(osm2pgsql_lib_SOURCES
   wkb.hpp
 )
 
-if (LUA_FOUND)
+if (LUA_FOUND OR LUAJIT_FOUND)
   list(APPEND osm2pgsql_lib_SOURCES tagtransform-lua.cpp)
 endif()
 

CONTRIBUTING.md

@@ -68,7 +68,13 @@ actively tested on Debian, Ubuntu and FreeBSD by the maintainers.
 ## Testing
 
 The code also comes with a suite of tests which can be run by
-executing ``make check``.
+executing ``ctest``.
+
+Regression tests require python and psycopg to be installed. On Ubuntu run:
+
+```sh
+sudo apt-get install python-psycopg2
+```
 
 Most of these tests depend on being able to set up a database and run osm2pgsql
 against it. You need to ensure that PostgreSQL is running and that your user is

README.md

@@ -20,8 +20,11 @@ Nominatim, or general analysis.
 
 Most Linux distributions include osm2pgsql. It is also available on macOS with [Homebrew](http://brew.sh/).
 
-Unoffical builds for Windows are available from [AppVeyor](https://ci.appveyor.com/project/openstreetmap/osm2pgsql/history) but you need to find the right build artifacts. The latest
-release is [0.92.0](https://ci.appveyor.com/api/projects/openstreetmap/osm2pgsql/artifacts/osm2pgsql_Release.zip?tag=0.92.0).
+Unoffical builds for Windows are available from [AppVeyor](https://ci.appveyor.com/project/openstreetmap/osm2pgsql/history) but you need to find the right build artifacts.
+For the latest release 0.96.0, you may download a
+[32bit version](https://ci.appveyor.com/api/projects/openstreetmap/osm2pgsql/artifacts/osm2pgsql_Release_x86.zip?tag=0.96.0&job=Environment%3A%20arch%3Dx86) or [64bit version](https://ci.appveyor.com/api/projects/openstreetmap/osm2pgsql/artifacts/osm2pgsql_Release_x64.zip?tag=0.96.0&job=Environment%3A%20arch%3Dx64).
+
+## Building ##
 
 The latest source code is available in the osm2pgsql git repository on GitHub
 and can be downloaded as follows:
@@ -30,8 +33,6 @@ and can be downloaded as follows:
 $ git clone git://github.com/openstreetmap/osm2pgsql.git
 ```
 
-## Building ##
-
 Osm2pgsql uses the cross-platform [CMake build system](https://cmake.org/)
 to configure and build itself and requires
 
@@ -44,6 +45,8 @@ Required libraries are
 * [Boost libraries](http://www.boost.org/), including system and filesystem
 * [PostgreSQL](http://www.postgresql.org/) client libraries
 * [Lua](http://www.lua.org/) (Optional, used for [Lua tag transforms](docs/lua.md))
+* [Python](https://python.org/) (only for running tests)
+* [Psycopg](http://initd.org/psycopg/) (only for running tests)
 
 It also requires access to a database server running
 [PostgreSQL](http://www.postgresql.org/) 9.1+ and [PostGIS](http://www.postgis.net/) 2.0+.

appveyor.yml

 environment:
   global:
+    BZIP2_VER: 1.0.6
+    EXPAT_VER: 2.2.5
+    PROJ4_VER: 4.9.3
+    ZLIB_VER: 1.2.11
+    LUA_VER: 5.3.4
+    LUA_INCLUDE_DIR: C:\lua\include
+    LUA_LIBRARIES: C:\lua\lua53.lib
+    LUA_DLL: C:\lua\lua53.dll
+    WINGETOPT_VER: v0.95
+    GETOPT_INCLUDE_DIR: C:\wingetopt\src
+    GETOPT_LIBRARY: C:\wingetopt\build\wingetopt.lib
+    BOOST_PATH: C:\Libraries\boost_1_63_0
+    POSTGRESQL_VER: 9.6.6
+    TESTS_POSTGRESQL_ROOT: C:\Progra~1\PostgreSQL\9.6
+    POSTGIS_VER: 2.4.3
+    POSTGIS_URL: "https://lonvia.dev.openstreetmap.org/osm2pgsql-winbuild/postgis-bundle-pg96-2.4.3x64.zip"
     PGUSER: postgres
     PGPASSWORD: Password12!
-    BOOST_ROOT: c:/libs/boost
-    PREFIX: c:\libs
-    PSQL_ROOT: C:/Program Files/PostgreSQL/9.4
-    CMAKE_PREFIX_PATH: c:\libs;C:\Program Files\PostgreSQL\9.4
-    POSTGIS_FILE: postgis-pg94-binaries-2.3.2w64gcc48
   matrix:
-  - configuration: Release
-
-# Operating system (build VM template)
+    - arch: x86
+    - arch: x64
 
 os: Visual Studio 2015
 
-cache:
-  - C:\libs\%POSTGIS_FILE%.zip -> appveyor.yml
-  - C:\osm2pgsql_win_deps_release.7z -> appveyor.yml
-
 services:
-  - postgresql94 # enable when Postgis will be available
-
-# scripts that are called at very beginning, before repo cloning
-init:
-  - git config --global core.autocrlf input
+  - postgresql96
 
-# clone directory
 clone_folder: c:\osm2pgsql
 
 clone_depth: 1
 
-platform: x64
+init:
+  - git config --global core.autocrlf input
+  - if "%arch%"=="x86" (
+      set vcvarsall_arg=x86&&
+      set conda_path=C:\Miniconda36\Scripts&&
+      set conda_library_path=C:\Miniconda36\envs\osm2pgsql\Library&&
+      set lua_url=https://lonvia.dev.openstreetmap.org/osm2pgsql-winbuild/lua_x86.zip&&
+      set build_type=Release)
+  - if "%arch%"=="x64" (
+      set vcvarsall_arg=amd64&&
+      set conda_path=C:\Miniconda36-x64\Scripts&&
+      set conda_library_path=C:\Miniconda36-x64\envs\osm2pgsql\Library&&
+      set lua_url=https://lonvia.dev.openstreetmap.org/osm2pgsql-winbuild/lua_x64.zip&&
+      set build_type=Release)
+  - '"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall" %vcvarsall_arg%'
 
 install:
-  # by default, all script lines are interpreted as batch
+  - set PATH=%TESTS_POSTGRESQL_ROOt%\bin;%PATH%;%conda_path%
   - cd c:\
-  - if exist libs ( rmdir c:\libs /s /q )
-  - mkdir libs
-  - if not exist osm2pgsql_win_deps_release.7z ( curl -O http://lonvia.dev.openstreetmap.org/osm2pgsql-winbuild/osm2pgsql_win_deps_release.7z )
-  - 7z x osm2pgsql_win_deps_release.7z | find ":"
-  - cd c:\libs
-  - echo Downloading and installing PostGIS:
-  - if not exist %POSTGIS_FILE%.zip ( curl -L -O -S -s http://lonvia.dev.openstreetmap.org/osm2pgsql-winbuild/%POSTGIS_FILE%.zip )
-  - 7z x %POSTGIS_FILE%.zip
-  - echo xcopy /e /y /q %POSTGIS_FILE% %PSQL_ROOT%
-  - xcopy /e /y /q %POSTGIS_FILE% "%PSQL_ROOT%"
-  - echo Creating tablespace for tablespace test...
-  - mkdir temp
-  - cacls temp /T /E /G Users:F
-  - cacls temp /T /E /G "Network Service":F
-  - echo Installing psycopg2 Python module...
+  - conda config --set always_yes yes
+  - conda create --name osm2pgsql
+  - activate osm2pgsql
+  - conda install bzip2=%BZIP2_VER% expat=%EXPAT_VER% proj4=%PROJ4_VER% zlib=%ZLIB_VER% postgresql=%POSTGRESQL_VER%
+  - ps: if (!(Test-Path "C:\lua_$env:arch.zip")){(new-object net.webclient).DownloadFile($env:lua_url, "C:\lua_$env:arch.zip")}
+  - 7z x -y C:\lua_%arch%.zip -oC:\lua
+  - ps: if (!(Test-Path "C:\postgis.zip")){(new-object net.webclient).DownloadFile($env:POSTGIS_URL, "C:\postgis.zip")}
+  - 7z x C:\postgis.zip -oC:\postgis
+  - xcopy /e /y /q C:\postgis\postgis-bundle-pg96-2.4.3x64 %TESTS_POSTGRESQL_ROOT%
+  - git clone https://github.com/alex85k/wingetopt -b %WINGETOPT_VER% C:\wingetopt
   - python -V
   - pip install psycopg2
 
+before_build:
+  - cd C:\wingetopt
+  - mkdir build
+  - cd build
+  - cmake .. -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=%build_type%
+  - nmake
+
 build_script:
   - mkdir c:\osm2pgsql\build
   - cd c:\osm2pgsql\build
-  - echo Running cmake...
-  - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
-  - cmake .. -LA -G "NMake Makefiles" -DBOOST_ROOT=%BOOST_ROOT% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%PREFIX% -DBoost_USE_STATIC_LIBS=ON -DBUILD_TESTS=ON
+  - >
+    cmake .. -LA -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=%build_type% -DBUILD_TESTS=ON
+    -DLUA_INCLUDE_DIR=%LUA_INCLUDE_DIR% -DLUA_LIBRARIES=%LUA_LIBRARIES% 
+    -DGETOPT_INCLUDE_DIR=%GETOPT_INCLUDE_DIR% -DGETOPT_LIBRARY=%GETOPT_LIBRARY%
+    -DBOOST_ROOT=%BOOST_PATH% -DBoost_USE_STATIC_LIBS=ON
   - nmake
+
+after_build:
   - mkdir osm2pgsql-bin
   - copy /y *.exe osm2pgsql-bin
   - copy /y ..\*.style osm2pgsql-bin
   - copy /y ..\*.lua osm2pgsql-bin
-  - copy /y %PREFIX%\bin\*.dll osm2pgsql-bin
-  - copy /y "%PSQL_ROOT%\bin\libpq.dll" osm2pgsql-bin
-  - copy /y "%PSQL_ROOT%\bin\libintl-8.dll" osm2pgsql-bin
-  - copy /y "%PSQL_ROOT%\bin\libeay32.dll" osm2pgsql-bin
-  - copy /y "%PSQL_ROOT%\bin\ssleay32.dll" osm2pgsql-bin
-  - 7z a c:\osm2pgsql\osm2pgsql_%Configuration%.zip osm2pgsql-bin -tzip
+  - copy /y %conda_library_path%\bin\libpq.dll osm2pgsql-bin
+  - copy /y %conda_library_path%\bin\libeay32.dll osm2pgsql-bin
+  - copy /y %conda_library_path%\bin\ssleay32.dll osm2pgsql-bin
+  - copy /y %conda_library_path%\bin\zlib.dll osm2pgsql-bin
+  - copy /y %conda_library_path%\bin\expat.dll osm2pgsql-bin
+  - copy /y %conda_library_path%\bin\libbz2.dll osm2pgsql-bin
+  - copy /y %LUA_DLL% osm2pgsql-bin
+  - 7z a c:\osm2pgsql\osm2pgsql_%build_type%_%arch%.zip osm2pgsql-bin -tzip
 
-test_script:
+before_test:
+  - cd c:\
+  - mkdir temp
+  - cacls temp /T /E /G Users:F
+  - cacls temp /T /E /G "Network Service":F
   - |
-    "%PSQL_ROOT%/bin/psql" -c "CREATE TABLESPACE tablespacetest LOCATION 'c:/libs/temp'"
+    psql -c "CREATE TABLESPACE tablespacetest LOCATION 'c:/temp'"
   - set PATH=c:\osm2pgsql\build\osm2pgsql-bin;%PATH%
-  - set PROJ_LIB=c:\libs\share
+  - set PROJ_LIB=%conda_library_path%\share
+
+test_script:
   - cd c:\osm2pgsql\build
   #- ctest -VV -L NoDB
   - ctest -VV -LE FlatNodes # enable when Postgis will be available
 
-
 artifacts:
-  - path: osm2pgsql_Release.zip
-    name: osm2pgsql_Release.zip
+  - path: osm2pgsql_%build_type%_%arch%.zip
+
+cache:
+  - C:\lua_%arch%.zip -> appveyor.yml
+  - C:\postgis.zip -> appveyor.yml

cmake/FindLuaJIT.cmake

+# Locate LuaJIT library
+# This module defines
+#  LUAJIT_FOUND, if false, do not try to link to Lua
+#  LUAJIT_LIBRARIES
+#  LUAJIT_INCLUDE_DIR, where to find luajit.h
+#  LUAJIT_VERSION_STRING, the version of Lua found (since CMake 2.8.8)
+
+## Copied from default CMake FindLua51.cmake
+
+find_path(LUAJIT_INCLUDE_DIR luajit.h
+  HINTS
+    ENV LUA_DIR
+  PATH_SUFFIXES include/luajit-2.0 include
+  PATHS
+  ~/Library/Frameworks
+  /Library/Frameworks
+  /sw # Fink
+  /opt/local # DarwinPorts
+  /opt/csw # Blastwave
+  /opt
+)
+
+find_library(LUAJIT_LIBRARY
+  NAMES luajit-5.1
+  HINTS
+    ENV LUA_DIR
+  PATH_SUFFIXES lib
+  PATHS
+  ~/Library/Frameworks
+  /Library/Frameworks
+  /sw
+  /opt/local
+  /opt/csw
+  /opt
+)
+
+if(LUAJIT_LIBRARY)
+  # include the math library for Unix
+  if(UNIX AND NOT APPLE)
+    find_library(LUA_MATH_LIBRARY m)
+    set( LUAJIT_LIBRARIES "${LUAJIT_LIBRARY};${LUA_MATH_LIBRARY}" CACHE STRING "LuaJIT Libraries")
+  # For Windows and Mac, don't need to explicitly include the math library
+  else()
+    set( LUAJIT_LIBRARIES "${LUAJIT_LIBRARY}" CACHE STRING "LuaJIT Libraries")
+  endif()
+endif()
+
+if(LUAJIT_INCLUDE_DIR AND EXISTS "${LUAJIT_INCLUDE_DIR}/luajit.h")
+  file(STRINGS "${LUAJIT_INCLUDE_DIR}/luajit.h" luajit_version_str REGEX "^#define[ \t]+LUAJIT_VERSION[ \t]+\"LuaJIT .+\"")
+
+  string(REGEX REPLACE "^#define[ \t]+LUAJIT_VERSION[ \t]+\"LuaJIT ([^\"]+)\".*" "\\1" LUAJIT_VERSION_STRING "${luajit_version_str}")
+  unset(luajit_version_str)
+endif()
+
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set LUAJIT_FOUND to TRUE if
+# all listed variables are TRUE
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LuaJIT
+                                  REQUIRED_VARS LUAJIT_LIBRARIES LUAJIT_INCLUDE_DIR
+                                  VERSION_VAR LUAJIT_VERSION_STRING)
+
+mark_as_advanced(LUAJIT_INCLUDE_DIR LUAJIT_LIBRARIES LUAJIT_LIBRARY LUA_MATH_LIBRARY)
+

cmake/FindOsmium.cmake

@@ -2,8 +2,8 @@
 #
 #  FindOsmium.cmake
 #
-#  Find the Libosmium headers and, optionally, several components needed for
-#  different Libosmium functions.
+#  Find the Libosmium headers and, optionally, several components needed
+#  for different Libosmium functions.
 #
 #----------------------------------------------------------------------
 #
@@ -18,9 +18,12 @@
 #
 #    Then add the following in your CMakeLists.txt:
 #
-#      find_package(Osmium REQUIRED COMPONENTS <XXX>)
+#      find_package(Osmium [version] REQUIRED COMPONENTS <XXX>)
 #      include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS})
 #
+#    The version number is optional. If it is not set, any version of
+#    libosmium will do.
+#
 #    For the <XXX> substitute a space separated list of one or more of the
 #    following components:
 #
@@ -51,10 +54,8 @@
 #
 #----------------------------------------------------------------------
 
-# Look for the header file.
-find_path(OSMIUM_INCLUDE_DIR osmium/osm.hpp
-    PATH_SUFFIXES include
-    PATHS
+# This is the list of directories where we look for osmium includes.
+set(_osmium_include_path
         ../libosmium
         ~/Library/Frameworks
         /Library/Frameworks
@@ -62,6 +63,22 @@ find_path(OSMIUM_INCLUDE_DIR osmium/osm.hpp
         /opt
 )
 
+# Look for the header file.
+find_path(OSMIUM_INCLUDE_DIR osmium/version.hpp
+    PATH_SUFFIXES include
+    PATHS ${_osmium_include_path}
+)
+
+# Check libosmium version number
+if(Osmium_FIND_VERSION)
+    file(STRINGS "${OSMIUM_INCLUDE_DIR}/osmium/version.hpp" _libosmium_version_define REGEX "#define LIBOSMIUM_VERSION_STRING")
+    if("${_libosmium_version_define}" MATCHES "#define LIBOSMIUM_VERSION_STRING \"([0-9.]+)\"")
+        set(_libosmium_version "${CMAKE_MATCH_1}")
+    else()
+        set(_libosmium_version "unknown")
+    endif()
+endif()
+
 set(OSMIUM_INCLUDE_DIRS "${OSMIUM_INCLUDE_DIR}")
 
 #----------------------------------------------------------------------
@@ -94,18 +111,17 @@ endif()
 if(Osmium_USE_PBF)
     find_package(ZLIB)
     find_package(Threads)
+    find_package(Protozero 1.5.1)
 
-    list(APPEND OSMIUM_EXTRA_FIND_VARS ZLIB_FOUND Threads_FOUND)
-    if(ZLIB_FOUND AND Threads_FOUND)
+    list(APPEND OSMIUM_EXTRA_FIND_VARS ZLIB_FOUND Threads_FOUND PROTOZERO_INCLUDE_DIR)
+    if(ZLIB_FOUND AND Threads_FOUND AND PROTOZERO_FOUND)
         list(APPEND OSMIUM_PBF_LIBRARIES
             ${ZLIB_LIBRARIES}
             ${CMAKE_THREAD_LIBS_INIT}
         )
-        if(WIN32)
-            list(APPEND OSMIUM_PBF_LIBRARIES ws2_32)
-        endif()
         list(APPEND OSMIUM_INCLUDE_DIRS
             ${ZLIB_INCLUDE_DIR}
+            ${PROTOZERO_INCLUDE_DIR}
         )
     else()
         message(WARNING "Osmium: Can not find some libraries for PBF input/output, please install them or configure the paths.")
@@ -256,12 +272,28 @@ endif()
 if(OSMIUM_EXTRA_FIND_VARS)
     list(REMOVE_DUPLICATES OSMIUM_EXTRA_FIND_VARS)
 endif()
-# Handle the QUIETLY and REQUIRED arguments and set OSMIUM_FOUND to TRUE if
-# all listed variables are TRUE.
+# Handle the QUIETLY and REQUIRED arguments and the optional version check
+# and set OSMIUM_FOUND to TRUE if all listed variables are TRUE.
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Osmium REQUIRED_VARS OSMIUM_INCLUDE_DIR ${OSMIUM_EXTRA_FIND_VARS})
+find_package_handle_standard_args(Osmium
+                                  REQUIRED_VARS OSMIUM_INCLUDE_DIR ${OSMIUM_EXTRA_FIND_VARS}
+                                  VERSION_VAR _libosmium_version)
 unset(OSMIUM_EXTRA_FIND_VARS)
 
+#----------------------------------------------------------------------
+#
+#  A function for setting the -pthread option in compilers/linkers
+#
+#----------------------------------------------------------------------
+function(set_pthread_on_target _target)
+    if(NOT MSVC)
+        set_target_properties(${_target} PROPERTIES COMPILE_FLAGS "-pthread")
+        if(NOT APPLE)
+            set_target_properties(${_target} PROPERTIES LINK_FLAGS "-pthread")
+        endif()
+    endif()
+endfunction()
+
 #----------------------------------------------------------------------
 #
 #  Add compiler flags
@@ -285,6 +317,10 @@ if(MSVC)
     # old compilers anyway.
     add_definitions(-wd4351)
 
+    # Disable warning C4503: "decorated name length exceeded, name was truncated"
+    # there are more than 150 of generated names in libosmium longer than 4096 symbols supported in MSVC
+    add_definitions(-wd4503)
+
     add_definitions(-DNOMINMAX -DWIN32_LEAN_AND_MEAN -D_CRT_SECURE_NO_WARNINGS)
 endif()
 
@@ -309,9 +345,10 @@ endif()
 set(OSMIUM_DRACONIC_CLANG_OPTIONS "-Wdocumentation -Wunused-exception-parameter -Wmissing-declarations -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-unused-macros -Wno-exit-time-destructors -Wno-global-constructors -Wno-padded -Wno-switch-enum -Wno-missing-prototypes -Wno-weak-vtables -Wno-cast-align -Wno-float-equal")
 
 if(Osmium_DEBUG)
-    message(STATUS "OSMIUM_XML_LIBRARIES=" ${OSMIUM_XML_LIBRARIES})
-    message(STATUS "OSMIUM_PBF_LIBRARIES=" ${OSMIUM_PBF_LIBRARIES})
-    message(STATUS "OSMIUM_IO_LIBRARIES=" ${OSMIUM_IO_LIBRARIES})
-    message(STATUS "OSMIUM_LIBRARIES=" ${OSMIUM_LIBRARIES})
-    message(STATUS "OSMIUM_INCLUDE_DIRS=" ${OSMIUM_INCLUDE_DIRS})
+    message(STATUS "OSMIUM_XML_LIBRARIES=${OSMIUM_XML_LIBRARIES}")
+    message(STATUS "OSMIUM_PBF_LIBRARIES=${OSMIUM_PBF_LIBRARIES}")
+    message(STATUS "OSMIUM_IO_LIBRARIES=${OSMIUM_IO_LIBRARIES}")
+    message(STATUS "OSMIUM_LIBRARIES=${OSMIUM_LIBRARIES}")
+    message(STATUS "OSMIUM_INCLUDE_DIRS=${OSMIUM_INCLUDE_DIRS}")
 endif()
+

cmake/FindProtozero.cmake

+#----------------------------------------------------------------------
+#
+#  FindProtozero.cmake
+#
+#  Find the protozero headers.
+#
+#----------------------------------------------------------------------
+#
+#  Usage:
+#
+#    Copy this file somewhere into your project directory, where cmake can
+#    find it. Usually this will be a directory called "cmake" which you can
+#    add to the CMake module search path with the following line in your
+#    CMakeLists.txt:
+#
+#      list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+#
+#    Then add the following in your CMakeLists.txt:
+#
+#      find_package(Protozero [version] [REQUIRED])
+#      include_directories(SYSTEM ${PROTOZERO_INCLUDE_DIR})
+#
+#    The version number is optional. If it is not set, any version of
+#    protozero will do.
+#
+#      if(NOT PROTOZERO_FOUND)
+#          message(WARNING "Protozero not found!\n")
+#      endif()
+#
+#----------------------------------------------------------------------
+#
+#  Variables:
+#
+#    PROTOZERO_FOUND        - True if Protozero was found.
+#    PROTOZERO_INCLUDE_DIR  - Where to find include files.
+#
+#----------------------------------------------------------------------
+
+# find include path
+find_path(PROTOZERO_INCLUDE_DIR protozero/version.hpp
+    PATH_SUFFIXES include
+    PATHS ${CMAKE_SOURCE_DIR}/../protozero
+)
+
+# Check version number
+if(Protozero_FIND_VERSION)
+    file(STRINGS "${PROTOZERO_INCLUDE_DIR}/protozero/version.hpp" _version_define REGEX "#define PROTOZERO_VERSION_STRING")
+    if("${_version_define}" MATCHES "#define PROTOZERO_VERSION_STRING \"([0-9.]+)\"")
+        set(_version "${CMAKE_MATCH_1}")
+    else()
+        set(_version "unknown")
+    endif()
+endif()
+
+#set(PROTOZERO_INCLUDE_DIRS "${PROTOZERO_INCLUDE_DIR}")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Protozero
+                                  REQUIRED_VARS PROTOZERO_INCLUDE_DIR
+                                  VERSION_VAR _version)
+
+
+#----------------------------------------------------------------------

debian/changelog

-osm2pgsql (0.94.0+ds-2) UNRELEASED; urgency=medium
+osm2pgsql (0.96.0+ds-1) unstable; urgency=medium
 
+  * New upstream release.
+  * Move from experimental to unstable.
+
+ -- Bas Couwenberg <sebastic@debian.org>  Thu, 03 May 2018 07:09:33 +0200
+
+osm2pgsql (0.96.0~rc1+ds-1~exp1) experimental; urgency=medium
+
+  * New upstream release candidate.
   * Strip trailing whitespace from changelog & rules.
   * Update copyright-format URL to use HTTPS.
   * Bump Standards-Version to 4.1.4, no changes.
   * Update Vcs-* URLs for Salsa.
+  * Also exclude embedded copy of protozero.
+  * Override dh_auto_configure to use external libosmium & protozero.
+  * Bump minimum required libosmium2-dev to 2.14.0.
 
- -- Bas Couwenberg <sebastic@debian.org>  Wed, 15 Nov 2017 22:04:23 +0100
+ -- Bas Couwenberg <sebastic@debian.org>  Sat, 21 Apr 2018 10:16:43 +0200
 
 osm2pgsql (0.94.0+ds-1) unstable; urgency=medium
 

debian/control

@@ -13,7 +13,7 @@ Build-Depends: debhelper (>= 9),
                libboost-filesystem-dev,
                libbz2-dev,
                libexpat1-dev,
-               libosmium2-dev (>= 2.13.1),
+               libosmium2-dev (>= 2.14.0),
                libpq-dev,
                libproj-dev,
                zlib1g-dev,

debian/copyright

@@ -5,6 +5,7 @@ Source: https://github.com/openstreetmap/osm2pgsql/releases
  The upstream tarballs are repacked to exclude embedded copies of
  dependencies.
 Files-Excluded: contrib/libosmium/*
+ contrib/protozero/*
 
 Files: *
 Copyright: 2007-2008, Artem Pavlenko <artem@mapnik.org>

debian/rules

@@ -18,6 +18,10 @@ CXXFLAGS += $(CPPFLAGS)
 		--buildsystem cmake \
 		--parallel
 
+override_dh_auto_configure:
+	dh_auto_configure -- -DEXTERNAL_LIBOSMIUM=ON \
+	                     -DEXTERNAL_PROTOZERO=ON
+
 override_dh_auto_test:
 	dh_auto_test || echo "Ignoring test failures"
 

docs/osm2pgsql.1

@@ -22,12 +22,12 @@ imports data from OSM file(s) into a PostgreSQL database
 suitable for use by the Mapnik renderer or the Nominatim geocoder.
 .br
 OSM planet snapshots can be downloaded from
-http://planet.openstreetmap.org/. Partial planet files
+https://planet.openstreetmap.org/. Partial planet files
 ("extracts") for various countries are available, see
-http://wiki.openstreetmap.org/wiki/Planet.osm.
+https://wiki.openstreetmap.org/wiki/Planet.osm.
 .PP
 Extracts in PBF (ProtoBufBinary) format are also available from
-http://download.geofabrik.de/osm/.
+https://download.geofabrik.de/osm/.
 .PP
 When operating in "slim" mode (and on a database created in "slim" mode!),
 .B osm2pgsql

middle-pgsql.cpp

@@ -24,7 +24,7 @@ using namespace std;
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
-#include <future>
+#include <functional>
 
 #include <boost/format.hpp>
 #include <osmium/memory/buffer.hpp>
@@ -1068,21 +1068,24 @@ void middle_pgsql_t::pgsql_stop_one(table_desc *table)
     fprintf(stderr, "Stopped table: %s in %is\n", table->name, (int)(end - start));
 }
 
-
-void middle_pgsql_t::stop(void)
+void middle_pgsql_t::stop(osmium::thread::Pool &pool)
 {
     cache.reset();
-    if (out_options->flat_node_cache_enabled) persistent_cache.reset();
-
-    std::vector<std::future<void>> futures;
-    futures.reserve(num_tables);
+    if (out_options->flat_node_cache_enabled) {
+        persistent_cache.reset();
+    }
 
+    if (out_options->droptemp) {
+        // Dropping the tables is fast, so do it synchronously to guarantee
+        // that the space is freed before creating the other indices.
         for (int i = 0; i < num_tables; ++i) {
-        futures.push_back(std::async(&middle_pgsql_t::pgsql_stop_one, this, &tables[i]));
+            pgsql_stop_one(&tables[i]);
+        }
+    } else {
+        for (int i = 0; i < num_tables; ++i) {
+            pool.submit(
+                std::bind(&middle_pgsql_t::pgsql_stop_one, this, &tables[i]));
         }
-
-    for (auto &f : futures) {
-        f.get();
     }
 }
 

middle-pgsql.hpp

@@ -21,7 +21,7 @@ struct middle_pgsql_t : public slim_middle_t {
     virtual ~middle_pgsql_t();
 
     void start(const options_t *out_options_) override;
-    void stop(void) override;
+    void stop(osmium::thread::Pool &pool) override;
     void analyze(void) override;
     void end(void) override;
     void commit(void) override;

middle-ram.cpp

@@ -167,7 +167,7 @@ void middle_ram_t::start(const options_t *out_options_)
         new node_ram_cache(out_options->alloc_chunkwise, out_options->cache));
 }
 
-void middle_ram_t::stop(void)
+void middle_ram_t::stop(osmium::thread::Pool &pool)
 {
     cache.reset(nullptr);
 

middle-ram.hpp

@@ -85,7 +85,7 @@ struct middle_ram_t : public middle_t {
     virtual ~middle_ram_t();
 
     void start(const options_t *out_options_) override;
-    void stop(void) override;
+    void stop(osmium::thread::Pool &pool) override;
     void analyze(void) override;
     void end(void) override;
     void commit(void) override;

middle.hpp

@@ -12,6 +12,8 @@
 #include <cstddef>
 #include <memory>
 
+#include <osmium/thread/pool.hpp>
+
 #include "osmtypes.hpp"
 #include "reprojection.hpp"
 
@@ -84,7 +86,7 @@ struct middle_t : public middle_query_t {
     virtual ~middle_t() {}
 
     virtual void start(const options_t *out_options_) = 0;
-    virtual void stop(void) = 0;
+    virtual void stop(osmium::thread::Pool &pool) = 0;
     virtual void analyze(void) = 0;
     virtual void end(void) = 0;
     virtual void commit(void) = 0;

node-persistent-cache.cpp

@@ -56,12 +56,13 @@ node_persistent_cache::node_persistent_cache(
                                  "of the flat node file was not set.");
     }
 
-    auto fname = options->flat_node_file->c_str();
-    fprintf(stderr, "Mid: loading persistent node cache from %s\n", fname);
+    m_fname = options->flat_node_file->c_str();
+    m_remove_file = options->droptemp;
+    fprintf(stderr, "Mid: loading persistent node cache from %s\n", m_fname);
 
-    m_fd = open(fname, O_RDWR | O_CREAT, 0644);
+    m_fd = open(m_fname, O_RDWR | O_CREAT, 0644);
     if (m_fd < 0) {
-        fprintf(stderr, "Cannot open location cache file '%s': %s\n", fname,
+        fprintf(stderr, "Cannot open location cache file '%s': %s\n", m_fname,
                 std::strerror(errno));
         throw std::runtime_error("Unable to open flatnode file\n");
     }
@@ -75,4 +76,9 @@ node_persistent_cache::~node_persistent_cache()
     if (m_fd >= 0) {
         close(m_fd);
     }
+
+    if (m_remove_file) {
+        fprintf(stderr, "Mid: removing persistent node cache at %s\n", m_fname);
+        unlink(m_fname);
+    }
 }

node-persistent-cache.hpp

@@ -32,6 +32,8 @@ private:
     std::shared_ptr<node_ram_cache> m_ram_cache;
     int m_fd;
     std::unique_ptr<index_t> m_index;
+    bool m_remove_file;
+    const char *m_fname;
 };
 
 #endif