CHANGELOG.md

@@ -13,6 +13,14 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 ### Fixed
 
 
+## [2.14.2] - 2018-07-23
+
+### Fixed
+
+* PBF reader and writer depended on byte order of system architecture.
+* Removed an unreliable test that didn't work on some architectures.
+
+
 ## [2.14.1] - 2018-07-23
 
 ### Changed
@@ -861,7 +869,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   Doxygen (up to version 1.8.8). This version contains a workaround to fix
   this.
 
-[unreleased]: https://github.com/osmcode/libosmium/compare/v2.14.1...HEAD
+[unreleased]: https://github.com/osmcode/libosmium/compare/v2.14.2...HEAD
+[2.14.2]: https://github.com/osmcode/libosmium/compare/v2.14.1...v2.14.2
 [2.14.1]: https://github.com/osmcode/libosmium/compare/v2.14.0...v2.14.1
 [2.14.0]: https://github.com/osmcode/libosmium/compare/v2.13.1...v2.14.0
 [2.13.1]: https://github.com/osmcode/libosmium/compare/v2.13.0...v2.13.1

CMakeLists.txt

@@ -25,7 +25,7 @@ project(libosmium)
 
 set(LIBOSMIUM_VERSION_MAJOR 2)
 set(LIBOSMIUM_VERSION_MINOR 14)
-set(LIBOSMIUM_VERSION_PATCH 1)
+set(LIBOSMIUM_VERSION_PATCH 2)
 
 set(LIBOSMIUM_VERSION
     "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}")

build-msys2.bat

@@ -13,6 +13,9 @@ bash -lc "pacman -S --needed --noconfirm mingw-w64-x86_64-gcc mingw-w64-x86_64-g
 bash -lc "pacman -S --needed --noconfirm mingw-w64-x86_64-postgresql mingw-w64-x86_64-netcdf mingw-w64-x86_64-crypto++"
 call C:\msys64\mingw64\bin\gem.cmd install json
 
+REM Workaround for problem with gdal (see https://github.com/osmcode/libosmium/issues/262)
+copy /y C:\msys64\mingw64\bin\libjson-c-4.dll C:\msys64\mingw64\bin\libjson-c-3.dll
+
 echo "Setting PROJ_LIB variable for correct PROJ.4 working"
 set PROJ_LIB=c:\msys64\mingw64\share\proj
 

debian/changelog

+libosmium (2.14.2-1) unstable; urgency=medium
+
+  * New upstream release.
+  * Drop byteorder patch, included upstream.
+  * Don't ignore test failures on sh4, problematic test removed.
+  * Check DEB_BUILD_OPTIONS for nocheck in dh_auto_test override.
+
+ -- Bas Couwenberg <sebastic@debian.org>  Tue, 24 Jul 2018 07:08:15 +0200
+
 libosmium (2.14.1-2) unstable; urgency=medium
 
   * Ignore test failures on sh4.

debian/patches/0001-Rewrite-code-to-be-byte-order-independent.patch

-Description: Rewrite code to be byte-order independent.
-Author: Jochen Topf <jochen@topf.org>
-Origin: https://github.com/osmcode/libosmium/commit/ff16782fb7421d1af8120d5f519bc1b4e6453e71
-Bug: https://github.com/osmcode/libosmium/issues/263
-
---- a/include/osmium/io/detail/pbf_input_format.hpp
-+++ b/include/osmium/io/detail/pbf_input_format.hpp
-@@ -44,7 +44,6 @@ DEALINGS IN THE SOFTWARE.
- #include <osmium/thread/util.hpp>
- #include <osmium/util/config.hpp>
- 
--#include <protozero/byteswap.hpp>
- #include <protozero/pbf_message.hpp>
- #include <protozero/types.hpp>
- 
-@@ -97,18 +96,20 @@ namespace osmium {
-                  * the length of the following BlobHeader.
-                  */
-                 uint32_t read_blob_header_size_from_file() {
--                    uint32_t size_in_network_byte_order;
-+                    uint32_t size;
- 
-                     try {
--                        const std::string input_data{read_from_input_queue(sizeof(size_in_network_byte_order))};
--                        size_in_network_byte_order = *reinterpret_cast<const uint32_t*>(input_data.data());
-+                        // size is encoded in network byte order
-+                        const std::string input_data{read_from_input_queue(sizeof(size))};
-+                        const char* d = input_data.data();
-+                        size = (static_cast<uint32_t>(d[3])) |
-+                               (static_cast<uint32_t>(d[2]) << 8u) |
-+                               (static_cast<uint32_t>(d[1]) << 16u) |
-+                               (static_cast<uint32_t>(d[0]) << 24u);
-                     } catch (const osmium::pbf_error&) {
-                         return 0; // EOF
-                     }
- 
--                    uint32_t size = size_in_network_byte_order;
--                    ::protozero::byteswap_inplace(&size);
--
-                     if (size > static_cast<uint32_t>(max_blob_header_size)) {
-                         throw osmium::pbf_error{"invalid BlobHeader size (> max_blob_header_size)"};
-                     }
---- a/include/osmium/io/detail/pbf_output_format.hpp
-+++ b/include/osmium/io/detail/pbf_output_format.hpp
-@@ -62,7 +62,6 @@ DEALINGS IN THE SOFTWARE.
- #include <osmium/util/misc.hpp>
- #include <osmium/visitor.hpp>
- 
--#include <protozero/byteswap.hpp>
- #include <protozero/pbf_builder.hpp>
- #include <protozero/pbf_writer.hpp>
- #include <protozero/types.hpp>
-@@ -193,13 +192,16 @@ namespace osmium {
-                     // data plus a few header bytes (https://zlib.net/zlib_tech.html).
-                     pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, static_cast<int32_t>(blob_data.size()));
- 
--                    auto sz = static_cast<uint32_t>(blob_header_data.size());
--                    ::protozero::byteswap_inplace(&sz);
-+                    const auto size = static_cast<uint32_t>(blob_header_data.size());
- 
--                    // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob
-+                    // write to output: the 4-byte BlobHeader size in network
-+                    // byte order followed by the BlobHeader followed by the Blob
-                     std::string output;
--                    output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size());
--                    output.append(reinterpret_cast<const char*>(&sz), sizeof(sz));
-+                    output.reserve(4 + blob_header_data.size() + blob_data.size());
-+                    output += static_cast<char>((size >> 24u) & 0xffu);
-+                    output += static_cast<char>((size >> 16u) & 0xffu);
-+                    output += static_cast<char>((size >>  8u) & 0xffu);
-+                    output += static_cast<char>( size         & 0xffu);
-                     output.append(blob_header_data);
-                     output.append(blob_data);
- 

debian/patches/series

-0001-Rewrite-code-to-be-byte-order-independent.patch

debian/rules

@@ -20,10 +20,7 @@ override_dh_auto_build-indep:
 	(cd build && $(MAKE) doc)
 
 override_dh_auto_test-arch:
-# Ignore test failures on problematic architectures only
-ifneq (,$(filter $(DEB_BUILD_ARCH),sh4))
-	(cd build && ctest -V || echo "Ignoring test failures")
-else
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
 	(cd build && ctest -V)
 endif
 

include/osmium/io/detail/pbf_input_format.hpp

@@ -44,7 +44,6 @@ DEALINGS IN THE SOFTWARE.
 #include <osmium/thread/util.hpp>
 #include <osmium/util/config.hpp>
 
-#include <protozero/byteswap.hpp>
 #include <protozero/pbf_message.hpp>
 #include <protozero/types.hpp>
 
@@ -97,18 +96,20 @@ namespace osmium {
                  * the length of the following BlobHeader.
                  */
                 uint32_t read_blob_header_size_from_file() {
-                    uint32_t size_in_network_byte_order;
+                    uint32_t size;
 
                     try {
-                        const std::string input_data{read_from_input_queue(sizeof(size_in_network_byte_order))};
-                        size_in_network_byte_order = *reinterpret_cast<const uint32_t*>(input_data.data());
+                        // size is encoded in network byte order
+                        const std::string input_data{read_from_input_queue(sizeof(size))};
+                        const char* d = input_data.data();
+                        size = (static_cast<uint32_t>(d[3])) |
+                               (static_cast<uint32_t>(d[2]) << 8u) |
+                               (static_cast<uint32_t>(d[1]) << 16u) |
+                               (static_cast<uint32_t>(d[0]) << 24u);
                     } catch (const osmium::pbf_error&) {
                         return 0; // EOF
                     }
 
-                    uint32_t size = size_in_network_byte_order;
-                    ::protozero::byteswap_inplace(&size);
-
                     if (size > static_cast<uint32_t>(max_blob_header_size)) {
                         throw osmium::pbf_error{"invalid BlobHeader size (> max_blob_header_size)"};
                     }

include/osmium/io/detail/pbf_output_format.hpp

@@ -62,7 +62,6 @@ DEALINGS IN THE SOFTWARE.
 #include <osmium/util/misc.hpp>
 #include <osmium/visitor.hpp>
 
-#include <protozero/byteswap.hpp>
 #include <protozero/pbf_builder.hpp>
 #include <protozero/pbf_writer.hpp>
 #include <protozero/types.hpp>
@@ -193,13 +192,16 @@ namespace osmium {
                     // data plus a few header bytes (https://zlib.net/zlib_tech.html).
                     pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, static_cast<int32_t>(blob_data.size()));
 
-                    auto sz = static_cast<uint32_t>(blob_header_data.size());
-                    ::protozero::byteswap_inplace(&sz);
+                    const auto size = static_cast<uint32_t>(blob_header_data.size());
 
-                    // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob
+                    // write to output: the 4-byte BlobHeader size in network
+                    // byte order followed by the BlobHeader followed by the Blob
                     std::string output;
-                    output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size());
-                    output.append(reinterpret_cast<const char*>(&sz), sizeof(sz));
+                    output.reserve(4 + blob_header_data.size() + blob_data.size());
+                    output += static_cast<char>((size >> 24u) & 0xffu);
+                    output += static_cast<char>((size >> 16u) & 0xffu);
+                    output += static_cast<char>((size >>  8u) & 0xffu);
+                    output += static_cast<char>( size         & 0xffu);
                     output.append(blob_header_data);
                     output.append(blob_data);
 

include/osmium/version.hpp

@@ -35,8 +35,8 @@ DEALINGS IN THE SOFTWARE.
 
 #define LIBOSMIUM_VERSION_MAJOR 2
 #define LIBOSMIUM_VERSION_MINOR 14
-#define LIBOSMIUM_VERSION_PATCH 1
+#define LIBOSMIUM_VERSION_PATCH 2
 
-#define LIBOSMIUM_VERSION_STRING "2.14.1"
+#define LIBOSMIUM_VERSION_STRING "2.14.2"
 
 #endif // OSMIUM_VERSION_HPP

test/t/util/test_memory.cpp

@@ -6,29 +6,9 @@
 
 TEST_CASE("Check memory usage") {
 #ifdef __linux__
-    const int size_in_mbytes = 10;
-
     osmium::MemoryUsage m1;
     REQUIRE(m1.current() > 1);
     REQUIRE(m1.peak() > 1);
-
-// Memory reporting on M68k architecture doesn't work properly.
-# ifndef __m68k__
-    {
-        std::vector<int> v;
-        v.reserve(size_in_mbytes * 1024 * 1024);
-
-        osmium::MemoryUsage m2;
-        REQUIRE(m2.current() >= m1.current() + size_in_mbytes);
-        REQUIRE(m2.peak() >= m1.peak() + size_in_mbytes);
-        REQUIRE(m2.peak() - m2.current() <= 1);
-    }
-
-    osmium::MemoryUsage m3;
-    REQUIRE(m3.current() > 1);
-    REQUIRE(m3.current() <= m3.peak());
-    REQUIRE(m3.peak() >= m1.peak() + size_in_mbytes);
-# endif
 #else
     osmium::MemoryUsage m;
     REQUIRE(m.current() == 0);