README.md

@@ -35,7 +35,7 @@ required to run the full build and test suite.
 
     ./docker.sh
 
-Osmosis is built using the Gradle (http://gradle.org) built tool.  Gradle itself
+Osmosis is built using the Gradle (http://gradle.org) build tool.  Gradle itself
 does not need to be installed because the gradlew script will install Gradle on
 first usage.  The only requirements are a 1.7 JDK, and an Internet connection.
 Note that in the docker environment all downloads will still occur and be cached

build-support/docker/build/Dockerfile

-FROM java:8
+FROM openjdk:11
 
 ARG CURRENT_USER_ID
 ARG CURRENT_GROUP_ID

build-support/docker/db/Dockerfile

-FROM fedora:24
+FROM fedora:28
 
 # Install UTF8 locale support
 RUN dnf install -y langpacks-en.noarch

build.gradle

-task wrapper(type: Wrapper) {
-	gradleVersion = '4.2'
-}
-
 apply plugin: 'idea'
 
 /* Build collections containing each type of project.  These collections will

debian/changelog

-osmosis (0.46-8) UNRELEASED; urgency=medium
+osmosis (0.47-1) unstable; urgency=medium
 
+  * New upstream release.
   * Bump Standards-Version to 4.2.1, no changes.
+  * Drop example-shell-script-fails-syntax-check.patch, applied upstream.
+  * Add patch to revert protobuf 3.6.1 changes.
+  * Add patch to regenerate protobuf files with protobuf 3.0.0.
 
- -- Bas Couwenberg <sebastic@debian.org>  Tue, 28 Aug 2018 14:48:58 +0200
+ -- Bas Couwenberg <sebastic@debian.org>  Sat, 29 Sep 2018 10:55:11 +0200
 
 osmosis (0.46-7) unstable; urgency=medium
 

debian/patches/04-osmosis-version.patch

@@ -4,7 +4,7 @@ Forwarded: not-needed
 
 --- a/build.gradle
 +++ b/build.gradle
-@@ -23,7 +23,7 @@ subprojects {
+@@ -19,7 +19,7 @@ subprojects {
  
  	// Load the project version dynamically from Git.  For release builds, don't add a suffix.
  	def versionSuffix = "RELEASE".equals(osmosisBuildType) ? '' : '-' + osmosisBuildType

debian/patches/example-shell-script-fails-syntax-check.patch

-Description: Fix `sh -n` syntax check failure.
- Also fix typo and quoting issues.
-Author: Bas Couwenberg <sebastic@debian.org>
-Forwarded: https://github.com/openstreetmap/osmosis/pull/42
-Applied-Upstream: https://github.com/openstreetmap/osmosis/commit/c87f5d39b6dba815177d43a4fcc5965ed49b7482
-
---- a/package/script/munin/osm_replication_lag
-+++ b/package/script/munin/osm_replication_lag
-@@ -1,35 +1,35 @@
--#!/bin/sh
--# -*- sh -*-
--
--# load the munin plugin helper
--. $MUNIN_LIBDIR/plugins/plugin.sh
--
--# if no workingDirectory has been configures
--if [ ! $workingDirectory ]; then
--	# exit with an error
--	echo "no workingDirectory configured" >&2
--	exit 1
--fi
--
--# path to osmosis binary
--[ $osmosis ]  || osmosis="osmosis"
--
--# configuration section
--if [ "$1" = "config" ]; then
--
--	echo 'graph_title OSM PostGIS Database Replag'
--	echo 'graph_args --base 1000'
--	echo 'graph_vlabel seconds behind main database'
--	echo 'graph_category osm'
--	
--	echo 'lag.label replication lag'
--	echo 'lag.draw LINE'
--	
--	exit 0
--fi
--
--# invoke osmosis to calculate the replication lag
--lag=$($osmosis --read-replication-lag workingDirectory="$workingDirectory" 2>/dev/null)
--echo "lag.value $lag"
--
--exit 0
-+#!/bin/sh
-+# -*- sh -*-
-+
-+# load the munin plugin helper
-+. "$MUNIN_LIBDIR/plugins/plugin.sh"
-+
-+# if no workingDirectory has been configured
-+if [ ! "$workingDirectory" ]; then
-+	# exit with an error
-+	echo "no workingDirectory configured" >&2
-+	exit 1
-+fi
-+
-+# path to osmosis binary
-+[ "$osmosis" ] || osmosis="osmosis"
-+
-+# configuration section
-+if [ "$1" = "config" ]; then
-+
-+	echo 'graph_title OSM PostGIS Database Replag'
-+	echo 'graph_args --base 1000'
-+	echo 'graph_vlabel seconds behind main database'
-+	echo 'graph_category osm'
-+
-+	echo 'lag.label replication lag'
-+	echo 'lag.draw LINE'
-+
-+	exit 0
-+fi
-+
-+# invoke osmosis to calculate the replication lag
-+lag=$($osmosis --read-replication-lag workingDirectory="$workingDirectory" 2>/dev/null)
-+echo "lag.value $lag"
-+
-+exit 0

debian/patches/series

 01-fix_launcher.patch
 02-fix_plexus.patch
 04-osmosis-version.patch
-example-shell-script-fails-syntax-check.patch
 disable-netty3.patch
+revert-protobuf-3.6.1-changes.patch
+protobuf-3.0.0.patch

gradle.properties

@@ -5,20 +5,24 @@ org.gradle.daemon=true
 
 # 3rd Party Library Versions
 dependencyVersionClassworlds=2.5.2
-dependencyVersionCommonsCodec=1.10
-dependencyVersionCommonsCompress=1.14
+dependencyVersionCommonsCodec=1.11
+dependencyVersionCommonsCompress=1.18
+# Should we be using Dbcp2?
 dependencyVersionCommonsDbcp=1.4
+dependencyVersionGuava=26.0-jre
 dependencyVersionJpf=1.5
+# JUnit 5 is available, some re-write required
 dependencyVersionJunit=4.12
-dependencyVersionMySql=6.0.6
+dependencyVersionMySql=8.0.12
+# Need to migrate to Netty 4
 dependencyVersionNetty=3.10.6.Final
 dependencyVersionPostGis=2.2.1
-dependencyVersionPostgreSql=42.1.4
-dependencyVersionProtobuf=3.4.0
-dependencyVersionSpring=4.3.11.RELEASE
-dependencyVersionWoodstoxCore=5.0.3
-dependencyVersionWoodstoxStax2=4.0.0
-dependencyVersionXerces=2.11.0
+dependencyVersionPostgreSql=42.2.5
+dependencyVersionProtobuf=3.6.1
+dependencyVersionSpring=5.1.0.RELEASE
+dependencyVersionWoodstoxCore=5.1.0
+dependencyVersionWoodstoxStax2=4.1
+dependencyVersionXerces=2.12.0
 
 # Builds are signed if the osmosisSigningEnabled property is set to true.
 # To enable signing, it is recommended to leave this file untouched and to

gradle/wrapper/gradle-wrapper.properties

 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.2-bin.zip

osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbWriter.java

@@ -154,6 +154,9 @@ public class ApidbWriter implements Sink, EntityProcessor {
     private static final int INSERT_BULK_ROW_COUNT_RELATION = 100;
     private static final int INSERT_BULK_ROW_COUNT_RELATION_TAG = 100;
     private static final int INSERT_BULK_ROW_COUNT_RELATION_MEMBER = 100;
+    
+    private static final int TRANSACTION_SIZE = 100000;
+    
     private String insertSqlSingleNode;
     private String insertSqlBulkNode;
     private String insertSqlSingleNodeTag;
@@ -190,6 +193,7 @@ public class ApidbWriter implements Sink, EntityProcessor {
     private long minWayId;
     private long maxRelationId;
     private long minRelationId;
+    private long transactionSizeCount;
     private final TileCalculator tileCalculator;
     private final MemberTypeRenderer memberTypeRenderer;
     private boolean initialized;
@@ -254,6 +258,7 @@ public class ApidbWriter implements Sink, EntityProcessor {
         minWayId = Long.MAX_VALUE;
         maxRelationId = Long.MIN_VALUE;
         minRelationId = Long.MAX_VALUE;
+        transactionSizeCount = 0;
 
         tileCalculator = new TileCalculator();
         memberTypeRenderer = new MemberTypeRenderer();
@@ -593,10 +598,16 @@ public class ApidbWriter implements Sink, EntityProcessor {
 
                 populateNodeParameters(bulkNodeStatement, prmIndex, node);
                 prmIndex += INSERT_PRM_COUNT_NODE;
+                transactionSizeCount++;
             }
 
             try {
                 bulkNodeStatement.executeUpdate();
+                
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
+                
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert nodes into the database.", e);
             }
@@ -623,6 +634,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 addNodeTags(node);
             }
         }
+        
+        dbCtx.commit();
     }
 
     /**
@@ -641,10 +654,15 @@ public class ApidbWriter implements Sink, EntityProcessor {
             for (int i = 0; i < INSERT_BULK_ROW_COUNT_NODE_TAG; i++) {
                 populateEntityTagParameters(bulkNodeTagStatement, prmIndex, nodeTagBuffer.remove(0));
                 prmIndex += INSERT_PRM_COUNT_NODE_TAG;
+                transactionSizeCount++;
             }
 
             try {
                 bulkNodeTagStatement.executeUpdate();
+                
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert node tags into the database.", e);
             }
@@ -661,6 +679,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 }
             }
         }
+        
+        dbCtx.commit();
     }
 
     /**
@@ -687,10 +707,15 @@ public class ApidbWriter implements Sink, EntityProcessor {
 
                 populateWayParameters(bulkWayStatement, prmIndex, way);
                 prmIndex += INSERT_PRM_COUNT_WAY;
+                transactionSizeCount++;
             }
 
             try {
                 bulkWayStatement.executeUpdate();
+                
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert ways into the database.", e);
             }
@@ -719,6 +744,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 addWayNodes(way);
             }
         }
+        
+        dbCtx.commit();
     }
 
     /**
@@ -737,10 +764,15 @@ public class ApidbWriter implements Sink, EntityProcessor {
             for (int i = 0; i < INSERT_BULK_ROW_COUNT_WAY_TAG; i++) {
                 populateEntityTagParameters(bulkWayTagStatement, prmIndex, wayTagBuffer.remove(0));
                 prmIndex += INSERT_PRM_COUNT_WAY_TAG;
+                transactionSizeCount++;
             }
 
             try {
                 bulkWayTagStatement.executeUpdate();
+                
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert way tags into the database.", e);
             }
@@ -757,6 +789,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 }
             }
         }
+        
+        dbCtx.commit();
     }
 
     /**
@@ -775,10 +809,14 @@ public class ApidbWriter implements Sink, EntityProcessor {
             for (int i = 0; i < INSERT_BULK_ROW_COUNT_WAY_NODE; i++) {
                 populateWayNodeParameters(bulkWayNodeStatement, prmIndex, wayNodeBuffer.remove(0));
                 prmIndex += INSERT_PRM_COUNT_WAY_NODE;
+                transactionSizeCount++;
             }
 
             try {
                 bulkWayNodeStatement.executeUpdate();
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert way nodes into the database.", e);
             }
@@ -795,6 +833,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 }
             }
         }
+        
+        dbCtx.commit();
     }
 
     /**
@@ -821,10 +861,14 @@ public class ApidbWriter implements Sink, EntityProcessor {
 
                 populateRelationParameters(bulkRelationStatement, prmIndex, relation);
                 prmIndex += INSERT_PRM_COUNT_RELATION;
+                transactionSizeCount++;
             }
 
             try {
                 bulkRelationStatement.executeUpdate();
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert relations into the database.", e);
             }
@@ -853,6 +897,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 addRelationMembers(relation);
             }
         }
+        
+        dbCtx.commit();
     }
 
     /**
@@ -872,10 +918,14 @@ public class ApidbWriter implements Sink, EntityProcessor {
             for (int i = 0; i < INSERT_BULK_ROW_COUNT_RELATION_TAG; i++) {
                 populateEntityTagParameters(bulkRelationTagStatement, prmIndex, relationTagBuffer.remove(0));
                 prmIndex += INSERT_PRM_COUNT_RELATION_TAG;
+                transactionSizeCount++;
             }
 
             try {
                 bulkRelationTagStatement.executeUpdate();
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert relation tags into the database.", e);
             }
@@ -892,6 +942,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 }
             }
         }
+        
+        dbCtx.commit();
     }
 
     /**
@@ -911,10 +963,14 @@ public class ApidbWriter implements Sink, EntityProcessor {
             for (int i = 0; i < INSERT_BULK_ROW_COUNT_RELATION_MEMBER; i++) {
                 populateRelationMemberParameters(bulkRelationMemberStatement, prmIndex, relationMemberBuffer.remove(0));
                 prmIndex += INSERT_PRM_COUNT_RELATION_MEMBER;
+                transactionSizeCount++;
             }
 
             try {
                 bulkRelationMemberStatement.executeUpdate();
+                if (transactionSizeCount % TRANSACTION_SIZE == 0) {
+                	dbCtx.commit();
+                }
             } catch (SQLException e) {
                 throw new OsmosisRuntimeException("Unable to bulk insert relation members into the database.", e);
             }
@@ -931,6 +987,8 @@ public class ApidbWriter implements Sink, EntityProcessor {
                 }
             }
         }
+        
+        dbCtx.commit();
     }
     
     

osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/impl/ChangeWriter.java

@@ -67,7 +67,7 @@ public class ChangeWriter implements Completable {
     	"INSERT INTO ways (way_id, version, timestamp, visible, changeset_id) VALUES (?, ?, ?, ?, ?)";
 
     private static final String UPDATE_SQL_WAY =
-    	"UPDATE current_ways SET timestamp = ?, visible = ?, changeset_id = ? WHERE id = ? AND version = ?";
+    	"UPDATE ways SET timestamp = ?, visible = ?, changeset_id = ? WHERE way_id = ? AND version = ?";
 
     private static final String SELECT_SQL_WAY_COUNT =
     	"SELECT Count(way_id) AS rowCount FROM ways WHERE way_id = ? AND version = ?";

osmosis-core/src/main/java/org/openstreetmap/osmosis/core/OsmosisException.java

@@ -14,7 +14,7 @@ public abstract class OsmosisException extends Exception {
 	
 	
 	/**
-     * Constructs a new exception with <code>null</code> as its detail message.
+     * Constructs a new exception.
      */
     public OsmosisException() {
 		super();
@@ -22,9 +22,7 @@ public abstract class OsmosisException extends Exception {
 	
 	
     /**
-     * Constructs a new exception with the specified detail message.  The
-     * cause is not initialized, and may subsequently be initialized by
-     * a call to {@link #initCause}.
+     * Constructs a new exception with the specified detail message.
      *
      * @param message the detail message.
      */
@@ -34,9 +32,7 @@ public abstract class OsmosisException extends Exception {
 	
     
     /**
-     * Constructs a new exception with the specified cause and a detail
-     * message of <tt>(cause==null ? null : cause.toString())</tt> (which
-     * typically contains the class and detail message of <tt>cause</tt>).
+     * Constructs a new exception with the specified cause.
      * 
      * @param cause the cause.
      */

osmosis-core/src/main/java/org/openstreetmap/osmosis/core/OsmosisRuntimeException.java

@@ -14,7 +14,7 @@ public class OsmosisRuntimeException extends RuntimeException {
 	
 	
 	/**
-     * Constructs a new exception with <code>null</code> as its detail message.
+     * Constructs a new exception.
      */
     public OsmosisRuntimeException() {
 		super();
@@ -22,9 +22,7 @@ public class OsmosisRuntimeException extends RuntimeException {
     
 	
     /**
-     * Constructs a new exception with the specified detail message.  The
-     * cause is not initialized, and may subsequently be initialized by
-     * a call to {@link #initCause}.
+     * Constructs a new exception with the specified detail message.
      *
      * @param message the detail message.
      */
@@ -34,9 +32,7 @@ public class OsmosisRuntimeException extends RuntimeException {
     
 	
     /**
-     * Constructs a new exception with the specified cause and a detail
-     * message of <tt>(cause==null ? null : cause.toString())</tt> (which
-     * typically contains the class and detail message of <tt>cause</tt>).
+     * Constructs a new exception with the specified cause.
      * 
      * @param cause the cause.
      */

osmosis-core/src/main/java/org/openstreetmap/osmosis/core/domain/v0_6/WayNode.java

@@ -14,8 +14,15 @@ import org.openstreetmap.osmosis.core.store.Storeable;
  */
 public class WayNode implements Comparable<WayNode>, Storeable {
 
-	private long nodeId;
+	/**
+	 * When sending way nodes with location information included through the Osmosis pipeline, a metadata key with this
+	 * value should be set to true so that downstream tasks know that they are able to use it.
+	 */
+	public static final String METADATA_KEY_LOCATION_INCLUDED = "way_node.location_included";
 	
+	private long nodeId;
+	private double latitude;
+	private double longitude;
 	
 	/**
 	 * Creates a new instance.
@@ -27,6 +34,21 @@ public class WayNode implements Comparable<WayNode>, Storeable {
 		this.nodeId = nodeId;
 	}
 	
+	/**
+	 * Creates a new instance.
+	 * 
+	 * @param nodeId
+	 *            The unique identifier of the node being referred to.
+	 * @param latitude
+	 *            The node's latitude
+	 * @param longitude
+	 *            The node's longitude
+	 */
+	public WayNode(long nodeId, double latitude, double longitude) {
+		this(nodeId);
+		this.latitude = latitude;
+		this.longitude = longitude;
+	}
 	
 	/**
 	 * Creates a new instance.
@@ -38,7 +60,7 @@ public class WayNode implements Comparable<WayNode>, Storeable {
 	 *            within the store.
 	 */
 	public WayNode(StoreReader sr, StoreClassRegister scr) {
-		this(sr.readLong());
+		this(sr.readLong(), sr.readDouble(), sr.readDouble());
 	}
 	
 	
@@ -47,6 +69,8 @@ public class WayNode implements Comparable<WayNode>, Storeable {
 	 */
 	public void store(StoreWriter sw, StoreClassRegister scr) {
 		sw.writeLong(nodeId);
+		sw.writeDouble(latitude);
+		sw.writeDouble(longitude);
 	}
 	
 	
@@ -73,6 +97,19 @@ public class WayNode implements Comparable<WayNode>, Storeable {
 		}
 	}
 	
+	/**
+	 * @return The latitude (if the PBF file was built with osmium command 'add-locations-to-ways')
+	 */
+	public double getLatitude() {
+		return latitude;
+	}
+
+	/**
+	 * @return The longitude (if the PBF file was built with osmium command 'add-locations-to-ways')
+	 */
+	public double getLongitude() {
+		return longitude;
+	}
 
 	/**
 	 * @return The nodeId.

osmosis-core/src/main/java/org/openstreetmap/osmosis/core/store/EndOfStoreException.java

@@ -15,7 +15,7 @@ public class EndOfStoreException extends OsmosisRuntimeException {
 	
 	
 	/**
-     * Constructs a new exception with <code>null</code> as its detail message.
+     * Constructs a new exception.
      */
     public EndOfStoreException() {
 		super();
@@ -23,9 +23,7 @@ public class EndOfStoreException extends OsmosisRuntimeException {
     
 	
     /**
-     * Constructs a new exception with the specified detail message.  The
-     * cause is not initialized, and may subsequently be initialized by
-     * a call to {@link #initCause}.
+     * Constructs a new exception with the specified detail message.
      *
      * @param message the detail message.
      */
@@ -35,9 +33,7 @@ public class EndOfStoreException extends OsmosisRuntimeException {
     
 	
     /**
-     * Constructs a new exception with the specified cause and a detail
-     * message of <tt>(cause==null ? null : cause.toString())</tt> (which
-     * typically contains the class and detail message of <tt>cause</tt>).
+     * Constructs a new exception with the specified cause.
      * 
      * @param cause the cause.
      */