Commit 6ce404e7 authored by Bas Couwenberg's avatar Bas Couwenberg

Imported Upstream version 2.1.3

parent 373bcb54
......@@ -13,6 +13,24 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Fixed
## [2.1.3] - 2016-03-30
### Added
- Add verbose option to osmcoastline_filter.
- osmcoastline_filter now shows memory used in verbose mode.
### Changed
- Optimized osmcoastline_filter program.
- Use more features from newest libosmium.
### Fixed
- Setting the sqlite output to unsynchronized speeds up writing to database.
- Now also works on GDAL 2. Fixes an error in the transaction handling.
## [2.1.2] - 2016-01-05
### Added
......@@ -59,7 +77,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
- Added man pages
[unreleased]: https://github.com/osmcode/osmium-tool/compare/v2.1.2...HEAD
[unreleased]: https://github.com/osmcode/osmium-tool/compare/v2.1.3...HEAD
[2.1.3]: https://github.com/osmcode/osmium-tool/compare/v2.1.2...v2.1.3
[2.1.2]: https://github.com/osmcode/osmium-tool/compare/v2.1.1...v2.1.2
[2.1.1]: https://github.com/osmcode/osmium-tool/compare/v2.1.0...v2.1.1
[2.1.0]: https://github.com/osmcode/osmium-tool/compare/v2.0.1...v2.1.0
......
......@@ -22,7 +22,7 @@ project(osmcoastline)
set(OSMCOASTLINE_VERSION_MAJOR 2)
set(OSMCOASTLINE_VERSION_MINOR 1)
set(OSMCOASTLINE_VERSION_PATCH 2)
set(OSMCOASTLINE_VERSION_PATCH 3)
set(OSMCOASTLINE_VERSION
${OSMCOASTLINE_VERSION_MAJOR}.${OSMCOASTLINE_VERSION_MINOR}.${OSMCOASTLINE_VERSION_PATCH})
......
......@@ -347,18 +347,22 @@ namespace gdalcpp {
}
Layer& start_transaction() {
#if GDAL_VERSION_MAJOR < 2
OGRErr result = m_layer->StartTransaction();
if (result != OGRERR_NONE) {
throw gdal_error(std::string("starting transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name());
}
#endif
return *this;
}
Layer& commit_transaction() {
#if GDAL_VERSION_MAJOR < 2
OGRErr result = m_layer->CommitTransaction();
if (result != OGRERR_NONE) {
throw gdal_error(std::string("committing transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name());
}
#endif
return *this;
}
......
......@@ -25,6 +25,8 @@
#include <unistd.h>
#include <osmium/io/any_input.hpp>
#include <osmium/util/memory.hpp>
#include <osmium/util/verbose_output.hpp>
#include <osmium/visitor.hpp>
#include "return_codes.hpp"
......@@ -38,7 +40,6 @@
#include "coastline_handlers.hpp"
#include "srs.hpp"
#include "util.hpp"
#include "verbose_output.hpp"
// The global SRS object is used in many places to transform
// from WGS84 to the output SRS etc.
......@@ -104,38 +105,10 @@ polygon_vector_type create_polygons(CoastlineRingCollection& coastline_rings, Ou
/* ================================================== */
std::pair<int, int> get_memory_usage() {
char filename[100];
sprintf(filename, "/proc/%d/status", getpid());
std::ifstream status_file(filename);
std::string line;
int vmpeak = 0;
int vmsize = 0;
if (status_file.is_open()) {
while (! status_file.eof() ) {
std::getline(status_file, line);
if (line.substr(0, 6) == "VmPeak") {
int f = line.find_first_of("0123456789");
int l = line.find_last_of("0123456789");
vmpeak = atoi(line.substr(f, l-f+1).c_str());
}
if (line.substr(0, 6) == "VmSize") {
int f = line.find_first_of("0123456789");
int l = line.find_last_of("0123456789");
vmsize = atoi(line.substr(f, l-f+1).c_str());
}
}
status_file.close();
}
return std::make_pair(vmsize / 1024, vmpeak / 1024);
}
std::string memory_usage() {
std::pair<int, int> mem = get_memory_usage();
osmium::MemoryUsage mem;
std::ostringstream s;
s << "Memory used currently: " << mem.first << " MB (Peak was: " << mem.second << " MB).\n";
s << "Memory used: current: " << mem.current() << " MBytes, peak: " << mem.peak() << " MBytes\n";
return s.str();
}
......@@ -152,11 +125,12 @@ int main(int argc, char *argv[]) {
// The vout object is an output stream we can write to instead of
// std::cerr. Nothing is written if we are not in verbose mode.
// The running time will be prepended to output lines.
VerboseOutput vout(options.verbose);
osmium::util::VerboseOutput vout(options.verbose);
debug = options.debug;
CPLSetConfigOption("OGR_ENABLE_PARTIAL_REPROJECTION", "TRUE");
CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF");
vout << "Using SRS " << options.epsg << " for output. (Change with the --srs/s option.)\n";
if (!srs.set_output(options.epsg)) {
std::cerr << "Setting up output transformation failed\n";
......@@ -320,7 +294,7 @@ int main(int argc, char *argv[]) {
vout << memory_usage();
vout << "Committing database transactions...\n";
output_database.set_meta(vout.runtime(), get_memory_usage().second, stats);
output_database.set_meta(vout.runtime(), osmium::MemoryUsage{}.peak(), stats);
output_database.commit();
vout << "All done.\n";
vout << memory_usage();
......
......@@ -20,9 +20,10 @@
*/
#include <algorithm>
#include <iostream>
#include <set>
#include <getopt.h>
#include <iostream>
#include <string>
#include <vector>
#include <osmium/io/any_input.hpp>
#include <osmium/io/input_iterator.hpp>
......@@ -30,6 +31,8 @@
#include <osmium/io/pbf_output.hpp>
#include <osmium/handler.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/util/memory.hpp>
#include <osmium/util/verbose_output.hpp>
#include "return_codes.hpp"
......@@ -38,22 +41,25 @@ void print_help() {
<< "\nOptions:\n"
<< " -h, --help - This help message\n"
<< " -o, --output=OSMFILE - Where to write output (default: none)\n"
<< " -v, --verbose - Verbose output\n"
<< " -V, --version - Show version and exit\n"
<< "\n";
}
int main(int argc, char* argv[]) {
std::string output_filename;
bool verbose = false;
static struct option long_options[] = {
{"help", no_argument, 0, 'h'},
{"output", required_argument, 0, 'o'},
{"verbose", no_argument, 0, 'v'},
{"version", no_argument, 0, 'V'},
{0, 0, 0, 0}
};
while (1) {
int c = getopt_long(argc, argv, "ho:V", long_options, 0);
int c = getopt_long(argc, argv, "ho:vV", long_options, 0);
if (c == -1)
break;
......@@ -64,6 +70,9 @@ int main(int argc, char* argv[]) {
case 'o':
output_filename = optarg;
break;
case 'v':
verbose = true;
break;
case 'V':
std::cout << "osmcoastline_filter version " OSMCOASTLINE_VERSION "\n"
<< "Copyright (C) 2012-2016 Jochen Topf <jochen@topf.org>\n"
......@@ -76,6 +85,11 @@ int main(int argc, char* argv[]) {
}
}
// The vout object is an output stream we can write to instead of
// std::cerr. Nothing is written if we are not in verbose mode.
// The running time will be prepended to output lines.
osmium::util::VerboseOutput vout(verbose);
if (output_filename.empty()) {
std::cerr << "Missing -o/--output=OSMFILE option\n";
exit(return_code_cmdline);
......@@ -98,6 +112,7 @@ int main(int argc, char* argv[]) {
std::vector<osmium::object_id_type> ids;
vout << "Reading ways (1st pass through input file)...\n";
{
osmium::io::Reader reader(infile, osmium::osm_entity_bits::way);
auto ways = osmium::io::make_input_iterator_range<const osmium::Way>(reader);
......@@ -113,17 +128,32 @@ int main(int argc, char* argv[]) {
reader.close();
}
vout << "Preparing node ID list...\n";
std::sort(ids.begin(), ids.end());
auto last = std::unique(ids.begin(), ids.end());
vout << "Reading nodes (2nd pass through input file)...\n";
{
osmium::io::Reader reader(infile, osmium::osm_entity_bits::node);
auto nodes = osmium::io::make_input_iterator_range<const osmium::Node>(reader);
std::copy_if(nodes.cbegin(), nodes.cend(), output_it, [&ids, &last](const osmium::Node& node){
auto first = ids.begin();
std::copy_if(nodes.cbegin(), nodes.cend(), output_it, [&first, &last](const osmium::Node& node){
while (*first < node.id() && first != last) {
++first;
}
if (node.id() == *first) {
if (first != last) {
++first;
}
return true;
}
const char* natural = node.get_value_by_key("natural");
return std::binary_search(ids.begin(), last, node.id()) ||
(natural && !strcmp(natural, "coastline"));
return natural && !strcmp(natural, "coastline");
});
reader.close();
}
......@@ -132,5 +162,12 @@ int main(int argc, char* argv[]) {
std::cerr << "io error: " << e.what() << "'\n";
exit(return_code_fatal);
}
vout << "All done.\n";
osmium::MemoryUsage mem;
if (mem.current() > 0) {
vout << "Memory used: current: " << mem.current() << " MBytes\n"
<< " peak: " << mem.peak() << " MBytes\n";
}
}
......@@ -51,7 +51,7 @@ public:
CoastlineWaysHandler(const std::string& db_filename) :
m_length(0.0),
m_dataset("SQLite", db_filename, gdalcpp::SRS{}, {"SPATIALITE=TRUE", "OGR_SQLITE_SYNCHRONOUS=OFF", "INIT_WITH_EPSG=no" }),
m_dataset("SQLite", db_filename, gdalcpp::SRS{}, {"SPATIALITE=TRUE", "INIT_WITH_EPSG=no" }),
m_layer_ways(m_dataset, "ways", wkbLineString) {
m_layer_ways.add_field("way_id", OFTString, 10);
......@@ -110,6 +110,8 @@ int main(int argc, char* argv[]) {
exit(return_code_cmdline);
}
CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF");
std::string input_osm_filename { argv[1] };
std::string output_db_filename { "coastline-ways.db" };
......
......@@ -33,7 +33,7 @@
OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_index) :
m_with_index(with_index),
m_srs(srs),
m_dataset("SQLite", outdb, gdalcpp::SRS(*srs.out()), { "SPATIALITE=TRUE", "OGR_SQLITE_SYNCHRONOUS=OFF", "INIT_WITH_EPSG=no" }),
m_dataset("SQLite", outdb, gdalcpp::SRS(*srs.out()), { "SPATIALITE=TRUE", "INIT_WITH_EPSG=no" }),
m_layer_error_points(m_dataset, "error_points", wkbPoint, layer_options()),
m_layer_error_lines(m_dataset, "error_lines", wkbLineString, layer_options()),
m_layer_rings(m_dataset, "rings", wkbPolygon, layer_options()),
......@@ -43,11 +43,9 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind
m_layer_error_points.add_field("osm_id", OFTString, 10);
m_layer_error_points.add_field("error", OFTString, 16);
m_layer_error_points.start_transaction();
m_layer_error_lines.add_field("osm_id", OFTString, 10);
m_layer_error_lines.add_field("error", OFTString, 16);
m_layer_error_lines.start_transaction();
m_layer_rings.add_field("osm_id", OFTString, 10);
m_layer_rings.add_field("nways", OFTInteger, 6);
......@@ -55,13 +53,6 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind
m_layer_rings.add_field("fixed", OFTInteger, 1);
m_layer_rings.add_field("land", OFTInteger, 1);
m_layer_rings.add_field("valid", OFTInteger, 1);
m_layer_rings.start_transaction();
m_layer_land_polygons.start_transaction();
m_layer_water_polygons.start_transaction();
m_layer_lines.start_transaction();
m_dataset.exec("CREATE TABLE options (overlap REAL, close_distance REAL, max_points_in_polygons INTEGER, split_large_polygons INTEGER)");
m_dataset.exec("CREATE TABLE meta ("
......@@ -76,6 +67,14 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind
"num_rings_turned_around INTEGER, "
"num_land_polygons_before_split INTEGER, "
"num_land_polygons_after_split INTEGER)");
m_dataset.start_transaction();
m_layer_rings.start_transaction();
m_layer_land_polygons.start_transaction();
m_layer_water_polygons.start_transaction();
m_layer_lines.start_transaction();
m_layer_error_points.start_transaction();
m_layer_error_lines.start_transaction();
}
void OutputDatabase::set_options(const Options& options) {
......@@ -119,12 +118,13 @@ void OutputDatabase::set_meta(int runtime, int memory_usage, const Stats& stats)
}
void OutputDatabase::commit() {
m_layer_error_lines.commit_transaction();
m_layer_error_points.commit_transaction();
m_layer_lines.commit_transaction();
m_layer_water_polygons.commit_transaction();
m_layer_land_polygons.commit_transaction();
m_layer_rings.commit_transaction();
m_layer_error_lines.commit_transaction();
m_layer_error_points.commit_transaction();
m_dataset.commit_transaction();
}
void OutputDatabase::add_error_point(std::unique_ptr<OGRPoint>&& point, const char* error, osmium::object_id_type id) {
......
#ifndef VERBOSE_OUTPUT_HPP
#define VERBOSE_OUTPUT_HPP
#include <iostream>
#include <sstream>
class VerboseOutput {
time_t m_start;
bool m_verbose;
bool m_newline;
public:
VerboseOutput(bool verbose) :
m_start(time(nullptr)),
m_verbose(verbose),
m_newline(true) {
}
int runtime() const {
return time(nullptr) - m_start;
}
void start_line() {
if (m_newline) {
int elapsed = time(nullptr) - m_start;
char timestr[20];
snprintf(timestr, sizeof(timestr)-1, "[%2d:%02d] ", elapsed / 60, elapsed % 60);
std::cerr << timestr;
m_newline = false;
}
}
template<typename T>
friend VerboseOutput& operator<<(VerboseOutput& out, T t) {
if (out.m_verbose) {
std::ostringstream o;
o << t;
out.start_line();
std::cerr << o.str();
if (o.str()[o.str().size()-1] == '\n') {
out.m_newline = true;
}
}
return out;
}
}; // class VerboseOutput
#endif // VERBOSE_OUTPUT_HPP
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment