Skip to content
GitLab
Explore
Sign in
Register
Commits on Source (2)
New upstream version 0.0.0+svn592
· 8db49148
Bas Couwenberg
authored
Jan 01, 2019
8db49148
New upstream version 0.0.0+svn595
· 8b04825e
Bas Couwenberg
authored
Dec 01, 2019
8b04825e
Hide whitespace changes
Inline
Side-by-side
resources/splitter-version.properties
View file @
8b04825e
svn.version
:
59
1
build.timestamp
:
201
8-03-09T14:31:20
+0000
svn.version
:
59
5
build.timestamp
:
201
9-11-16T09:57:34
+0000
src/uk/me/parabola/splitter/AbstractMapProcessor.java
View file @
8b04825e
...
...
@@ -18,39 +18,57 @@ import java.util.concurrent.BlockingQueue;
public
abstract
class
AbstractMapProcessor
implements
MapProcessor
{
public
static
final
int
UNASSIGNED
=
Short
.
MIN_VALUE
;
@Override
public
boolean
skipTags
(){
return
false
;
}
@Override
public
boolean
skipNodes
(){
return
false
;
}
@Override
public
boolean
skipWays
(){
return
false
;
}
@Override
public
boolean
skipRels
(){
return
false
;
}
public
void
boundTag
(
Area
bounds
){}
@Override
public
void
boundTag
(
Area
bounds
)
{
}
@Override
public
void
processNode
(
Node
n
)
{
}
public
void
processNode
(
Node
n
){}
@Override
public
void
processWay
(
Way
w
)
{
}
public
void
processWay
(
Way
w
){}
public
void
processRelation
(
Relation
r
)
{
}
@Override
public
void
processRelation
(
Relation
r
)
{
}
public
boolean
endMap
(){
@Override
public
boolean
endMap
()
{
return
true
;
}
@Override
public
int
getPhase
()
{
return
1
;
}
public
void
startFile
()
{};
@Override
public
void
startFile
()
{
}
/**
* Simple method that allows all processors to use the producer/consumer pattern
*/
@Override
public
final
boolean
consume
(
BlockingQueue
<
OSMMessage
>
queue
)
{
while
(
true
)
{
try
{
...
...
@@ -58,6 +76,8 @@ public abstract class AbstractMapProcessor implements MapProcessor {
switch
(
msg
.
type
)
{
case
ELEMENTS:
for
(
Element
el
:
msg
.
elements
)
{
if
(
el
==
null
)
break
;
if
(
el
instanceof
Node
)
processNode
((
Node
)
el
);
else
if
(
el
instanceof
Way
)
...
...
src/uk/me/parabola/splitter/Area.java
View file @
8b04825e
...
...
@@ -92,13 +92,11 @@ public class Area {
}
public
boolean
verify
(){
if
(
minLat
>
maxLat
||
minLong
>
maxLong
return
!
(
minLat
>
maxLat
||
minLong
>
maxLong
||
minLong
<
Utils
.
MIN_LON_MAP_UNITS
||
maxLong
>
Utils
.
MAX_LON_MAP_UNITS
||
minLat
<
Utils
.
MIN_LAT_MAP_UNITS
||
maxLat
>
Utils
.
MAX_LAT_MAP_UNITS
)
return
false
;
return
true
;
||
maxLat
>
Utils
.
MAX_LAT_MAP_UNITS
);
}
...
...
@@ -155,13 +153,13 @@ public class Area {
return
maxLat
-
minLat
;
}
@Override
public
String
toString
()
{
return
"("
+
Utils
.
toDegrees
(
minLat
)
+
','
+
Utils
.
toDegrees
(
minLong
)
+
") to ("
+
Utils
.
toDegrees
(
maxLat
)
+
','
+
Utils
.
toDegrees
(
maxLong
)
+
')'
;
+
Utils
.
toDegrees
(
maxLong
)
+
')'
;
}
public
String
toHexString
()
{
...
...
src/uk/me/parabola/splitter/AreaDictionary.java
View file @
8b04825e
...
...
@@ -64,7 +64,6 @@ public class AreaDictionary {
}
findSimpleNeigbours
(
rectangles
,
areaSets
);
System
.
out
.
println
(
"cached "
+
simpleNeighbours
.
size
()
+
" combinations of areas that form rectangles."
);
return
;
}
/**
...
...
@@ -105,18 +104,14 @@ public class AreaDictionary {
Rectangle
r1
=
rectangles
.
get
(
i
);
for
(
int
j
=
i
+
1
;
j
<
rectangles
.
size
();
j
++)
{
Rectangle
r2
=
rectangles
.
get
(
j
);
boolean
isSimple
=
false
;
if
(
r1
.
y
==
r2
.
y
&&
r1
.
height
==
r2
.
height
&&
(
r1
.
x
==
r2
.
getMaxX
()
||
r2
.
x
==
r1
.
getMaxX
()))
isSimple
=
true
;
else
if
(
r1
.
x
==
r2
.
x
&&
r1
.
width
==
r2
.
width
&&
(
r1
.
y
==
r2
.
getMaxY
()
||
r2
.
y
==
r1
.
getMaxY
()))
isSimple
=
true
;
boolean
isSimple
=
r1
.
y
==
r2
.
y
&&
r1
.
height
==
r2
.
height
&&
(
r1
.
x
==
r2
.
getMaxX
()
||
r2
.
x
==
r1
.
getMaxX
())
||
(
r1
.
x
==
r2
.
x
&&
r1
.
width
==
r2
.
width
&&
(
r1
.
y
==
r2
.
getMaxY
()
||
r2
.
y
==
r1
.
getMaxY
()));
if
(
isSimple
)
{
AreaSet
simpleNeighbour
=
new
AreaSet
(
areaSets
.
get
(
i
));
simpleNeighbour
.
or
(
areaSets
.
get
(
j
));
if
(
simpleNeighbour
.
cardinality
()
<=
10
&&
!
simpleNeighbours
.
contains
(
simpleNeighbour
))
{
simpleNeighbours
.
add
(
simpleNeighbour
);
// System.out.println("simple neighbor: " +
// getMapIds(simpleNeighbour));
Rectangle
pair
=
new
Rectangle
(
r1
);
pair
.
add
(
r2
);
newRectangles
.
add
(
pair
);
...
...
@@ -159,7 +154,7 @@ public class AreaDictionary {
}
public
boolean
mayCross
(
AreaSet
areaSet
)
{
return
simpleNeighbours
.
contains
(
areaSet
)
==
false
;
return
!
simpleNeighbours
.
contains
(
areaSet
);
}
public
Area
getArea
(
int
idx
)
{
...
...
src/uk/me/parabola/splitter/AreaGrid.java
View file @
8b04825e
...
...
@@ -38,23 +38,26 @@ public class AreaGrid implements AreaIndex{
grid
=
new
Grid
(
null
,
null
);
}
@Override
public
Area
getBounds
(){
return
grid
.
getBounds
();
}
@Override
public
AreaGridResult
get
(
final
Node
n
){
return
grid
.
get
(
n
.
getMapLat
(),
n
.
getMapLon
());
}
@Override
public
AreaGridResult
get
(
int
lat
,
int
lon
){
return
grid
.
get
(
lat
,
lon
);
}
private
class
Grid
{
private
final
static
int
TOP_GRID_DIM_LON
=
512
;
private
final
static
int
TOP_GRID_DIM_LAT
=
512
;
private
final
static
int
SUB_GRID_DIM_LON
=
32
;
private
final
static
int
SUB_GRID_DIM_LAT
=
32
;
private
static
final
int
TOP_GRID_DIM_LON
=
512
;
private
static
final
int
TOP_GRID_DIM_LAT
=
512
;
private
static
final
int
SUB_GRID_DIM_LON
=
32
;
private
static
final
int
SUB_GRID_DIM_LAT
=
32
;
private
static
final
int
MIN_GRID_LAT
=
2048
;
private
static
final
int
MIN_GRID_LON
=
2048
;
private
static
final
int
MAX_TESTS
=
10
;
...
...
@@ -101,13 +104,13 @@ public class AreaGrid implements AreaIndex{
*/
private
int
fillGrid
(
AreaSet
usedAreas
)
{
int
gridStepLon
,
gridStepLat
;
if
(
bounds
==
null
){
if
(
bounds
==
null
)
{
// calculate grid area
Area
tmpBounds
=
null
;
for
(
int
i
=
0
;
i
<
areaDictionary
.
getNumOfAreas
();
i
++)
{
Area
extBounds
=
areaDictionary
.
getExtendedArea
(
i
);
if
(
usedAreas
==
null
||
usedAreas
.
get
(
i
))
tmpBounds
=
(
tmpBounds
==
null
)
?
extBounds
:
tmpBounds
.
add
(
extBounds
);
tmpBounds
=
(
tmpBounds
==
null
)
?
extBounds
:
tmpBounds
.
add
(
extBounds
);
}
if
(
tmpBounds
==
null
)
return
0
;
...
...
@@ -168,21 +171,19 @@ public class AreaGrid implements AreaIndex{
areaSet
.
lock
();
if
(
testGrid
[
lon
].
get
(
lat
)){
int
numTests
=
areaSet
.
cardinality
();
if
(
numTests
>
MAX_TESTS
){
if
(
gridStepLat
>
MIN_GRID_LAT
&&
gridStepLon
>
MIN_GRID_LON
){
Area
gridPart
=
new
Area
(
gridMinLat
+
gridStepLat
*
lat
,
gridMinLon
+
gridStepLon
*
lon
,
gridMinLat
+
gridStepLat
*
(
lat
+
1
),
gridMinLon
+
gridStepLon
*
(
lon
+
1
));
// late allocation
if
(
subGrid
==
null
)
subGrid
=
new
Grid
[
gridDimLon
+
1
][
gridDimLat
+
1
];
usedSubGridElems
++;
if
(
numTests
>
MAX_TESTS
&&
gridStepLat
>
MIN_GRID_LAT
&&
gridStepLon
>
MIN_GRID_LON
){
Area
gridPart
=
new
Area
(
gridMinLat
+
gridStepLat
*
lat
,
gridMinLon
+
gridStepLon
*
lon
,
gridMinLat
+
gridStepLat
*
(
lat
+
1
),
gridMinLon
+
gridStepLon
*
(
lon
+
1
));
// late allocation
if
(
subGrid
==
null
)
subGrid
=
new
Grid
[
gridDimLon
+
1
][
gridDimLat
+
1
];
usedSubGridElems
++;
subGrid
[
lon
][
lat
]
=
new
Grid
(
areaSet
,
gridPart
);
numTests
=
subGrid
[
lon
][
lat
].
getMaxCompares
()
+
1
;
maxAreaSearch
=
Math
.
max
(
maxAreaSearch
,
numTests
);
continue
;
}
subGrid
[
lon
][
lat
]
=
new
Grid
(
areaSet
,
gridPart
);
numTests
=
subGrid
[
lon
][
lat
].
getMaxCompares
()
+
1
;
maxAreaSearch
=
Math
.
max
(
maxAreaSearch
,
numTests
);
continue
;
}
maxAreaSearch
=
Math
.
max
(
maxAreaSearch
,
numTests
);
}
...
...
src/uk/me/parabola/splitter/AreaList.java
View file @
8b04825e
...
...
@@ -194,25 +194,24 @@ public class AreaList {
try
(
PrintWriter
pw
=
new
PrintWriter
(
filename
))
{
pw
.
println
(
"area"
);
for
(
int
i
=
0
;
i
<
shapes
.
size
();
i
++){
for
(
int
i
=
0
;
i
<
shapes
.
size
();
i
++)
{
List
<
Point
>
shape
=
shapes
.
get
(
i
);
if
(
Utils
.
clockwise
(
shape
))
pw
.
println
(
i
+
1
);
else
pw
.
println
(
"!"
+
(
i
+
1
));
pw
.
println
(
i
+
1
);
else
pw
.
println
(
"!"
+
(
i
+
1
));
Point
point
=
null
;
for
(
int
j
=
0
;
j
<
shape
.
size
();
j
++){
for
(
int
j
=
0
;
j
<
shape
.
size
();
j
++)
{
point
=
shape
.
get
(
j
);
if
(
j
>
0
&&
j
+
1
<
shape
.
size
()){
if
(
j
>
0
&&
j
+
1
<
shape
.
size
())
{
Point
lastPoint
=
shape
.
get
(
j
-
1
);
Point
nextPoint
=
shape
.
get
(
j
+
1
);
if
(
point
.
x
==
nextPoint
.
x
&&
point
.
x
==
lastPoint
.
x
)
continue
;
if
(
point
.
y
==
nextPoint
.
y
&&
point
.
y
==
lastPoint
.
y
)
Point
nextPoint
=
shape
.
get
(
j
+
1
);
if
((
point
.
x
==
nextPoint
.
x
&&
point
.
x
==
lastPoint
.
x
)
||
(
point
.
y
==
nextPoint
.
y
&&
point
.
y
==
lastPoint
.
y
))
continue
;
}
pw
.
format
(
Locale
.
ROOT
,
" %f %f%n"
,
Utils
.
toDegrees
(
point
.
x
)
,
Utils
.
toDegrees
(
point
.
y
));
pw
.
format
(
Locale
.
ROOT
,
" %f %f%n"
,
Utils
.
toDegrees
(
point
.
x
),
Utils
.
toDegrees
(
point
.
y
));
}
pw
.
println
(
"END"
);
}
...
...
@@ -322,17 +321,16 @@ public class AreaList {
* @param polygons
* @param kmlOutputFile
* @param outputType
* @throws IOException
*/
public
void
writeListFiles
(
File
fileOutputDir
,
List
<
PolygonDesc
>
polygons
,
String
kmlOutputFile
,
String
outputType
)
throws
IOException
{
for
(
PolygonDesc
pd
:
polygons
){
public
void
writeListFiles
(
File
fileOutputDir
,
List
<
PolygonDesc
>
polygons
,
String
kmlOutputFile
,
String
outputType
)
{
for
(
PolygonDesc
pd
:
polygons
)
{
List
<
uk
.
me
.
parabola
.
splitter
.
Area
>
areasPart
=
new
ArrayList
<>();
for
(
uk
.
me
.
parabola
.
splitter
.
Area
a
:
areas
){
for
(
uk
.
me
.
parabola
.
splitter
.
Area
a
:
areas
)
{
if
(
pd
.
getArea
().
intersects
(
a
.
getRect
()))
areasPart
.
add
(
a
);
}
if
(
kmlOutputFile
!=
null
){
if
(
kmlOutputFile
!=
null
)
{
File
out
=
new
File
(
kmlOutputFile
);
String
kmlOutputFilePart
=
pd
.
getName
()
+
"-"
+
out
.
getName
();
if
(
out
.
getParent
()
!=
null
)
...
...
@@ -346,9 +344,9 @@ public class AreaList {
AreaList
al
=
new
AreaList
(
areasPart
,
null
);
al
.
setGeoNamesFile
(
geoNamesFile
);
al
.
writePoly
(
new
File
(
fileOutputDir
,
pd
.
getName
()
+
"-"
+
"areas.poly"
).
getPath
());
al
.
writeArgsFile
(
new
File
(
fileOutputDir
,
pd
.
getName
()
+
"-"
+
"template.args"
).
getPath
(),
outputType
,
pd
.
getMapId
());
al
.
writeArgsFile
(
new
File
(
fileOutputDir
,
pd
.
getName
()
+
"-"
+
"template.args"
).
getPath
(),
outputType
,
pd
.
getMapId
());
}
}
}
}
src/uk/me/parabola/splitter/Main.java
View file @
8b04825e
...
...
@@ -431,7 +431,7 @@ public class Main {
// plausibility checks and default handling
if
(
keepComplete
)
{
if
(
fileNameList
.
size
()
>
1
)
{
System
.
err
.
println
(
"Warning: --keep-complete is only used for the first input file."
);
System
.
err
.
println
(
"Warning: --keep-complete is only used for the first input file.
Further files must use higher ids.
"
);
}
if
(
overlapAmount
>
0
)
{
System
.
err
.
println
(
"Warning: --overlap is used in combination with --keep-complete=true "
);
...
...
@@ -461,12 +461,9 @@ public class Main {
}
private
static
void
checkOptionalFileOption
(
String
fname
,
String
option
)
{
if
(
fname
!=
null
)
{
if
(
testAndReportFname
(
fname
,
option
)
==
false
)
{
throw
new
IllegalArgumentException
();
}
if
(
fname
!=
null
&&
!
testAndReportFname
(
fname
,
option
))
{
throw
new
IllegalArgumentException
();
}
}
private
OSMWriter
[]
createWriters
(
List
<
Area
>
areas
)
{
...
...
src/uk/me/parabola/splitter/MultiTileProcessor.java
View file @
8b04825e
...
...
@@ -122,12 +122,10 @@ class MultiTileProcessor extends AbstractMapProcessor {
@Override
public
void
processNode
(
Node
node
)
{
if
(
phase
==
PHASE3_NODES_AND_WAYS
){
if
(
neededNodes
.
get
(
node
.
getId
())){
storeCoord
(
node
);
// return memory to GC
neededNodes
.
clear
(
node
.
getId
());
}
if
(
phase
==
PHASE3_NODES_AND_WAYS
&&
neededNodes
.
get
(
node
.
getId
()))
{
storeCoord
(
node
);
// return memory to GC
neededNodes
.
clear
(
node
.
getId
());
}
}
...
...
@@ -360,15 +358,13 @@ class MultiTileProcessor extends AbstractMapProcessor {
continue
;
for
(
int
i
=
0
;
i
<
rel
.
numMembers
;
i
++){
long
memId
=
rel
.
memRefs
[
i
];
if
(
rel
.
memTypes
[
i
]
==
MEM_REL_TYPE
){
if
(
problemRels
.
get
(
memId
)){
problemRels
.
set
(
rel
.
getId
());
rel
.
setAddedAsParent
();
System
.
out
.
println
(
"Adding parent of problem rel "
+
memId
+
" to problem list: "
+
rel
.
getId
());
changed
=
true
;
break
;
}
}
if
(
rel
.
memTypes
[
i
]
==
MEM_REL_TYPE
&&
problemRels
.
get
(
memId
))
{
problemRels
.
set
(
rel
.
getId
());
rel
.
setAddedAsParent
();
System
.
out
.
println
(
"Adding parent of problem rel "
+
memId
+
" to problem list: "
+
rel
.
getId
());
changed
=
true
;
break
;
}
}
}
if
(!
changed
)
...
...
@@ -704,9 +700,8 @@ class MultiTileProcessor extends AbstractMapProcessor {
if
(
numWriters
==
0
)
needsCrossTileCheck
=
true
;
else
if
(
numWriters
>
1
){
if
(
dataStorer
.
getAreaDictionary
().
mayCross
(
writerSet
))
needsCrossTileCheck
=
true
;
else
if
(
numWriters
>
1
&&
dataStorer
.
getAreaDictionary
().
mayCross
(
writerSet
))
{
needsCrossTileCheck
=
true
;
}
}
if
(
needsCrossTileCheck
){
...
...
@@ -875,11 +870,9 @@ class MultiTileProcessor extends AbstractMapProcessor {
else
mpBbox
.
add
(
wayBbox
);
if
(
mpBbox
.
x
<
0
&&
mpBbox
.
getMaxX
()
>
0
&&
mpBbox
.
width
>=
PROBLEM_WIDTH
){
if
(
complainedAboutSize
==
false
){
System
.
out
.
println
(
"rel crosses -180/180: "
+
rel
.
getId
());
complainedAboutSize
=
true
;
}
if
(!
complainedAboutSize
&&
mpBbox
.
x
<
0
&&
mpBbox
.
getMaxX
()
>
0
&&
mpBbox
.
width
>=
PROBLEM_WIDTH
){
System
.
out
.
println
(
"rel crosses -180/180: "
+
rel
.
getId
());
complainedAboutSize
=
true
;
}
}
...
...
src/uk/me/parabola/splitter/OSMFileHandler.java
View file @
8b04825e
...
...
@@ -55,9 +55,6 @@ public class OSMFileHandler {
private
int
maxThreads
=
1
;
/** if this is true we may not want to use producer/consumer pattern */
private
MapProcessor
realProcessor
;
public
void
setFileNames
(
List
<
String
>
filenames
)
{
this
.
filenames
=
filenames
;
}
...
...
@@ -132,18 +129,19 @@ public class OSMFileHandler {
RuntimeException
exception
=
null
;
public
boolean
execute
(
MapProcessor
processor
)
{
realProcessor
=
processor
;
if
(
maxThreads
==
1
)
return
process
(
processor
);
// use two threads
// use two threads
BlockingQueue
<
OSMMessage
>
queue
=
new
ArrayBlockingQueue
<>(
10
);
QueueProcessor
queueProcessor
=
new
QueueProcessor
(
queue
,
realP
rocessor
);
QueueProcessor
queueProcessor
=
new
QueueProcessor
(
queue
,
p
rocessor
);
// start producer thread
new
Thread
(
"producer for "
+
realProcessor
.
getClass
().
getSimpleName
()){
public
void
run
(){
new
Thread
(
"producer for "
+
processor
.
getClass
().
getSimpleName
())
{
@Override
public
void
run
()
{
try
{
process
(
queueProcessor
);
}
catch
(
SplitFailedException
e
)
{
...
...
@@ -156,7 +154,7 @@ public class OSMFileHandler {
}
}
}.
start
();
boolean
done
=
realP
rocessor
.
consume
(
queue
);
boolean
done
=
p
rocessor
.
consume
(
queue
);
if
(
exception
!=
null
)
throw
exception
;
return
done
;
...
...
src/uk/me/parabola/splitter/OSMMessage.java
View file @
8b04825e
...
...
@@ -13,34 +13,35 @@
package
uk.me.parabola.splitter
;
import
java.util.List
;
/**
* For OSM data which is passed between parsers and processors
* @author Gerd Petermann
*
*/
public
class
OSMMessage
{
public
enum
Type
{
START_FILE
,
ELEMENTS
,
BOUNDS
,
END_MAP
,
EXIT
}
;
public
enum
Type
{
START_FILE
,
ELEMENTS
,
BOUNDS
,
END_MAP
,
EXIT
}
// either el or bounds must be null
List
<
Element
>
elements
;
Area
bounds
;
Type
type
;
final
Element
[]
elements
;
final
Area
bounds
;
final
Type
type
;
public
OSMMessage
(
List
<
Element
>
elements
)
{
public
OSMMessage
(
Element
[]
elements
)
{
this
.
elements
=
elements
;
type
=
Type
.
ELEMENTS
;
bounds
=
null
;
}
public
OSMMessage
(
Area
bounds
)
{
this
.
bounds
=
bounds
;
type
=
Type
.
BOUNDS
;
elements
=
null
;
}
public
OSMMessage
(
Type
t
)
{
assert
!
t
.
equals
(
Type
.
BOUNDS
);
assert
!
t
.
equals
(
Type
.
ELEMENTS
);
assert
t
!=
Type
.
BOUNDS
&&
t
!=
Type
.
ELEMENTS
;
elements
=
null
;
bounds
=
null
;
type
=
t
;
}
}
src/uk/me/parabola/splitter/ProblemListProcessor.java
View file @
8b04825e
...
...
@@ -31,8 +31,8 @@ import java.util.regex.Pattern;
*
*/
class
ProblemListProcessor
extends
AbstractMapProcessor
{
private
final
static
int
PHASE1_NODES_AND_WAYS
=
1
;
private
final
static
int
PHASE2_RELS_ONLY
=
2
;
private
static
final
int
PHASE1_NODES_AND_WAYS
=
1
;
private
static
final
int
PHASE2_RELS_ONLY
=
2
;
private
final
SparseLong2IntMap
coords
;
private
final
SparseLong2IntMap
ways
;
...
...
@@ -55,6 +55,7 @@ class ProblemListProcessor extends AbstractMapProcessor {
private
final
HashSet
<
String
>
wantedBoundaryAdminLevels
=
new
HashSet
<>();
private
final
HashSet
<
String
>
wantedBoundaryTagValues
;
private
final
HashSet
<
String
>
wantedRouteTagValues
;
ProblemListProcessor
(
DataStorer
dataStorer
,
int
areaOffset
,
int
numAreasThisPass
,
SplitterParams
mainOptions
)
{
...
...
@@ -83,6 +84,13 @@ class ProblemListProcessor extends AbstractMapProcessor {
wantedBoundaryTagValues
=
new
HashSet
<>(
Arrays
.
asList
(
boundaryTags
));
}
setWantedAdminLevel
(
mainOptions
.
getWantedAdminLevel
());
String
routeRelationValuesParm
=
mainOptions
.
getRouteRelValues
();
if
(
routeRelationValuesParm
.
isEmpty
())
{
wantedRouteTagValues
=
null
;
}
else
{
String
[]
routeValues
=
routeRelationValuesParm
.
split
(
Pattern
.
quote
(
","
));
wantedRouteTagValues
=
new
HashSet
<>(
Arrays
.
asList
(
routeValues
));
}
}
public
void
setWantedAdminLevel
(
int
adminLevel
)
{
...
...
@@ -96,28 +104,21 @@ class ProblemListProcessor extends AbstractMapProcessor {
@Override
public
boolean
skipTags
()
{
if
(
phase
==
PHASE1_NODES_AND_WAYS
)
return
true
;
return
false
;
return
phase
==
PHASE1_NODES_AND_WAYS
;
}
@Override
public
boolean
skipNodes
()
{
if
(
phase
==
PHASE2_RELS_ONLY
)
return
true
;
return
false
;
return
phase
==
PHASE2_RELS_ONLY
;
}
@Override
public
boolean
skipWays
()
{
if
(
phase
==
PHASE2_RELS_ONLY
)
return
true
;
return
false
;
return
phase
==
PHASE2_RELS_ONLY
;
}
@Override
public
boolean
skipRels
()
{
if
(
phase
==
PHASE2_RELS_ONLY
)
return
false
;
return
true
;
return
phase
!=
PHASE2_RELS_ONLY
;
}
@Override
...
...
@@ -131,7 +132,6 @@ class ProblemListProcessor extends AbstractMapProcessor {
return
;
int
countAreas
=
0
;
int
lastUsedArea
=
UNASSIGNED
;
int
areaIdx
=
UNASSIGNED
;
AreaGridResult
areaCandidates
=
areaIndex
.
get
(
node
);
if
(
areaCandidates
==
null
)
return
;
...
...
@@ -139,24 +139,24 @@ class ProblemListProcessor extends AbstractMapProcessor {
areaSet
.
clear
();
for
(
int
n
:
areaCandidates
.
set
)
{
if
(
n
<
areaOffset
||
n
>
lastAreaOffset
)
continue
;
if
(
areaCandidates
.
testNeeded
?
areaDictionary
.
getArea
(
n
).
contains
(
node
)
:
true
)
{
if
(
n
>=
areaOffset
&&
n
<=
lastAreaOffset
&&
(!
areaCandidates
.
testNeeded
||
areaDictionary
.
getArea
(
n
).
contains
(
node
)))
{
areaSet
.
set
(
n
);
++
countAreas
;
lastUsedArea
=
n
;
}
}
if
(
countAreas
>
0
){
if
(
countAreas
>
0
)
{
int
areaIdx
;
if
(
countAreas
>
1
)
areaIdx
=
areaDictionary
.
translate
(
areaSet
);
else
areaIdx
=
AreaDictionary
.
translate
(
lastUsedArea
);
// no need to do lookup in the dictionary
coords
.
put
(
node
.
getId
(),
areaIdx
);
++
countCoords
;
if
(
countCoords
%
10_000_000
==
0
){
System
.
out
.
println
(
"coord MAP occupancy: "
+
Utils
.
format
(
countCoords
)
+
", number of area dictionary entries: "
+
areaDictionary
.
size
());
if
(
countCoords
%
10_000_000
==
0
)
{
System
.
out
.
println
(
"coord MAP occupancy: "
+
Utils
.
format
(
countCoords
)
+
", number of area dictionary entries: "
+
areaDictionary
.
size
());
}
}
}
...
...
@@ -177,16 +177,14 @@ class ProblemListProcessor extends AbstractMapProcessor {
maybeChanged
=
true
;
}
}
if
(!
isFirstPass
&&
maybeChanged
||
(
isLastPass
&
!
isFirstPass
)){
if
(!
isFirstPass
&&
maybeChanged
||
(
isLastPass
&
&
!
isFirstPass
)){
int
wayAreaIdx
=
ways
.
get
(
way
.
getId
());
if
(
wayAreaIdx
!=
UNASSIGNED
)
areaSet
.
or
(
areaDictionary
.
getSet
(
wayAreaIdx
));
}
if
(
isLastPass
){
if
(
checkIfMultipleAreas
(
areaSet
)){
problemWays
.
add
(
way
.
getId
());
}
if
(
isLastPass
&&
checkIfMultipleAreas
(
areaSet
)){
problemWays
.
add
(
way
.
getId
());
}
if
(
maybeChanged
&&
!
areaSet
.
isEmpty
()){
ways
.
put
(
way
.
getId
(),
areaDictionary
.
translate
(
areaSet
));
...
...
@@ -194,7 +192,7 @@ class ProblemListProcessor extends AbstractMapProcessor {
}
// default exclude list for boundary tag
private
final
static
HashSet
<
String
>
unwantedBoundaryTagValues
=
new
HashSet
<>(
private
static
final
HashSet
<
String
>
unwantedBoundaryTagValues
=
new
HashSet
<>(
Arrays
.
asList
(
"administrative"
,
"postal_code"
,
"political"
));
@Override
...
...
@@ -204,7 +202,9 @@ class ProblemListProcessor extends AbstractMapProcessor {
boolean
useThis
=
false
;
boolean
isMPRelType
=
false
;
boolean
hasBoundaryTag
=
false
;
boolean
isWantedBoundary
=
(
wantedBoundaryTagValues
==
null
)
?
true
:
false
;
boolean
isWantedBoundary
=
wantedBoundaryTagValues
==
null
;
boolean
isRouteRelType
=
false
;
boolean
isWantedRoute
=
wantedRouteTagValues
!=
null
;
Iterator
<
Element
.
Tag
>
tags
=
rel
.
tagsIterator
();
String
admin_level
=
null
;
while
(
tags
.
hasNext
())
{
...
...
@@ -214,6 +214,8 @@ class ProblemListProcessor extends AbstractMapProcessor {
useThis
=
true
;
// no need to check other tags
else
if
(
"multipolygon"
.
equals
((
t
.
value
))
||
"boundary"
.
equals
((
t
.
value
)))
isMPRelType
=
true
;
else
if
(
"route"
.
equals
(
t
.
value
))
isRouteRelType
=
true
;
else
if
(
"associatedStreet"
.
equals
((
t
.
value
))
||
"street"
.
equals
((
t
.
value
)))
useThis
=
true
;
// no need to check other tags
}
else
if
(
"boundary"
.
equals
(
t
.
key
)){
...
...
@@ -228,32 +230,35 @@ class ProblemListProcessor extends AbstractMapProcessor {
}
else
if
(
"admin_level"
.
equals
(
t
.
key
)){
admin_level
=
t
.
value
;
}
if
(
wantedRouteTagValues
!=
null
&&
"route"
.
equals
((
t
.
key
))
&&
wantedRouteTagValues
.
contains
(
t
.
value
))
{
isWantedRoute
=
true
;
}
if
(
useThis
)
break
;
}
if
(
isMPRelType
&&
(
isWantedBoundary
||
!
hasBoundaryTag
))
useThis
=
true
;
else
if
(
isMPRelType
&&
hasBoundaryTag
&&
admin_level
!=
null
){
else
if
(
isMPRelType
&&
hasBoundaryTag
&&
admin_level
!=
null
)
{
if
(
wantedBoundaryAdminLevels
.
contains
(
admin_level
))
useThis
=
true
;
}
else
if
(
isRouteRelType
&&
isWantedRoute
)
{
useThis
=
true
;
}
if
(!
useThis
){
if
(!
useThis
)
{
return
;
}
areaSet
.
clear
();
Integer
relAreaIdx
;
if
(!
isFirstPass
){
if
(!
isFirstPass
)
{
relAreaIdx
=
dataStorer
.
getUsedRels
().
get
(
rel
.
getId
());
if
(
relAreaIdx
!=
null
)
areaSet
.
or
(
areaDictionary
.
getSet
(
relAreaIdx
));
}
int
oldclIndex
=
UNASSIGNED
;
int
oldwlIndex
=
UNASSIGNED
;
//System.out.println("r" + rel.getId() + " " + rel.getMembers().size());
for
(
Member
mem
:
rel
.
getMembers
())
{
long
id
=
mem
.
getRef
();
if
(
mem
.
getType
().
equals
(
"node"
))
{
if
(
"node"
.
equals
(
mem
.
getType
()
))
{
int
clIdx
=
coords
.
get
(
id
);
if
(
clIdx
!=
UNASSIGNED
){
...
...
@@ -264,7 +269,7 @@ class ProblemListProcessor extends AbstractMapProcessor {
}
}
else
if
(
mem
.
getType
()
.
equals
(
"way"
))
{
}
else
if
(
"way"
.
equals
(
mem
.
getType
()))
{
int
wlIdx
=
ways
.
get
(
id
);
if
(
wlIdx
!=
UNASSIGNED
){
...
...
src/uk/me/parabola/splitter/QueueProcessor.java
View file @
8b04825e
...
...
@@ -13,8 +13,6 @@
package
uk.me.parabola.splitter
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.concurrent.BlockingQueue
;
import
uk.me.parabola.splitter.OSMMessage.Type
;
...
...
@@ -31,9 +29,19 @@ public class QueueProcessor extends AbstractMapProcessor {
private
final
BlockingQueue
<
OSMMessage
>
queue
;
private
final
MapProcessor
realProcessor
;
/** number of OSM elements to collect before adding them to the queue */
private
static
final
int
NUM_STAGING
=
1000
;
private
Element
[]
staging
;
private
int
stagingPos
;
public
QueueProcessor
(
BlockingQueue
<
OSMMessage
>
queue
,
MapProcessor
realProcessor
)
{
this
.
queue
=
queue
;
this
.
realProcessor
=
realProcessor
;
initStaging
();
}
private
void
initStaging
()
{
staging
=
new
Element
[
NUM_STAGING
];
stagingPos
=
0
;
}
@Override
...
...
@@ -97,18 +105,15 @@ public class QueueProcessor extends AbstractMapProcessor {
return
true
;
}
@Override
public
int
getPhase
()
{
throw
new
UnsupportedOperationException
(
"call getPhase() of real processor"
);
}
/** number of OSM elements to collect before adding them to the queue */
private
static
final
int
NUM_STAGING
=
1000
;
private
List
<
Element
>
staging
=
new
ArrayList
<>(
NUM_STAGING
);
private
void
addToQueue
(
Element
el
)
{
try
{
staging
.
add
(
el
)
;
if
(
staging
.
size
()
>=
NUM_STAGING
)
staging
[
stagingPos
++]
=
el
;
if
(
staging
Pos
>=
NUM_STAGING
)
flush
();
}
catch
(
InterruptedException
e
)
{
throw
new
RuntimeException
(
e
);
...
...
@@ -125,9 +130,9 @@ public class QueueProcessor extends AbstractMapProcessor {
}
private
void
flush
()
throws
InterruptedException
{
if
(
staging
==
null
||
staging
.
isEmpty
()
)
if
(
staging
==
null
||
staging
Pos
==
0
)
return
;
queue
.
put
(
new
OSMMessage
(
staging
));
s
taging
=
new
ArrayList
<>(
NUM_STAGING
);
initS
taging
(
);
}
}
src/uk/me/parabola/splitter/SplitProcessor.java
View file @
8b04825e
...
...
@@ -51,7 +51,7 @@ class SplitProcessor extends AbstractMapProcessor {
private
final
InputQueueInfo
[]
writerInputQueues
;
protected
final
BlockingQueue
<
InputQueueInfo
>
toProcess
;
private
final
ArrayList
<
Thread
>
workerThreads
;
protected
final
InputQueueInfo
STOP_MSG
=
new
InputQueueInfo
(
null
);
protected
final
InputQueueInfo
stopMsg
=
new
InputQueueInfo
(
null
);
private
AreaSet
usedWriters
;
...
...
@@ -125,29 +125,26 @@ class SplitProcessor extends AbstractMapProcessor {
@Override
public
void
processWay
(
Way
w
)
{
usedWriters
.
clear
();
int
multiTileWriterIdx
=
(
wayWriterMap
!=
null
)
?
wayWriterMap
.
getSeq
(
w
.
getId
()):
UNASSIGNED
;
if
(
multiTileWriterIdx
!=
UNASSIGNED
){
int
multiTileWriterIdx
=
(
wayWriterMap
!=
null
)
?
wayWriterMap
.
getSeq
(
w
.
getId
())
:
UNASSIGNED
;
if
(
multiTileWriterIdx
!=
UNASSIGNED
)
{
setUsedWriters
(
multiTileWriterIdx
);
}
else
{
}
else
{
int
oldclIndex
=
UNASSIGNED
;
for
(
long
id
:
w
.
getRefs
())
{
// Get the list of areas that the way is in.
// Get the list of areas that the way is in.
int
clIdx
=
coords
.
get
(
id
);
if
(
clIdx
!=
UNASSIGNED
){
if
(
oldclIndex
!=
clIdx
){
usedWriters
.
or
(
writerDictionary
.
getSet
(
clIdx
));
if
(
wayWriterMap
!=
null
){
// we can stop here because all other nodes
// will be in the same tile
break
;
}
oldclIndex
=
clIdx
;
if
(
clIdx
!=
UNASSIGNED
&&
oldclIndex
!=
clIdx
)
{
usedWriters
.
or
(
writerDictionary
.
getSet
(
clIdx
));
if
(
wayWriterMap
!=
null
)
{
// we can stop here because all other nodes
// will be in the same tile
break
;
}
oldclIndex
=
clIdx
;
}
}
}
if
(!
usedWriters
.
isEmpty
()){
if
(!
usedWriters
.
isEmpty
())
{
// store these areas in ways map
ways
.
put
(
w
.
getId
(),
writerDictionary
.
translate
(
usedWriters
));
++
countWays
;
...
...
@@ -182,9 +179,8 @@ class SplitProcessor extends AbstractMapProcessor {
int
oldclIndex
=
UNASSIGNED
;
int
oldwlIndex
=
UNASSIGNED
;
for
(
Member
mem
:
rel
.
getMembers
())
{
// String role = mem.getRole();
long
id
=
mem
.
getRef
();
if
(
mem
.
getType
().
equals
(
"node"
))
{
if
(
"node"
.
equals
(
mem
.
getType
()
))
{
int
clIdx
=
coords
.
get
(
id
);
if
(
clIdx
!=
UNASSIGNED
){
...
...
@@ -193,7 +189,7 @@ class SplitProcessor extends AbstractMapProcessor {
}
oldclIndex
=
clIdx
;
}
}
else
if
(
mem
.
getType
()
.
equals
(
"way"
))
{
}
else
if
(
"way"
.
equals
(
mem
.
getType
()))
{
int
wlIdx
=
ways
.
get
(
id
);
if
(
wlIdx
!=
UNASSIGNED
){
...
...
@@ -226,14 +222,12 @@ class SplitProcessor extends AbstractMapProcessor {
try
{
writerInputQueues
[
i
].
stop
();
}
catch
(
InterruptedException
e
)
{
throw
new
SplitFailedException
(
"Failed to add the stop element for worker thread "
+
i
,
e
);
throw
new
SplitFailedException
(
"Failed to add the stop element for worker thread "
+
i
,
e
);
}
}
try
{
if
(
maxThreads
>
1
)
toProcess
.
put
(
STOP_MSG
);
// Magic flag used to indicate that all data is done.
toProcess
.
put
(
stopMsg
);
// Magic flag used to indicate that all data is done.
}
catch
(
InterruptedException
e1
)
{
e1
.
printStackTrace
();
...
...
@@ -243,8 +237,7 @@ class SplitProcessor extends AbstractMapProcessor {
try
{
workerThread
.
join
();
}
catch
(
InterruptedException
e
)
{
throw
new
SplitFailedException
(
"Failed to join for thread "
+
workerThread
.
getName
(),
e
);
throw
new
SplitFailedException
(
"Failed to join for thread "
+
workerThread
.
getName
(),
e
);
}
}
for
(
int
i
=
writerOffset
;
i
<=
lastWriter
;
i
++)
{
...
...
@@ -294,11 +287,9 @@ class SplitProcessor extends AbstractMapProcessor {
// this node is part of a multi-tile-polygon, add it to all tiles covered by the parent
AreaSet
nodeWriters
=
writerDictionary
.
getSet
(
multiTileWriterIdx
);
for
(
int
i
:
nodeWriters
)
{
if
(
i
<
writerOffset
||
i
>
lastWriter
)
if
(
i
<
writerOffset
||
i
>
lastWriter
||
usedWriters
.
get
(
i
)
)
continue
;
if
(
usedWriters
.
get
(
i
)
)
continue
;
if
(
maxThreads
>
1
)
{
addToWorkingQueue
(
i
,
currentNode
);
}
else
{
...
...
@@ -381,7 +372,6 @@ class SplitProcessor extends AbstractMapProcessor {
}
void
flush
()
throws
InterruptedException
{
// System.out.println("Flush");
inputQueue
.
put
(
staging
);
staging
=
new
ArrayList
<>(
STAGING_SIZE
);
toProcess
.
put
(
this
);
...
...
@@ -392,14 +382,11 @@ class SplitProcessor extends AbstractMapProcessor {
}
}
public
static
final
int
NO_ELEMENTS
=
3
;
final
int
STAGING_SIZE
=
300
;
static
final
int
NO_ELEMENTS
=
3
;
static
final
int
STAGING_SIZE
=
300
;
private
class
OSMWriterWorker
implements
Runnable
{
public
OSMWriterWorker
()
{
}
@Override
public
void
run
()
{
boolean
finished
=
false
;
...
...
@@ -411,9 +398,9 @@ class SplitProcessor extends AbstractMapProcessor {
e1
.
printStackTrace
();
continue
;
}
if
(
workPackage
==
STOP_MSG
)
{
if
(
workPackage
==
stopMsg
)
{
try
{
toProcess
.
put
(
STOP_MSG
);
// Re-inject it so that other
toProcess
.
put
(
stopMsg
);
// Re-inject it so that other
// threads know that we're
// exiting.
}
catch
(
InterruptedException
e
)
{
...
...
src/uk/me/parabola/splitter/Utils.java
View file @
8b04825e
...
...
@@ -24,7 +24,7 @@ import java.io.IOException;
import
java.io.InputStream
;
import
java.io.InputStreamReader
;
import
java.io.Reader
;
import
java.nio.charset.Charset
;
import
java.nio.charset.
Standard
Charset
s
;
import
java.text.NumberFormat
;
import
java.util.ArrayList
;
import
java.util.List
;
...
...
@@ -69,10 +69,10 @@ public class Utils {
* @return An integer value in map units.
*/
public
static
int
toMapUnit
(
double
l
)
{
double
DELTA
=
360.0
D
/
(
1
<<
24
)
/
2
;
//Correct rounding
double
delta
=
360.0
D
/
(
1
<<
24
)
/
2
;
//
Correct rounding
if
(
l
>
0
)
return
(
int
)
((
l
+
DELTA
)
*
(
1
<<
24
)
/
360
);
return
(
int
)
((
l
-
DELTA
)
*
(
1
<<
24
)
/
360
);
return
(
int
)
((
l
+
delta
)
*
(
1
<<
24
)
/
360
);
return
(
int
)
((
l
-
delta
)
*
(
1
<<
24
)
/
360
);
}
/**
...
...
@@ -116,7 +116,7 @@ public class Utils {
if
(
backgroundReader
)
{
is
=
new
BackgroundInputStream
(
is
);
}
return
new
InputStreamReader
(
is
,
Charset
.
forName
(
"UTF-8"
)
);
return
new
InputStreamReader
(
is
,
StandardCharsets
.
UTF_8
);
}
public
static
Rectangle
area2Rectangle
(
Area
area
,
int
overlap
){
...
...
@@ -164,7 +164,7 @@ public class Utils {
case
PathIterator
.
SEG_MOVETO
:
case
PathIterator
.
SEG_CLOSE
:
if
((
type
==
PathIterator
.
SEG_MOVETO
&&
points
!=
null
)
||
type
==
PathIterator
.
SEG_CLOSE
)
{
if
(
points
.
size
()
>
2
&&
points
.
get
(
0
).
equals
(
points
.
get
(
points
.
size
()
-
1
))
==
false
)
{
if
(
points
.
size
()
>
2
&&
!
points
.
get
(
0
).
equals
(
points
.
get
(
points
.
size
()
-
1
)))
{
points
.
add
(
points
.
get
(
0
));
}
if
(
points
.
size
()
>
3
){
...
...
@@ -208,7 +208,7 @@ public class Utils {
}
/**
* Convert area with coordinates in degrees to area in MapUnits
* Convert area with coordinates in degrees to area in MapUnits
.
* @param area
* @return
*/
...
...
src/uk/me/parabola/splitter/args/SplitterParams.java
View file @
8b04825e
...
...
@@ -112,7 +112,6 @@ public interface SplitterParams {
+
"Used to filter boundary relations for problem-list processing. Ignored when keep-complete is false."
)
int
getWantedAdminLevel
();
@Option
(
defaultValue
=
"200000"
,
description
=
"Search limit in split algo. Higher values may find better splits, but will take longer."
)
int
getSearchLimit
();
...
...
@@ -123,4 +122,8 @@ public interface SplitterParams {
@Option
(
defaultValue
=
"false"
,
description
=
"Specify if splitter should ignore bounds tags in input files"
)
boolean
getIgnoreOsmBounds
();
@Option
(
defaultValue
=
""
,
description
=
"A comma separated list of tag values for route relations. "
+
"Can be used to keep route relations of the given type complete. Only route values listed are kept complete. Default is empty."
)
String
getRouteRelValues
();
}
src/uk/me/parabola/splitter/solver/DensityMap.java
View file @
8b04825e
...
...
@@ -281,17 +281,15 @@ public class DensityMap {
details
.
addToBounds
(
Integer
.
parseInt
(
items
[
2
]),
Integer
.
parseInt
(
items
[
3
]));
}
inLine
=
problemReader
.
readLine
();
if
(
inLine
!=
null
){
if
(
"no_bounds_in_input"
.
equals
(
inLine
)
==
false
){
items
=
csvSplitter
.
split
(
inLine
);
if
(
items
.
length
!=
4
){
System
.
out
.
println
(
"Error: Invalid format in map file, line number "
+
problemReader
.
getLineNumber
()
+
": "
+
inLine
);
return
null
;
}
collectorBounds
=
new
Area
(
Integer
.
parseInt
(
items
[
0
]),
Integer
.
parseInt
(
items
[
1
]),
Integer
.
parseInt
(
items
[
2
]),
Integer
.
parseInt
(
items
[
3
]));
if
(
inLine
!=
null
&&
!
"no_bounds_in_input"
.
equals
(
inLine
))
{
items
=
csvSplitter
.
split
(
inLine
);
if
(
items
.
length
!=
4
)
{
System
.
out
.
println
(
"Error: Invalid format in map file, line number "
+
problemReader
.
getLineNumber
()
+
": "
+
inLine
);
return
null
;
}
collectorBounds
=
new
Area
(
Integer
.
parseInt
(
items
[
0
]),
Integer
.
parseInt
(
items
[
1
]),
Integer
.
parseInt
(
items
[
2
]),
Integer
.
parseInt
(
items
[
3
]));
}
while
((
inLine
=
problemReader
.
readLine
())
!=
null
)
{
items
=
csvSplitter
.
split
(
inLine
);
...
...
@@ -336,8 +334,7 @@ public class DensityMap {
assert
y
>=
0
;
assert
width2
>
0
;
assert
height2
>
0
;
Area
area
=
new
Area
(
yToLat
(
y
),
xToLon
(
x
),
yToLat
(
y
+
height2
),
xToLon
(
x
+
width2
));
return
area
;
return
new
Area
(
yToLat
(
y
),
xToLon
(
x
),
yToLat
(
y
+
height2
),
xToLon
(
x
+
width2
));
}
/**
...
...
src/uk/me/parabola/splitter/solver/DensityMapCollector.java
View file @
8b04825e
...
...
@@ -104,7 +104,7 @@ class DensityMapCollector extends AbstractMapProcessor{
}
public
void
saveMap
(
String
fileName
)
{
if
(
details
!=
null
&&
details
.
getBounds
()
!=
null
)
if
(
details
.
getBounds
()
!=
null
)
densityMap
.
saveMap
(
fileName
,
details
.
getBounds
(),
bounds
);
}
public
void
readMap
(
String
fileName
)
{
...
...
src/uk/me/parabola/splitter/solver/PrecompSeaReader.java
View file @
8b04825e
...
...
@@ -19,7 +19,7 @@ import java.io.InputStream;
import
java.io.InputStreamReader
;
import
java.io.LineNumberReader
;
import
java.io.Reader
;
import
java.nio.charset.Charset
;
import
java.nio.charset.
Standard
Charset
s
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.regex.Pattern
;
...
...
@@ -28,6 +28,7 @@ import java.util.zip.ZipEntry;
import
java.util.zip.ZipFile
;
import
org.xmlpull.v1.XmlPullParserException
;
import
crosby.binary.file.BlockInputStream
;
import
uk.me.parabola.splitter.Area
;
import
uk.me.parabola.splitter.SplitFailedException
;
...
...
@@ -45,7 +46,7 @@ import uk.me.parabola.splitter.parser.OSMXMLParser;
public
class
PrecompSeaReader
{
/** The size (lat and long) of the precompiled sea tiles */
private
final
static
int
PRECOMP_RASTER
=
1
<<
15
;
private
static
final
int
PRECOMP_RASTER
=
1
<<
15
;
private
static
final
byte
SEA_TILE
=
's'
;
private
static
final
byte
LAND_TILE
=
'l'
;
...
...
@@ -57,7 +58,7 @@ public class PrecompSeaReader {
private
static
final
int
MAX_LAT
=
Utils
.
toMapUnit
(
90.0
);
private
static
final
int
MIN_LON
=
Utils
.
toMapUnit
(-
180.0
);
private
static
final
int
MAX_LON
=
Utils
.
toMapUnit
(
180.0
);
private
final
static
Pattern
keySplitter
=
Pattern
.
compile
(
Pattern
.
quote
(
"_"
));
private
static
final
Pattern
keySplitter
=
Pattern
.
compile
(
Pattern
.
quote
(
"_"
));
private
final
Area
bounds
;
private
final
File
precompSeaDir
;
...
...
@@ -92,7 +93,7 @@ public class PrecompSeaReader {
blockinput
.
close
();
}
else
{
// No, try XML.
try
(
Reader
reader
=
new
InputStreamReader
(
is
,
Charset
.
forName
(
"UTF-8"
));
)
{
try
(
Reader
reader
=
new
InputStreamReader
(
is
,
StandardCharsets
.
UTF_8
)
)
{
OSMXMLParser
parser
=
new
OSMXMLParser
(
processor
,
true
);
parser
.
setReader
(
reader
);
parser
.
parse
();
...
...
@@ -116,7 +117,7 @@ public class PrecompSeaReader {
try
{
if
(
precompSeaDir
.
isDirectory
())
{
File
indexFile
=
new
File
(
precompSeaDir
,
indexFileName
);
if
(
indexFile
.
exists
()
==
false
)
{
if
(
!
indexFile
.
exists
())
{
// check if the unzipped index file exists
indexFileName
=
"index.txt"
;
indexFile
=
new
File
(
precompSeaDir
,
indexFileName
);
...
...
@@ -125,8 +126,9 @@ public class PrecompSeaReader {
try
(
InputStream
indexStream
=
new
FileInputStream
(
indexFile
))
{
loadIndex
(
indexStream
,
indexFileName
);
}
}
else
}
else
{
throw
new
IllegalArgumentException
(
"Cannot find required index.txt[.gz] in "
+
precompSeaDir
);
}
}
else
if
(
precompSeaDir
.
getName
().
endsWith
(
".zip"
))
{
zipFile
=
new
ZipFile
(
precompSeaDir
);
internalPath
=
"sea/"
;
...
...
@@ -145,8 +147,9 @@ public class PrecompSeaReader {
precompZipFileInternalPath
=
internalPath
;
loadIndex
(
indexStream
,
indexFileName
);
}
}
else
}
else
{
throw
new
SplitFailedException
(
"Don't know how to read "
+
precompSeaDir
);
}
}
else
{
throw
new
SplitFailedException
(
"Don't know how to read "
+
precompSeaDir
);
}
...
...
@@ -216,7 +219,7 @@ public class PrecompSeaReader {
StringBuilder
sb
=
new
StringBuilder
(
prefix
);
sb
.
append
(
precompKey
);
sb
.
append
(
ext
);
if
(
items
[
1
].
equals
(
sb
.
toString
())
==
false
)
{
if
(
!
items
[
1
].
equals
(
sb
.
toString
()))
{
throw
new
IllegalArgumentException
(
"Unexpected file name in index file: "
+
indexLine
);
}
}
...
...
@@ -324,8 +327,8 @@ public class PrecompSeaReader {
String
[]
tileCoords
=
keySplitter
.
split
(
precompKey
);
byte
type
=
'?'
;
if
(
tileCoords
.
length
==
2
)
{
int
lat
=
Integer
.
valueOf
(
tileCoords
[
0
]);
int
lon
=
Integer
.
valueOf
(
tileCoords
[
1
]);
int
lat
=
Integer
.
parseInt
(
tileCoords
[
0
]);
int
lon
=
Integer
.
parseInt
(
tileCoords
[
1
]);
int
latIndex
=
(
MAX_LAT
-
lat
)
/
PRECOMP_RASTER
;
int
lonIndex
=
(
MAX_LON
-
lon
)
/
PRECOMP_RASTER
;
...
...
src/uk/me/parabola/splitter/solver/Solution.java
View file @
8b04825e
...
...
@@ -26,7 +26,7 @@ public class Solution {
/**
*
*/
private
static
enum
sides
{
TOP
,
RIGHT
,
BOTTOM
,
LEFT
}
private
enum
sides
{
TOP
,
RIGHT
,
BOTTOM
,
LEFT
}
private
final
List
<
Tile
>
tiles
;
private
final
long
maxNodes
;
...
...
@@ -38,14 +38,14 @@ public class Solution {
this
.
maxNodes
=
maxNodes
;
}
public
Solution
copy
(){
public
Solution
copy
()
{
Solution
s
=
new
Solution
(
this
.
maxNodes
);
for
(
Tile
t
:
tiles
)
s
.
add
(
t
);
return
s
;
}
public
boolean
add
(
Tile
tile
){
public
boolean
add
(
Tile
tile
)
{
tiles
.
add
(
tile
);
double
aspectRatio
=
tile
.
getAspectRatio
();
if
(
aspectRatio
<
1.0
)
...
...
@@ -59,11 +59,11 @@ public class Solution {
* Combine this solution with the other.
* @param other
*/
public
void
merge
(
Solution
other
){
public
void
merge
(
Solution
other
)
{
if
(
other
.
tiles
.
isEmpty
())
return
;
if
(
tiles
.
isEmpty
()){
if
(
tiles
.
isEmpty
())
{
worstAspectRatio
=
other
.
worstAspectRatio
;
worstMinNodes
=
other
.
worstMinNodes
;
}
else
{
...
...
@@ -79,19 +79,19 @@ public class Solution {
return
tiles
;
}
public
long
getWorstMinNodes
(){
public
long
getWorstMinNodes
()
{
return
worstMinNodes
;
}
public
double
getWorstAspectRatio
(){
public
double
getWorstAspectRatio
()
{
return
worstAspectRatio
;
}
public
boolean
isEmpty
(){
public
boolean
isEmpty
()
{
return
tiles
.
isEmpty
();
}
public
int
size
(){
public
int
size
()
{
return
tiles
.
size
();
}
...
...
@@ -100,7 +100,7 @@ public class Solution {
* @param other
* @return -1 if this is better, 1 if other is better, 0 if both are equal
*/
public
int
compareTo
(
Solution
other
){
public
int
compareTo
(
Solution
other
)
{
if
(
other
==
null
)
return
-
1
;
if
(
other
==
this
)
...
...
@@ -110,7 +110,7 @@ public class Solution {
if
(
isNice
()
!=
other
.
isNice
())
return
isNice
()
?
-
1
:
1
;
if
(
worstMinNodes
!=
other
.
worstMinNodes
){
if
(
worstMinNodes
!=
other
.
worstMinNodes
)
{
// ignore minNodes when both are bad
if
(
Math
.
max
(
worstMinNodes
,
other
.
worstMinNodes
)
>
1000
)
return
(
worstMinNodes
>
other
.
worstMinNodes
)
?
-
1
:
1
;
...
...
@@ -135,7 +135,7 @@ public class Solution {
* Trim tiles without creating holes or gaps between tiles
*/
public
void
trimOuterTiles
()
{
while
(
true
){
while
(
true
)
{
boolean
trimmedAny
=
false
;
int
minX
=
Integer
.
MAX_VALUE
;
...
...
@@ -143,67 +143,53 @@ public class Solution {
int
minY
=
Integer
.
MAX_VALUE
;
int
maxY
=
Integer
.
MIN_VALUE
;
for
(
Tile
tile
:
tiles
){
for
(
Tile
tile
:
tiles
)
{
if
(
minX
>
tile
.
x
)
minX
=
tile
.
x
;
if
(
minY
>
tile
.
y
)
minY
=
tile
.
y
;
if
(
maxX
<
tile
.
getMaxX
())
maxX
=
(
int
)
tile
.
getMaxX
();
if
(
maxY
<
tile
.
getMaxY
())
maxY
=
(
int
)
tile
.
getMaxY
();
}
for
(
sides
side:
sides
.
values
()){
for
(
int
direction
=
-
1
;
direction
<=
1
;
direction
+=
2
){
for
(
sides
side:
sides
.
values
())
{
for
(
int
direction
=
-
1
;
direction
<=
1
;
direction
+=
2
)
{
int
trimToPos
=
-
1
;
switch
(
side
){
switch
(
side
)
{
case
LEFT:
case
BOTTOM:
trimToPos
=
Integer
.
MAX_VALUE
;
break
;
case
TOP:
case
RIGHT:
trimToPos
=
-
1
;
}
while
(
true
){
while
(
true
)
{
Tile
candidate
=
null
;
boolean
trimmed
=
false
;
for
(
Tile
tile
:
tiles
){
for
(
Tile
tile
:
tiles
)
{
if
(
tile
.
getCount
()
==
0
)
continue
;
switch
(
side
){
switch
(
side
)
{
case
LEFT:
if
(
minX
==
tile
.
x
){
if
(
candidate
==
null
)
candidate
=
tile
;
else
if
(
direction
<
0
&&
candidate
.
y
>
tile
.
y
)
candidate
=
tile
;
else
if
(
direction
>
0
&&
candidate
.
getMaxY
()
<
tile
.
getMaxY
())
candidate
=
tile
;
if
(
minX
==
tile
.
x
&&
(
candidate
==
null
||
(
direction
<
0
&&
candidate
.
y
>
tile
.
y
)
||
(
direction
>
0
&&
candidate
.
getMaxY
()
<
tile
.
getMaxY
())))
{
candidate
=
tile
;
}
break
;
case
RIGHT:
if
(
maxX
==
tile
.
getMaxX
()){
if
(
candidate
==
null
)
candidate
=
tile
;
else
if
(
direction
<
0
&&
candidate
.
y
>
tile
.
y
)
candidate
=
tile
;
else
if
(
direction
>
0
&&
candidate
.
getMaxY
()
<
tile
.
getMaxY
())
candidate
=
tile
;
case
RIGHT:
if
(
maxX
==
tile
.
getMaxX
()
&&
(
candidate
==
null
||
(
direction
<
0
&&
candidate
.
y
>
tile
.
y
)
||
(
direction
>
0
&&
candidate
.
getMaxY
()
<
tile
.
getMaxY
())))
{
candidate
=
tile
;
}
break
;
case
BOTTOM:
if
(
minY
==
tile
.
y
){
if
(
candidate
==
null
)
candidate
=
tile
;
else
if
(
direction
<
0
&&
candidate
.
x
>
tile
.
x
)
candidate
=
tile
;
else
if
(
direction
>
0
&&
candidate
.
getMaxX
()
<
tile
.
getMaxX
())
candidate
=
tile
;
case
BOTTOM:
if
(
minY
==
tile
.
y
&&
(
candidate
==
null
||
(
direction
<
0
&&
candidate
.
x
>
tile
.
x
)
||
(
direction
>
0
&&
candidate
.
getMaxX
()
<
tile
.
getMaxX
())))
{
candidate
=
tile
;
}
break
;
case
TOP:
if
(
maxY
==
tile
.
getMaxY
()){
if
(
candidate
==
null
)
candidate
=
tile
;
else
if
(
direction
<
0
&&
candidate
.
x
>
tile
.
x
)
candidate
=
tile
;
else
if
(
direction
>
0
&&
candidate
.
getMaxX
()
<
tile
.
getMaxX
())
candidate
=
tile
;
case
TOP:
if
(
maxY
==
tile
.
getMaxY
()
&&
(
candidate
==
null
||
(
direction
<
0
&&
candidate
.
x
>
tile
.
x
)
||
(
direction
>
0
&&
candidate
.
getMaxX
()
<
tile
.
getMaxX
())))
{
candidate
=
tile
;
}
break
;
}
...
...
@@ -211,9 +197,9 @@ public class Solution {
if
(
candidate
==
null
)
break
;
Rectangle
before
=
new
Rectangle
(
candidate
);
switch
(
side
){
switch
(
side
)
{
case
LEFT:
while
(
candidate
.
x
<
trimToPos
&&
candidate
.
getColSum
(
0
)
==
0
){
while
(
candidate
.
x
<
trimToPos
&&
candidate
.
getColSum
(
0
)
==
0
)
{
candidate
.
x
++;
candidate
.
width
--;
}
...
...
@@ -221,14 +207,14 @@ public class Solution {
trimToPos
=
candidate
.
x
;
break
;
case
RIGHT:
while
((
candidate
.
getMaxX
()
>
trimToPos
)
&&
candidate
.
getColSum
(
candidate
.
width
-
1
)
==
0
){
while
((
candidate
.
getMaxX
()
>
trimToPos
)
&&
candidate
.
getColSum
(
candidate
.
width
-
1
)
==
0
)
{
candidate
.
width
--;
}
if
(
candidate
.
getMaxX
()
>
trimToPos
)
trimToPos
=
(
int
)
candidate
.
getMaxX
();
break
;
case
BOTTOM:
while
(
candidate
.
y
<
trimToPos
&&
candidate
.
getRowSum
(
0
)
==
0
){
while
(
candidate
.
y
<
trimToPos
&&
candidate
.
getRowSum
(
0
)
==
0
)
{
candidate
.
y
++;
candidate
.
height
--;
}
...
...
@@ -236,14 +222,14 @@ public class Solution {
trimToPos
=
candidate
.
y
;
break
;
case
TOP:
while
(
candidate
.
getMaxY
()
>
trimToPos
&&
candidate
.
getRowSum
(
candidate
.
height
-
1
)
==
0
){
while
(
candidate
.
getMaxY
()
>
trimToPos
&&
candidate
.
getRowSum
(
candidate
.
height
-
1
)
==
0
)
{
candidate
.
height
--;
}
if
(
candidate
.
getMaxX
()
>
trimToPos
)
trimToPos
=
(
int
)
candidate
.
getMaxY
();
break
;
}
if
(
before
.
equals
(
candidate
)
==
false
){
if
(
!
before
.
equals
(
candidate
))
{
trimmed
=
true
;
trimmedAny
=
true
;
}
...
...
@@ -276,7 +262,8 @@ public class Solution {
return
true
;
}
public
String
toString
(){
@Override
public
String
toString
()
{
double
ratio
=
(
double
)
Math
.
round
(
worstAspectRatio
*
100
)
/
100
;
long
percentage
=
100
*
worstMinNodes
/
maxNodes
;
if
(
isEmpty
())
...
...
src/uk/me/parabola/splitter/tools/SparseLong2IntMap.java
View file @
8b04825e
...
...
@@ -100,7 +100,7 @@ public final class SparseLong2IntMap {
private
static
final
int
LARGE_VECTOR_SIZE
=
(
int
)
(
CHUNK_ID_MASK
/
CHUNK_SIZE
+
1
);
private
static
final
int
MAX_Y_VAL
=
LARGE_VECTOR_SIZE
/
CHUNK_STORE_ELEMS
+
1
;
/** The part of the key that contains the offset in the chunk. */
private
static
final
long
CHUNK_OFFSET_MASK
=
CHUNK_SIZE
-
1
;
private
static
final
long
CHUNK_OFFSET_MASK
=
CHUNK_SIZE
-
1
L
;
/** First 58 bits of a long. If this part of the key changes, a different chunk is needed. */
private
static
final
long
OLD_CHUNK_ID_MASK
=
~
CHUNK_OFFSET_MASK
;
...
...
@@ -187,7 +187,7 @@ public final class SparseLong2IntMap {
freePosInStore
=
new
int
[
MAX_STORED_BYTES_FOR_CHUNK
];
reusableChunks
=
new
Int2ObjectOpenHashMap
<>(
0
,
Hash
.
VERY_FAST_LOAD_FACTOR
);
largeVector
=
new
int
[
LARGE_VECTOR_SIZE
];
estimatedBytes
=
LARGE_VECTOR_SIZE
*
Integer
.
BYTES
estimatedBytes
=
(
long
)
LARGE_VECTOR_SIZE
*
Integer
.
BYTES
+
(
MAX_STORED_BYTES_FOR_CHUNK
)
*
(
8
+
1
*
Integer
.
BYTES
)
+
3
*
(
24
+
16
)
+
190
;
}
...
...
@@ -479,7 +479,7 @@ public final class SparseLong2IntMap {
if
(
useRLE
)
{
flag1
|=
FLAG1_COMP_METHOD_RLE
;
flag1
|=
((
bitsForRLE
<<
2
)
&
FLAG1_RUNLEN_MASK
)
;
boolean
writeIndex
=
useDict
&
(
dict
.
size
()
>
2
);
boolean
writeIndex
=
useDict
&
&
(
dict
.
size
()
>
2
);
int
pos
=
1
;
// first val is written with different method
bitWriter
.
putn
(
tmpChunk
[
pos
++]
-
1
,
bitsForRLE
);
...
...
@@ -539,7 +539,6 @@ public final class SparseLong2IntMap {
putVal
(
bufEncoded
,
currentChunk
[
i
],
4
);
}
}
return
;
}
/**
...
...
@@ -554,10 +553,11 @@ public final class SparseLong2IntMap {
private
void
storeVal
(
int
val
,
int
nb
,
int
sign
)
{
if
(
sign
==
0
)
bitWriter
.
sputn
(
val
,
nb
);
else
if
(
sign
==
1
){
bitWriter
.
putn
(
val
,
nb
-
1
);
}
else
bitWriter
.
putn
(-
val
,
nb
-
1
);
else
if
(
sign
==
1
)
{
bitWriter
.
putn
(
val
,
nb
-
1
);
}
else
{
bitWriter
.
putn
(-
val
,
nb
-
1
);
}
}
private
static
int
readVal
(
BitReader
br
,
int
bits
,
int
sign
)
{
...
...
@@ -743,8 +743,7 @@ public final class SparseLong2IntMap {
flag
=
inBuf
.
get
();
if
((
flag
&
FLAG1_COMP_METHOD_BITS
)
!=
0
)
{
inBuf
.
position
(
inBuf
.
position
()
-
1
);
int
val
=
decodeBits
(
chunkMask
,
targetChunk
,
chunkOffset
,
inBuf
);
return
val
;
return
decodeBits
(
chunkMask
,
targetChunk
,
chunkOffset
,
inBuf
);
}
bytesToUse
=
(
flag
&
FLAG1_USED_BYTES_MASK
)
+
1
;
}
...
...
@@ -814,9 +813,8 @@ public final class SparseLong2IntMap {
if
(
index
==
0
)
return
val
;
int
dictSize
=
dictSizeIs2
?
2
:
1
;
if
(
useDict
)
{
if
(!
dictSizeIs2
)
dictSize
=
br
.
get
(
FLAG_BITS_FOR_DICT_SIZE
)
+
1
;
if
(
useDict
&&
!
dictSizeIs2
)
{
dictSize
=
br
.
get
(
FLAG_BITS_FOR_DICT_SIZE
)
+
1
;
}
int
[]
dict
=
new
int
[
dictSize
];
if
(
useDict
)
{
...
...
@@ -851,8 +849,9 @@ public final class SparseLong2IntMap {
if
(
useRLE
)
{
runLength
=
br
.
get
(
bitsForRLE
)
+
1
;
nVals
+=
runLength
;
}
else
}
else
{
nVals
++;
}
if
(
index
<
nVals
)
return
val
;
if
(
targetChunk
!=
null
)
{
...
...
@@ -863,8 +862,11 @@ public final class SparseLong2IntMap {
if
(
nVals
>=
n
)
break
;
if
(
useDict
)
{
dictPos
=
readIndex
?
br
.
get
(
bitsForPos
)
:
(
dictPos
==
0
)
?
1
:
0
;
;
if
(
readIndex
)
{
dictPos
=
br
.
get
(
bitsForPos
);
}
else
{
dictPos
=
dictPos
==
0
?
1
:
0
;
}
val
=
dict
[
dictPos
];
}
else
{
val
=
readVal
(
br
,
bits
,
sign
)
+
bias
;
...
...
@@ -914,7 +916,6 @@ public final class SparseLong2IntMap {
currentMem
=
null
;
bias1
=
null
;
size
=
0
;
// test();
}
public
long
size
()
{
...
...
@@ -938,21 +939,22 @@ public final class SparseLong2IntMap {
System
.
out
.
println
(
dataDesc
+
" Map is empty"
);
return
;
}
long
totalBytes
=
currentChunk
.
length
*
Integer
.
BYTES
;
long
totalBytes
=
(
long
)
currentChunk
.
length
*
Integer
.
BYTES
;
long
totalChunks
=
1
;
// current chunk
for
(
ChunkMem
mem
:
topMap
.
values
())
{
totalChunks
+=
mem
.
getChunkCount
();
totalBytes
+=
mem
.
estimatedBytes
;
}
float
bytesPerKey
=
(
float
)
(
totalBytes
*
100
/
size
())
/
100
;
long
bytesPerKey
=
Math
.
round
((
double
)
totalBytes
/
size
());
System
.
out
.
println
(
dataDesc
+
" Map: "
+
Utils
.
format
(
size
())
+
" stored long/int pairs require ca. "
+
bytesPerKey
+
" bytes per pair. "
+
Utils
.
format
(
totalChunks
)
+
" chunks are used, the avg. number of values in one "
+
CHUNK_SIZE
+
"-chunk is "
+
(
totalChunks
==
0
?
0
:
(
size
()
/
totalChunks
))
+
"."
);
if
(
msgLevel
>=
0
)
{
String
details
=
dataDesc
+
" Map details: ~"
+
bytesToMB
(
totalBytes
)
+
", including "
+
topMap
.
size
()
+
" array(s) with "
+
bytesToMB
(
LARGE_VECTOR_SIZE
*
Integer
.
BYTES
);
+
" array(s) with "
+
bytesToMB
(
(
long
)
LARGE_VECTOR_SIZE
*
Integer
.
BYTES
);
System
.
out
.
println
(
details
);
}
System
.
out
.
println
();
...
...
Prev
1
2
Next