Support osmium locations-on-ways format (#386)

Author: systemed

include/coordinates.h

@@ -49,6 +49,19 @@ struct LatpLon {
 	int32_t latp;
 	int32_t lon;
 };
+inline bool operator==(const LatpLon &a, const LatpLon &b) { return a.latp==b.latp && a.lon==b.lon; }
+namespace std {
+	/// Hashing function so we can use an unordered_set
+	template<>
+	struct hash<LatpLon> {
+		size_t operator()(const LatpLon &ll) const {
+			return std::hash<int32_t>()(ll.latp) ^ std::hash<int32_t>()(ll.lon);
+		}
+	};
+}
+
+typedef std::vector<LatpLon> LatpLonVec;
+typedef std::deque<LatpLon> LatpLonDeque;
 
 double deg2rad(double deg);
 double rad2deg(double rad);

include/geom.h

@@ -43,9 +43,7 @@ typedef boost::geometry::index::rtree< IndexValue, boost::geometry::index::quadr
 typedef uint64_t NodeID;
 typedef uint64_t WayID;
 
-typedef std::vector<NodeID> NodeVec;
 typedef std::vector<WayID> WayVec;
-typedef std::deque<NodeID> NodeDeque;
 
 // Perform self-intersection aware simplification of geometry types
 Linestring simplify(Linestring const &ls, double max_distance);

include/osm_lua_processing.h

@@ -75,7 +75,7 @@ class OsmLuaProcessing {
 	void setNode(NodeID id, LatpLon node, const tag_map_t &tags);
 
 	/// \brief We are now processing a way
-	void setWay(WayID wayId, NodeVec const &nodeVec, const tag_map_t &tags);
+	void setWay(WayID wayId, LatpLonVec const &llVec, const tag_map_t &tags);
 
 	/** \brief We are now processing a relation
 	 * (note that we store relations as ways with artificial IDs, and that
@@ -194,7 +194,7 @@ class OsmLuaProcessing {
 	/// Internal: clear current cached state
 	inline void reset() {
 		outputs.clear();
-		nodeVecPtr = nullptr;
+		llVecPtr = nullptr;
 		outerWayVecPtr = nullptr;
 		innerWayVecPtr = nullptr;
 		linestringInited = false;
@@ -228,7 +228,7 @@ class OsmLuaProcessing {
 	int relationSubscript = -1;				// in processWay, position in the relation list
 
 	int32_t lon,latp;						///< Node coordinates
-	NodeVec const *nodeVecPtr;
+	LatpLonVec const *llVecPtr;
 	WayVec const *outerWayVecPtr;
 	WayVec const *innerWayVecPtr;
 

include/osm_store.h

@@ -280,59 +280,59 @@ class RelationScanStore {
 class WayStore {
 
 public:
-	using nodeid_vector_t = std::vector<NodeID, mmap_allocator<NodeID>>;
-	using element_t = std::pair<NodeID, nodeid_vector_t>;
+	using latplon_vector_t = std::vector<LatpLon, mmap_allocator<LatpLon>>;
+	using element_t = std::pair<WayID, latplon_vector_t>;
 	using map_t = std::deque<element_t, mmap_allocator<element_t>>;
 
 	void reopen() {
-		mNodeLists = std::make_unique<map_t>();
+		mLatpLonLists = std::make_unique<map_t>();
 	}
 
 	// @brief Lookup a node list
 	// @param i OSM ID of a way
 	// @return A node list
 	// @exception NotFound
-	nodeid_vector_t const &at(WayID wayid) const {
+	latplon_vector_t const &at(WayID wayid) const {
 		std::lock_guard<std::mutex> lock(mutex);
 
-		auto iter = std::lower_bound(mNodeLists->begin(), mNodeLists->end(), wayid, [](auto const &e, auto wayid) { 
+		auto iter = std::lower_bound(mLatpLonLists->begin(), mLatpLonLists->end(), wayid, [](auto const &e, auto wayid) { 
 			return e.first < wayid; 
 		});
 
-		if(iter == mNodeLists->end() || iter->first != wayid)
+		if(iter == mLatpLonLists->end() || iter->first != wayid)
 			throw std::out_of_range("Could not find way with id " + std::to_string(wayid));
 
 		return iter->second;
 	}
 
 	// @brief Insert a node list.
 	// @param i OSM ID of a way
-	// @param nodeVec a node vector to be inserted
+	// @param llVec a coordinate vector to be inserted
 	// @invariant The OSM ID i must be larger than previously inserted OSM IDs of ways
 	//			  (though unnecessarily for current impl, future impl may impose that)
 	void insert_back(std::vector<element_t> &new_ways) {
 		std::lock_guard<std::mutex> lock(mutex);
-		auto i = mNodeLists->size();
-		mNodeLists->resize(i + new_ways.size());
-		std::copy(std::make_move_iterator(new_ways.begin()), std::make_move_iterator(new_ways.end()), mNodeLists->begin() + i); 
+		auto i = mLatpLonLists->size();
+		mLatpLonLists->resize(i + new_ways.size());
+		std::copy(std::make_move_iterator(new_ways.begin()), std::make_move_iterator(new_ways.end()), mLatpLonLists->begin() + i); 
 	}
 
 	// @brief Make the store empty
 	void clear() { 
 		std::lock_guard<std::mutex> lock(mutex);
-		mNodeLists->clear(); 
+		mLatpLonLists->clear(); 
 	}
 
 	std::size_t size() const { 
 		std::lock_guard<std::mutex> lock(mutex);
-		return mNodeLists->size(); 
+		return mLatpLonLists->size(); 
 	}
 
 	void sort(unsigned int threadNum);
 
 private:	
 	mutable std::mutex mutex;
-	std::unique_ptr<map_t> mNodeLists;
+	std::unique_ptr<map_t> mLatpLonLists;
 };
 
 // relation store
@@ -624,7 +624,7 @@ class OSMStore
 	// Relation -> MultiPolygon or MultiLinestring
 	MultiPolygon wayListMultiPolygon(WayVec::const_iterator outerBegin, WayVec::const_iterator outerEnd, WayVec::const_iterator innerBegin, WayVec::const_iterator innerEnd) const;
 	MultiLinestring wayListMultiLinestring(WayVec::const_iterator outerBegin, WayVec::const_iterator outerEnd) const;
-	void mergeMultiPolygonWays(std::vector<NodeDeque> &results, std::map<WayID,bool> &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const;
+	void mergeMultiPolygonWays(std::vector<LatpLonDeque> &results, std::map<WayID,bool> &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const;
 
 	///It is not really meaningful to try using a relation as a linestring. Not normally used but included
 	///if Lua script attempts to do this.
@@ -640,7 +640,7 @@ class OSMStore
 	}
 
 	template<class WayIt>
-	Polygon nodeListPolygon(WayIt begin, WayIt end) const {
+	Polygon llListPolygon(WayIt begin, WayIt end) const {
 		Polygon poly;
 		fillPoints(poly.outer(), begin, end);
 		boost::geometry::correct(poly);
@@ -649,20 +649,19 @@ class OSMStore
 
 	// Way -> Linestring
 	template<class WayIt>
-	Linestring nodeListLinestring(WayIt begin, WayIt end) const {
+	Linestring llListLinestring(WayIt begin, WayIt end) const {
 		Linestring ls;
 		fillPoints(ls, begin, end);
 		return ls;
 	}
 
 private:
 	// helper
-	template<class PointRange, class NodeIt>
-	void fillPoints(PointRange &points, NodeIt begin, NodeIt end) const {
+	template<class PointRange, class LatpLonIt>
+	void fillPoints(PointRange &points, LatpLonIt begin, LatpLonIt end) const {
 		for (auto it = begin; it != end; ++it) {
 			try {
-				LatpLon ll = nodes_at(*it);
-				boost::geometry::range::push_back(points, boost::geometry::make<Point>(ll.lon/10000000.0, ll.latp/10000000.0));
+				boost::geometry::range::push_back(points, boost::geometry::make<Point>(it->lon/10000000.0, it->latp/10000000.0));
 			} catch (std::out_of_range &err) {
 				if (require_integrity) throw err;
 			}

include/osmformat.proto

@@ -200,6 +200,8 @@ message Way {
    optional Info info = 4;
 
    repeated sint64 refs = 8 [packed = true];  // DELTA coded
+   repeated sint64 lats = 9  [packed = true];
+   repeated sint64 lons = 10 [packed = true];
 }
 
 message Relation {

include/read_pbf.h

@@ -45,10 +45,11 @@ class PbfReader
 	}
 
 private:
-	bool ReadBlock(std::istream &infile, OsmLuaProcessing &output, std::pair<std::size_t, std::size_t> progress, std::size_t datasize, std::unordered_set<std::string> const &nodeKeys, ReadPhase phase = ReadPhase::All);
+	bool ReadBlock(std::istream &infile, OsmLuaProcessing &output, std::pair<std::size_t, std::size_t> progress, std::size_t datasize, 
+	               std::unordered_set<std::string> const &nodeKeys, bool locationsOnWays, ReadPhase phase = ReadPhase::All);
 	bool ReadNodes(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb, const std::unordered_set<int> &nodeKeyPositions);
 
-	bool ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb);
+	bool ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb, bool locationsOnWays);
 	bool ScanRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb);
 	bool ReadRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb);
 

src/osm_lua_processing.cpp

@@ -281,7 +281,7 @@ double OsmLuaProcessing::Length() {
 const Linestring &OsmLuaProcessing::linestringCached() {
 	if (!linestringInited) {
 		linestringInited = true;
-		linestringCache = osmStore.nodeListLinestring(nodeVecPtr->cbegin(),nodeVecPtr->cend());
+		linestringCache = osmStore.llListLinestring(llVecPtr->cbegin(),llVecPtr->cend());
 	}
 	return linestringCache;
 }
@@ -297,7 +297,7 @@ const MultiLinestring &OsmLuaProcessing::multiLinestringCached() {
 const Polygon &OsmLuaProcessing::polygonCached() {
 	if (!polygonInited) {
 		polygonInited = true;
-		polygonCache = osmStore.nodeListPolygon(nodeVecPtr->cbegin(), nodeVecPtr->cend());
+		polygonCache = osmStore.llListPolygon(llVecPtr->cbegin(), llVecPtr->cend());
 	}
 	return polygonCache;
 }
@@ -561,13 +561,13 @@ void OsmLuaProcessing::setNode(NodeID id, LatpLon node, const tag_map_t &tags) {
 }
 
 // We are now processing a way
-void OsmLuaProcessing::setWay(WayID wayId, NodeVec const &nodeVec, const tag_map_t &tags) {
+void OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const tag_map_t &tags) {
 	reset();
 	osmID = (wayId & OSMID_MASK) | OSMID_WAY;
 	originalOsmID = wayId;
 	isWay = true;
 	isRelation = false;
-	nodeVecPtr = &nodeVec;
+	llVecPtr = &llVec;
 	outerWayVecPtr = nullptr;
 	innerWayVecPtr = nullptr;
 	linestringInited = polygonInited = multiPolygonInited = false;
@@ -579,7 +579,7 @@ void OsmLuaProcessing::setWay(WayID wayId, NodeVec const &nodeVec, const tag_map
 	}
 
 	try {
-		isClosed = nodeVecPtr->front()==nodeVecPtr->back();
+		isClosed = llVecPtr->front()==llVecPtr->back();
 
 	} catch (std::out_of_range &err) {
 		std::stringstream ss;
@@ -608,7 +608,7 @@ void OsmLuaProcessing::setWay(WayID wayId, NodeVec const &nodeVec, const tag_map
 		// create a list of tiles this way passes through (tileSet)
 		unordered_set<TileCoordinates> tileSet;
 		try {
-			insertIntermediateTiles(osmStore.nodeListLinestring(nodeVecPtr->cbegin(),nodeVecPtr->cend()), this->config.baseZoom, tileSet);
+			insertIntermediateTiles(osmStore.llListLinestring(llVecPtr->cbegin(),llVecPtr->cend()), this->config.baseZoom, tileSet);
 
 			// then, for each tile, store the OutputObject for each layer
 			bool polygonExists = false;
@@ -651,7 +651,7 @@ void OsmLuaProcessing::setRelation(int64_t relationId, WayVec const &outerWayVec
 	isRelation = true;
 	isClosed = isNativeMP;
 
-	nodeVecPtr = nullptr;
+	llVecPtr = nullptr;
 	outerWayVecPtr = &outerWayVec;
 	innerWayVecPtr = &innerWayVec;
 	currentTags = tags;

src/osm_store.cpp

@@ -270,12 +270,12 @@ void NodeStore::sort(unsigned int threadNum) {
 void WayStore::sort(unsigned int threadNum) { 
 	std::lock_guard<std::mutex> lock(mutex);
 	boost::sort::block_indirect_sort(
-		mNodeLists->begin(), mNodeLists->end(), 
+		mLatpLonLists->begin(), mLatpLonLists->end(), 
 		[](auto const &a, auto const &b) { return a.first < b.first; }, 
 		threadNum);
 }
 
-static inline bool isClosed(WayStore::nodeid_vector_t const &way) {
+static inline bool isClosed(WayStore::latplon_vector_t const &way) {
 	return way.begin() == way.end();
 }
 
@@ -345,8 +345,8 @@ MultiPolygon OSMStore::wayListMultiPolygon(WayVec::const_iterator outerBegin, Wa
 	MultiPolygon mp;
 	if (outerBegin == outerEnd) { return mp; } // no outers so quit
 
-	std::vector<NodeDeque> outers;
-	std::vector<NodeDeque> inners;
+	std::vector<LatpLonDeque> outers;
+	std::vector<LatpLonDeque> inners;
 	std::map<WayID,bool> done; // true=this way has already been added to outers/inners, don't reconsider
 
 	// merge constituent ways together
@@ -379,7 +379,7 @@ MultiLinestring OSMStore::wayListMultiLinestring(WayVec::const_iterator outerBeg
 	MultiLinestring mls;
 	if (outerBegin == outerEnd) { return mls; }
 
-	std::vector<NodeDeque> linestrings;
+	std::vector<LatpLonDeque> linestrings;
 	std::map<WayID,bool> done;
 
 	mergeMultiPolygonWays(linestrings, done, outerBegin, outerEnd);
@@ -399,11 +399,11 @@ MultiLinestring OSMStore::wayListMultiLinestring(WayVec::const_iterator outerBeg
 // - If no matches can be found, then one linestring is added (to 'attract' others)
 // - The process is rerun until no ways are left
 // There's quite a lot of copying going on here - could potentially be addressed
-void OSMStore::mergeMultiPolygonWays(std::vector<NodeDeque> &results, std::map<WayID,bool> &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const {
+void OSMStore::mergeMultiPolygonWays(std::vector<LatpLonDeque> &results, std::map<WayID,bool> &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const {
 
 	// Create maps of start/end nodes
-	std::unordered_map<NodeID, std::vector<NodeID>> startNodes;
-	std::unordered_map<NodeID, std::vector<NodeID>> endNodes;
+	std::unordered_map<LatpLon, std::vector<WayID>> startNodes;
+	std::unordered_map<LatpLon, std::vector<WayID>> endNodes;
 	for (auto it = itBegin; it != itEnd; ++it) {
 		if (done[*it]) { continue; }
 		try {
@@ -422,16 +422,16 @@ void OSMStore::mergeMultiPolygonWays(std::vector<NodeDeque> &results, std::map<W
 		}
 	}
 
-	auto deleteFromWayList = [&](NodeID n, NodeID w, bool which) {
+	auto deleteFromWayList = [&](LatpLon n, WayID w, bool which) {
 		auto &nodemap = which ? startNodes : endNodes;
-		std::vector<NodeID> &waylist = nodemap.find(n)->second;
+		std::vector<WayID> &waylist = nodemap.find(n)->second;
 		waylist.erase(std::remove(waylist.begin(), waylist.end(), w), waylist.end());
 		if (waylist.empty()) { nodemap.erase(nodemap.find(n)); }
 	};
-	auto removeWay = [&](NodeID w) {
+	auto removeWay = [&](WayID w) {
 		auto const &way = ways.at(w);
-		NodeID first = way.front();
-		NodeID last  = way.back();
+		LatpLon first = way.front();
+		LatpLon last  = way.back();
 		if (startNodes.find(first) != startNodes.end()) { deleteFromWayList(first, w, true ); }
 		if (startNodes.find(last)  != startNodes.end()) { deleteFromWayList(last,  w, true ); }
 		if (endNodes.find(first)   != endNodes.end()  ) { deleteFromWayList(first, w, false); }
@@ -446,8 +446,8 @@ void OSMStore::mergeMultiPolygonWays(std::vector<NodeDeque> &results, std::map<W
 		for (auto rt = results.begin(); rt != results.end(); rt++) {
 			bool working=true;
 			do {
-				NodeID rFirst = rt->front();
-				NodeID rLast  = rt->back();
+				LatpLon rFirst = rt->front();
+				LatpLon rLast  = rt->back();
 				if (rFirst==rLast) break;
 				if (startNodes.find(rLast)!=startNodes.end()) {
 					// append to the result
@@ -494,7 +494,7 @@ void OSMStore::mergeMultiPolygonWays(std::vector<NodeDeque> &results, std::map<W
 		for (int i=0; i<=1; i++) {
 			if (added>0) continue;
 			for (auto nt : (i==0 ? startNodes : endNodes)) {
-				NodeID w = nt.second.back();
+				WayID w = nt.second.back();
 				auto const &way = ways.at(w);
 				results.emplace_back(way.begin(), way.end());
 				added++;