intermediate commit

src/pfaedle/osm/OsmBuilder.cpp

@@ -507,6 +507,7 @@ void OsmBuilder::readBBoxNds(OsmSource* source, OsmIdSet* nodes,
   const OsmSourceNode* nd;
 
   while ((nd = source->nextNode())) {
+    std::cout << " GOT NODE " << nd->id << std::endl;
     if (bbox.contains(Point<double>(nd->lon, nd->lat))) {
       nodes->add(nd->id);
     } else {

src/pfaedle/osm/source/OsmSource.h

@@ -46,7 +46,7 @@ class OsmSource {
   virtual uint64_t nextMemberNode() = 0;
   virtual const OsmSourceRelationMember* nextMember() = 0;
   virtual const OsmSourceRelation* nextRel() = 0;
-  virtual void cont() = 0;
+  virtual bool cont() = 0;
 
   virtual ~OsmSource() {};
 

src/pfaedle/osm/source/PBFSource.cpp

@@ -6,7 +6,6 @@
 #include <unistd.h>
 
 #include "pfaedle/osm/source/PBFSource.h"
-#include "protozero/pbf_reader.hpp"
 #include "util/Misc.h"
 #ifndef PFXML_NO_ZLIB
 #include <zlib.h>
@@ -41,15 +40,15 @@ PBFSource::PBFSource(const std::string& path) : _path(path) {
   read(_file, _buf[_which], BUFFER_S);
   _c = _buf[_which];
 
+  // get header block
   getNextBlock();
 
-  getNextBlock();
-
-  exit(1);
+  // get first content block
+  checkGroup();
 }
 
 // _____________________________________________________________________________
-void PBFSource::getNextBlock() {
+bool PBFSource::getNextBlock() {
   // block begins by big-endian 32bit length of BlobHeader
   uint32_t blobHeaderLength = ((*(_c + 3)) << 0) | ((*(_c + 2)) << 8) |
                               ((*(_c + 1)) << 16) | ((*_c) << 24);
@@ -64,13 +63,15 @@ void PBFSource::getNextBlock() {
 
   if (header.type == "OSMHeader") {
     parseOSMHeader(parseBlob(header.datasize));
-    return;
+    return true;
   }
 
   if (header.type == "OSMData") {
-    parseOSMData(parseBlob(header.datasize));
-    return;
+    _curBlock = parseOSMData(parseBlob(header.datasize));
+    return true;
   }
+
+  return false;
 }
 
 // _____________________________________________________________________________
@@ -84,9 +85,10 @@ util::geo::Box<double> PBFSource::parseHeaderBBox(unsigned char*& c) {
     auto typeId = nextTypeAndId(c);
 
     if (typeId.second == 1) llx = parseVarInt(c);
-    if (typeId.second == 4) lly = parseVarInt(c);
-    if (typeId.second == 2) urx = parseVarInt(c);
-    if (typeId.second == 3) ury = parseVarInt(c);
+    else if (typeId.second == 4) lly = parseVarInt(c);
+    else if (typeId.second == 2) urx = parseVarInt(c);
+    else if (typeId.second == 3) ury = parseVarInt(c);
+    else skipType(typeId.first, c);
   }
 
   return {{llx * 1.0, lly * 1.0}, {urx * 1.0, ury * 1.0}};
@@ -105,8 +107,9 @@ PBFSource::PrimitiveBlock PBFSource::parseOSMData(const Blob& blob) {
     if (typeId.second == 1) {
       block.stringTable = parseStringTable(c);
     } else if (typeId.second == 2) {
-      block.primitiveGroups = c;
-      // TODO: skip
+      block.primitiveGroups.push(c);
+      size_t len = parseVarUInt(c);
+      c += len;
     } else if (typeId.second == 17) {
       block.granularity = parseVarUInt(c);
     } else if (typeId.second == 19) {
@@ -129,8 +132,6 @@ std::vector<std::string> PBFSource::parseStringTable(unsigned char *& c) {
 
   size_t len = parseVarUInt(c);
 
-  std::cout << len << std::endl;
-
   std::vector<std::string> table;
   table.reserve(len / 10);
 
@@ -297,7 +298,7 @@ uint64_t PBFSource::parseUInt(std::pair<VarType, uint8_t> typeId) {
 // _____________________________________________________________________________
 int64_t PBFSource::parseVarInt(unsigned char*& c) {
   int64_t i = parseVarUInt(c);
-  return (i << 1) ^ (i >> 31);
+  return (i >> 1) ^ (-(i & 1));
 }
 
 // _____________________________________________________________________________
@@ -389,7 +390,35 @@ PBFSource::~PBFSource() {
 }
 
 // _____________________________________________________________________________
-const OsmSourceNode* PBFSource::nextNode() { return 0; }
+const OsmSourceNode* PBFSource::nextNode() {
+  do {
+    if (_curBlock.denseNodePtr < _curBlock.curDenseNodes.size()) {
+      _curNode.id = _curBlock.curDenseNodes[_curBlock.denseNodePtr++].id;
+      return &_curNode;
+    }
+
+    auto typeId = nextTypeAndId(_curBlock.c);
+    if (typeId.second == 1) {
+      size_t len = parseVarUInt(_curBlock.c);
+      _curBlock.c += len;
+    } else if (typeId.second == 2) {
+      _curBlock.curDenseNodes = parseDenseNodes(_curBlock.c);
+      _curBlock.denseNodePtr = 0;
+
+      if (_curBlock.denseNodePtr < _curBlock.curDenseNodes.size()) {
+        _curNode.id = _curBlock.curDenseNodes[_curBlock.denseNodePtr++].id;
+        return &_curNode;
+      }
+    } else {
+      // ignore non-nodes here
+      size_t len = parseVarUInt(_curBlock.c);
+      _curBlock.c += len;
+
+    }
+  } while (checkGroup());
+
+  return 0;
+}
 
 // _____________________________________________________________________________
 void PBFSource::seekNodes() {}
@@ -401,7 +430,85 @@ void PBFSource::seekWays() {}
 void PBFSource::seekRels() {}
 
 // _____________________________________________________________________________
-void PBFSource::cont() {}
+bool PBFSource::checkGroup() {
+  while (true) {
+    // skip to first non-empty block
+    while (_curBlock.primitiveGroups.size() == 0 && getNextBlock()) {}
+
+    if (_curBlock.primitiveGroups.size() == 0) return false;
+
+    if (_curBlock.c == 0) {
+      _curBlock.c = _curBlock.primitiveGroups.front();
+      _curBlock.curGroupLen = parseVarUInt(_curBlock.c);
+      _curBlock.primitiveGroups.front() = _curBlock.c;
+
+      std::cout << "CUR GROUP LEN: " <<  _curBlock.curGroupLen << std::endl;
+    }
+
+    if ((size_t)(_curBlock.c - _curBlock.primitiveGroups.front()) >= _curBlock.curGroupLen) {
+      _curBlock.primitiveGroups.pop();
+      _curBlock.c = 0;
+      continue;
+    }
+
+    return true;
+  }
+}
+
+// _____________________________________________________________________________
+std::vector<PBFSource::Node> PBFSource::parseDenseNodes(unsigned char*& c) {
+  std::vector<PBFSource::Node> ret;
+
+  size_t len = parseVarUInt(c);
+  auto start = c;
+
+  while ((size_t)(c - start) < len) {
+    auto typeId = nextTypeAndId(c);
+
+    if (typeId.second == 1) {
+      // IDs
+      size_t len = parseVarUInt(c);
+      auto start = c;
+      size_t i = 0;
+      if (ret.size()) {
+        size_t i =0;
+        while ((size_t)(c - start) < len) {
+          auto nid = parseVarInt(c);
+          if (i > 0) nid = ret[i-1].id + nid;
+          ret[i++].id = nid;
+        }
+      } else {
+        while ((size_t)(c - start) < len) {
+          auto nid = parseVarInt(c);
+          if (ret.size() > 0) nid = ret.back().id + nid;
+          ret.push_back({0, 0, nid, {}});
+        }
+      }
+    } else if (typeId.second == 5) {
+      // denseinfo
+      size_t len = parseVarUInt(c);
+
+      c += len;
+    } else if (typeId.second == 8) {
+      // lat
+      size_t len = parseVarUInt(c);
+
+      c += len;
+    } else if (typeId.second == 9) {
+      // lon
+      size_t len = parseVarUInt(c);
+
+      c += len;
+    } else if (typeId.second == 10) {
+      // tags
+      size_t len = parseVarUInt(c);
+
+      c += len;
+    }
+  }
+
+  return ret;
+}
 
 // _____________________________________________________________________________
 const OsmSourceWay* PBFSource::nextWay() { return 0; }
@@ -416,7 +523,9 @@ uint64_t PBFSource::nextMemberNode() { return 0; }
 const OsmSourceRelation* PBFSource::nextRel() { return 0; }
 
 // _____________________________________________________________________________
-const OsmSourceAttr PBFSource::nextAttr() {}
+const OsmSourceAttr PBFSource::nextAttr() {
+  return {0, 0};
+}
 
 // _____________________________________________________________________________
 util::geo::Box<double> PBFSource::getBounds() {}

src/pfaedle/osm/source/PBFSource.h

@@ -5,6 +5,7 @@
 #ifndef PFAEDLE_OSM_SOURCE_PBFSOURCE_H_
 #define PFAEDLE_OSM_SOURCE_PBFSOURCE_H_
 
+#include <queue>
 #include "pfaedle/osm/source/OsmSource.h"
 #include "util/geo/Geo.h"
 
@@ -21,6 +22,12 @@ class PBFSource : public OsmSource {
     I = 5,
   };
 
+  struct Node {
+    double lat, lon;
+    uint64_t id;
+    std::vector<OsmSourceAttr> tags;
+  };
+
   struct OSMHeader {
     util::geo::Box<double> bbox;
     std::vector<std::string> requiredFeatures;
@@ -30,11 +37,17 @@ class PBFSource : public OsmSource {
 
   struct PrimitiveBlock {
     std::vector<std::string> stringTable;
-    std::vector<unsigned char*> primitiveGroups;
+    std::queue<unsigned char*> primitiveGroups;
     uint32_t granularity = 100;
     uint64_t latOffset = 0;
     uint64_t lonOffset = 0;
     uint32_t dateGranularity = 1000;
+
+    unsigned char* c = 0;
+    size_t curGroupLen = 0;
+
+    std::vector<Node> curDenseNodes;
+    size_t denseNodePtr = 0;
   };
 
   struct BlobHeader {
@@ -56,7 +69,7 @@ class PBFSource : public OsmSource {
   virtual uint64_t nextMemberNode();
   virtual const OsmSourceRelationMember* nextMember();
   virtual const OsmSourceRelation* nextRel();
-  virtual void cont();
+  virtual bool cont() {};
 
   virtual void seekNodes();
   virtual void seekWays();
@@ -77,7 +90,9 @@ class PBFSource : public OsmSource {
 
   uint8_t _which = 0;
 
-  void getNextBlock();
+  PrimitiveBlock _curBlock;
+
+  bool getNextBlock();
   OSMHeader parseOSMHeader(const Blob& blob);
   PrimitiveBlock parseOSMData(const Blob& blob);
   BlobHeader parseBlobHeader(size_t len);
@@ -98,7 +113,12 @@ class PBFSource : public OsmSource {
   uint64_t parseUInt(std::pair<VarType, uint8_t> typeId);
   void skipType(VarType type, unsigned char *& c);
   void skipType(VarType type);
+  virtual bool checkGroup();
   util::geo::Box<double> parseHeaderBBox(unsigned char*& c);
+
+  std::vector<Node> parseDenseNodes(unsigned char*& c);
+
+  OsmSourceNode _curNode;
 };
 
 }  // namespace source

src/pfaedle/osm/source/XMLSource.cpp

@@ -76,8 +76,8 @@ void XMLSource::seekRels() {
 }
 
 // _____________________________________________________________________________
-void XMLSource::cont() {
-  _xml.next();
+bool XMLSource::cont() {
+  return _xml.next();
 }
 
 // _____________________________________________________________________________

src/pfaedle/osm/source/XMLSource.h

@@ -22,7 +22,7 @@ class XMLSource : public OsmSource {
   virtual uint64_t nextMemberNode();
   virtual const OsmSourceRelationMember* nextMember();
   virtual const OsmSourceRelation* nextRel();
-  virtual void cont();
+  virtual bool cont();
 
   virtual void seekNodes();
   virtual void seekWays();