From 952ace7bab5ea978fd39b253b266d951c7630ce5 Mon Sep 17 00:00:00 2001 From: Thomas Rasch Date: Mon, 12 Aug 2024 17:09:05 +0200 Subject: [PATCH] cli tool accepts GeoJSON as input for some commands --- Package.resolved | 6 +- Package.swift | 2 +- Sources/MVTCLI/CLI.swift | 6 +- Sources/MVTCLI/Dump.swift | 25 ++-- Sources/MVTCLI/Info.swift | 18 +-- Sources/MVTCLI/Query.swift | 64 +++++++--- Sources/MVTTools/GeoJson.swift | 10 +- Sources/MVTTools/Info.swift | 74 ++++++++---- Sources/MVTTools/VectorTile.swift | 136 ++++++++++++---------- Tests/MVTToolsTests/VectorTileTests.swift | 9 +- 10 files changed, 216 insertions(+), 134 deletions(-) diff --git a/Package.resolved b/Package.resolved index 761a219..96a5d34 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "5d734874e4d66aa5e42e2e73f545fbb1c3e33e7f08847495a19cf43690d8a3e5", + "originHash" : "e36efc60ff6513f9fd73eec6246018b16a077ae5221e7e54d785cf0a8c172a1a", "pins" : [ { "identity" : "gis-tools", "kind" : "remoteSourceControl", "location" : "https://github.com/Outdooractive/gis-tools", "state" : { - "revision" : "831a87ebd1ae3bb013c46b1d1752400dbc7bc352", - "version" : "1.7.0" + "revision" : "a8118bcd5e715a69f640476446fbf058fe39973f", + "version" : "1.8.3" } }, { diff --git a/Package.swift b/Package.swift index d5dc808..8c46789 100644 --- a/Package.swift +++ b/Package.swift @@ -24,7 +24,7 @@ let package = Package( targets: ["MVTTools"]), ], dependencies: [ - .package(url: "https://github.com/Outdooractive/gis-tools", from: "1.7.0"), + .package(url: "https://github.com/Outdooractive/gis-tools", from: "1.8.3"), .package(url: "https://github.com/1024jp/GzipSwift.git", from: "5.2.0"), .package(url: "https://github.com/apple/swift-argument-parser", from: "1.4.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.6.1"), diff --git a/Sources/MVTCLI/CLI.swift b/Sources/MVTCLI/CLI.swift index a2e4150..119a118 100644 --- a/Sources/MVTCLI/CLI.swift +++ b/Sources/MVTCLI/CLI.swift @@ -12,7 +12,9 @@ struct CLI: AsyncParsableCommand { commandName: "mvt", abstract: "A utility for inspecting and working with vector tiles.", discussion: """ - The tile coordinate can be extracted from the path if it's either in the form '/z/x/y' or 'z_x_y'. + The tile coordinate of vector tiles can be extracted from the path + if it's either in the form '/z/x/y' or 'z_x_y'. + Tile coordinates are not necessary for GeoJSON input files. Examples: - Tests/MVTToolsTests/TestData/14_8716_8015.vector.mvt @@ -50,6 +52,8 @@ struct XYZOptions: ParsableArguments { @Option(name: .short, help: "Tile zoom level, if it can't be extracted from the path") var z: Int? + /// Try to extract x, y and z tile coordinates from some file paths or URLs, + /// if the were not given on the command line mutating func parseXYZ( fromPaths paths: [String]) throws -> (Int, Int, Int) diff --git a/Sources/MVTCLI/Dump.swift b/Sources/MVTCLI/Dump.swift index 29b6c6c..5f4027b 100644 --- a/Sources/MVTCLI/Dump.swift +++ b/Sources/MVTCLI/Dump.swift @@ -6,7 +6,7 @@ extension CLI { struct Dump: AsyncParsableCommand { - static let configuration = CommandConfiguration(abstract: "Print the vector tile as pretty-printed GeoJSON to the console") + static let configuration = CommandConfiguration(abstract: "Print the input file (mvt or GeoJSON) as pretty-printed GeoJSON to the console") @Option(name: .shortAndLong, help: "Dump only the specified layer (can be repeated)") var layer: [String] = [] @@ -18,28 +18,33 @@ extension CLI { var options: Options @Argument( - help: "The vector tile (file or URL)", - completion: .file(extensions: ["pbf", "mvt"])) + help: "The vector tile or GeoJSON (file or URL)", + completion: .file(extensions: ["pbf", "mvt", "geojson", "json"])) var path: String mutating func run() async throws { - let (x, y, z) = try xyzOptions.parseXYZ(fromPaths: [path]) + let layerAllowlist = layer.nonempty let url = try options.parseUrl(fromPath: path) - let layerAllowlist = layer.nonempty + var tile = VectorTile(contentsOfGeoJson: url, layerWhitelist: layerAllowlist, logger: options.verbose ? CLI.logger : nil) + if tile == nil, + let (x, y, z) = try? xyzOptions.parseXYZ(fromPaths: [path]) + { + tile = VectorTile(contentsOf: url, x: x, y: y, z: z, layerWhitelist: layerAllowlist, logger: options.verbose ? CLI.logger : nil) + } + + guard let tile else { + throw CLIError("Failed to parse the resource at '\(path)'") + } if options.verbose { - print("Dumping tile '\(url.lastPathComponent)' [\(x),\(y)]@\(z)") + print("Dumping tile '\(url.lastPathComponent)' [\(tile.x),\(tile.y)]@\(tile.z)") if let layerAllowlist { print("Layers: '\(layerAllowlist.joined(separator: ","))'") } } - guard let tile = VectorTile(contentsOf: url, x: x, y: y, z: z, layerWhitelist: layerAllowlist, logger: options.verbose ? CLI.logger : nil) else { - throw CLIError("Failed to parse the resource at '\(path)'") - } - guard let data = tile.toGeoJson(prettyPrinted: true) else { throw CLIError("Failed to extract the tile data as GeoJSON") } diff --git a/Sources/MVTCLI/Info.swift b/Sources/MVTCLI/Info.swift index dcd15d0..f45eed5 100644 --- a/Sources/MVTCLI/Info.swift +++ b/Sources/MVTCLI/Info.swift @@ -6,25 +6,21 @@ extension CLI { struct Info: AsyncParsableCommand { - static let configuration = CommandConfiguration(abstract: "Print information about the vector tile") + static let configuration = CommandConfiguration(abstract: "Print information about the input file (mvt or GeoJSON)") @OptionGroup var options: Options @Argument( - help: "The vector tile (file or URL)", - completion: .file(extensions: ["pbf", "mvt"])) + help: "The vector tile or GeoJSON (file or URL)", + completion: .file(extensions: ["pbf", "mvt", "geojson", "json"])) var path: String mutating func run() async throws { let url = try options.parseUrl(fromPath: path) - if options.verbose { - print("Info for tile '\(url.lastPathComponent)'") - } - - guard let tileInfo = VectorTile.tileInfo(at: url), - var layers = tileInfo["layers"] as? [[String: Any]] + guard var layers = VectorTile.tileInfo(at: url) + ?? VectorTile(contentsOfGeoJson: url)?.tileInfo() else { throw CLIError("Error retreiving the tile info for '\(path)'") } layers.sort { first, second in @@ -46,6 +42,10 @@ extension CLI { layers.compactMap({ ($0["version"] as? Int)?.toString }), ] + if options.verbose { + print("Info for tile '\(url.lastPathComponent)'") + } + let result = dumpSideBySide( table, asTableWithHeaders: tableHeader) diff --git a/Sources/MVTCLI/Query.swift b/Sources/MVTCLI/Query.swift index 67e457c..c22899c 100644 --- a/Sources/MVTCLI/Query.swift +++ b/Sources/MVTCLI/Query.swift @@ -10,11 +10,20 @@ extension CLI { struct Query: AsyncParsableCommand { - static let configuration = CommandConfiguration(abstract: "Query the features in a vector tile") + static let configuration = CommandConfiguration(abstract: "Query the features in the input file (mvt or GeoJSON)") + + @Option(name: .shortAndLong, help: "Output GeoJSON file (optional, default is console)") + var output: String? + + @Flag(name: .shortAndLong, help: "Force overwrite existing files") + var forceOverwrite = false @Option(name: .shortAndLong, help: "Search only in this layer (can be repeated)") var layer: [String] = [] + @Flag(name: .shortAndLong, help: "Pretty-print the output GeoJSON") + var prettyPrint = false + @OptionGroup var xyzOptions: XYZOptions @@ -22,14 +31,26 @@ extension CLI { var options: Options @Argument( - help: "The vector tile (file or URL)", - completion: .file(extensions: ["pbf", "mvt"])) + help: "The vector tile or GeoJSON (file or URL)", + completion: .file(extensions: ["pbf", "mvt", "geojson", "json"])) var path: String @Argument(help: "Search term, can be a string or a coordinate in the form 'latitude,longitude,tolerance(meters)'") var searchTerm: String mutating func run() async throws { + if let output { + let outputUrl = URL(fileURLWithPath: output) + if (try? outputUrl.checkResourceIsReachable()) ?? false { + if forceOverwrite { + print("Existing file '\(outputUrl.lastPathComponent)' will be overwritten") + } + else { + throw CLIError("Output file must not exist (use --force-overwrite to overwrite existing files)") + } + } + } + var coordinate: Coordinate3D? var tolerance: CLLocationDistance? @@ -47,17 +68,22 @@ extension CLI { } } - let (x, y, z) = try xyzOptions.parseXYZ(fromPaths: [path]) + let layerAllowlist = layer.nonempty let url = try options.parseUrl(fromPath: path) - let layerAllowlist = layer.nonempty + var tile = VectorTile(contentsOfGeoJson: url, layerWhitelist: layerAllowlist, logger: options.verbose ? CLI.logger : nil) + if tile == nil, + let (x, y, z) = try? xyzOptions.parseXYZ(fromPaths: [path]) + { + tile = VectorTile(contentsOf: url, x: x, y: y, z: z, layerWhitelist: layerAllowlist, logger: options.verbose ? CLI.logger : nil) + } - guard let tile = VectorTile(contentsOf: url, x: x, y: y, z: z, layerWhitelist: layerAllowlist, logger: options.verbose ? CLI.logger : nil) else { - throw CLIError("Failed to parse the resource at \(path)") + guard let tile else { + throw CLIError("Failed to parse the resource at '\(path)'") } if options.verbose { - print("Searching in tile '\(url.lastPathComponent)' [\(x),\(y)]@\(z)") + print("Searching in tile '\(url.lastPathComponent)' [\(tile.x),\(tile.y)]@\(tile.z)") if let layerAllowlist { print("Layers: '\(layerAllowlist.joined(separator: ","))'") @@ -80,15 +106,19 @@ extension CLI { result = search(term: searchTerm, in: tile) } - if let result, - let output = result.asJsonString(prettyPrinted: true) - { - print(output, terminator: "") - print() - - if options.verbose { - let count = result.features.count - print("Found \(count) \(count == 1 ? "result" : "results").") + if let result { + if let output { + let outputUrl = URL(fileURLWithPath: output) + try result.asJsonData(prettyPrinted: prettyPrint)?.write(to: outputUrl, options: .atomic) + } + else if let resultGeoJson = result.asJsonString(prettyPrinted: prettyPrint) { + print(resultGeoJson, terminator: "") + print() + + if options.verbose { + let count = result.features.count + print("Found \(count) \(count == 1 ? "result" : "results").") + } } } else { diff --git a/Sources/MVTTools/GeoJson.swift b/Sources/MVTTools/GeoJson.swift index 0a932d7..f45c495 100644 --- a/Sources/MVTTools/GeoJson.swift +++ b/Sources/MVTTools/GeoJson.swift @@ -70,7 +70,8 @@ extension VectorTile { public mutating func addGeoJson( geoJson: GeoJson, layerName: String? = nil, - propertyName: String? = nil) + propertyName: String? = nil, + layerAllowList: Set? = nil) { guard let features = geoJson.flattened?.features else { return } @@ -83,10 +84,12 @@ extension VectorTile { return mapping }, onKey: { key, features in + if let layerAllowList, !layerAllowList.contains(key) { return } appendFeatures(features, to: key) }) } else { + if let layerAllowList, !layerAllowList.contains(layerName) { return } appendFeatures(features, to: layerName) } } @@ -95,7 +98,8 @@ extension VectorTile { public mutating func setGeoJson( geoJson: GeoJson, layerName: String? = nil, - propertyName: String? = nil) + propertyName: String? = nil, + layerAllowList: Set? = nil) { guard let features = geoJson.flattened?.features else { return } @@ -108,10 +112,12 @@ extension VectorTile { return mapping }, onKey: { key, features in + if let layerAllowList, !layerAllowList.contains(key) { return } setFeatures(features, for: key) }) } else { + if let layerAllowList, !layerAllowList.contains(layerName) { return } setFeatures(features, for: layerName) } } diff --git a/Sources/MVTTools/Info.swift b/Sources/MVTTools/Info.swift index d84b494..fd886d0 100644 --- a/Sources/MVTTools/Info.swift +++ b/Sources/MVTTools/Info.swift @@ -4,7 +4,7 @@ import Foundation import GISTools -// MARK: Static functions +// MARK: Info functions extension VectorTile { @@ -20,22 +20,55 @@ extension VectorTile { return layerNames(from: data) } - // { layers: - // [ { name: 'world', - // features: 1, - // point_features: 0, - // linestring_features: 0, - // polygon_features: 1, - // unknown_features: 0, - // raster_features: 0, - // version: 2 } ], - // errors: false } - - /// Read a tile from `data` and return some information about the tile - public static func tileInfo(from data: Data) -> [String: Any]? { + // [{ + // name: 'world', + // features: 1, + // point_features: 0, + // linestring_features: 0, + // polygon_features: 1, + // unknown_features: 0, + // raster_features: 0, + // version: 2 + // }] + + /// Information about the features in a tile, per layer. + public func tileInfo() -> [[String: Any]]? { + var result: [[String: Any]] = [] + + for (layerName, layerContainer) in layers { + var pointFeatures = 0 + var lineStringFeatures = 0 + var polygonFeatures = 0 + var unknownFeatures = 0 + + for feature in layerContainer.features { + switch feature.geometry.type { + case .point, .multiPoint: pointFeatures += 1 + case .lineString, .multiLineString: lineStringFeatures += 1 + case .polygon, .multiPolygon: polygonFeatures += 1 + default: unknownFeatures += 1 + } + } + + let info: [String: Any] = [ + "name": layerName, + "features": pointFeatures + lineStringFeatures + polygonFeatures + unknownFeatures, + "point_features": pointFeatures, + "linestring_features": lineStringFeatures, + "polygon_features": polygonFeatures, + "unknown_features": unknownFeatures, + ] + result.append(info) + } + + return result + } + + /// Read a tile from `data` and return some information about the tile per layer. + public static func tileInfo(from data: Data) -> [[String: Any]]? { guard let tile = MVTDecoder.vectorTile(from: data) else { return nil } - var layers: [[String: Any]] = [] + var result: [[String: Any]] = [] for layer in tile.layers { var pointFeatures = 0 @@ -61,17 +94,14 @@ extension VectorTile { "polygon_features": polygonFeatures, "unknown_features": unknownFeatures, ] - layers.append(info) + result.append(info) } - return [ - "layers": layers, - "errors": false, - ] + return result } - /// Read a tile from `url` and return some information about the tile - public static func tileInfo(at url: URL) -> [String: Any]? { + /// Read a tile from `url` and return some information about the tile per layer. + public static func tileInfo(at url: URL) -> [[String: Any]]? { guard let data = try? Data(contentsOf: url) else { return nil } return tileInfo(from: data) } diff --git a/Sources/MVTTools/VectorTile.swift b/Sources/MVTTools/VectorTile.swift index a190b3c..72ef25e 100644 --- a/Sources/MVTTools/VectorTile.swift +++ b/Sources/MVTTools/VectorTile.swift @@ -200,7 +200,7 @@ public struct VectorTile: Sendable { self.boundingBox = MapTile(x: x, y: y, z: z).boundingBox(projection: projection) } - if let parsedLayers = MVTDecoder.layers( + guard let parsedLayers = MVTDecoder.layers( from: data, x: x, y: y, @@ -208,20 +208,12 @@ public struct VectorTile: Sendable { projection: projection, layerWhitelist: layerWhitelistSet, logger: logger) - { - self.layers = parsedLayers - self.layerNames = Array(layers.keys) - self.origin = .mvt - } - else if let featureCollection = FeatureCollection(jsonData: data) { - self.layers = [:] - self.layerNames = [] - self.origin = .geoJson - - setGeoJson(geoJson: featureCollection, propertyName: "vt_layer") - } else { return nil } + self.layers = parsedLayers + self.layerNames = Array(layers.keys) + self.origin = .mvt + if let sortOption { createIndex(sortOption: sortOption) } @@ -258,6 +250,59 @@ public struct VectorTile: Sendable { layerWhitelist: [String]? = nil, logger: Logger? = nil) { + guard let data = try? Data(contentsOf: url) else { + (logger ?? VectorTile.logger)?.warning("\(z)/\(x)/\(y): Failed to load vector tile from \(url)") + return nil + } + + self.init( + data: data, + x: x, + y: y, + z: z, + projection: projection, + indexed: sortOption, + layerWhitelist: layerWhitelist, + logger: logger) + } + + /// Create a vector tile by reading it from `url`, which must be in MVT format, at some tile coordinate. + public init?( + contentsOf url: URL, + tile: MapTile, + projection: Projection = .epsg4326, + indexed sortOption: RTreeSortOption? = nil, + layerWhitelist: [String]? = nil, + logger: Logger? = nil) + { + self.init( + contentsOf: url, + x: tile.x, + y: tile.y, + z: tile.z, + projection: projection, + indexed: sortOption, + layerWhitelist: layerWhitelist, + logger: logger) + } + + /// Create a vector tile from `data`, which must be some GeoJSON object. + public init?( + geoJsonData data: Data, + indexed sortOption: RTreeSortOption? = nil, + layerWhitelist: [String]? = nil, + logger: Logger? = nil) + { + guard let featureCollection = FeatureCollection(jsonData: data), + let fcBoundingBox = featureCollection.calculateBoundingBox() + else { return nil } + + // Find the minimal tile for the GeoJSON + let tile = MapTile(boundingBox: fcBoundingBox) + self.x = tile.x + self.y = tile.y + self.z = tile.z + guard x >= 0, y >= 0, z >= 0 else { (logger ?? VectorTile.logger)?.warning("\(z)/\(x)/\(y): Invalid tile coordinate") return nil @@ -269,10 +314,8 @@ public struct VectorTile: Sendable { return nil } - self.x = x - self.y = y - self.z = z - self.projection = projection + self.projection = .epsg4326 + self.boundingBox = tile.boundingBox(projection: projection) self.logger = logger // Note: A plain array might actually be faster for few entries -> check this @@ -283,63 +326,34 @@ public struct VectorTile: Sendable { nil } - guard let data = try? Data(contentsOf: url) else { - (logger ?? VectorTile.logger)?.warning("\(z)/\(x)/\(y): Failed to load vector tile from \(url)") - return nil - } - - switch projection { - case .noSRID: - self.boundingBox = BoundingBox( - southWest: Coordinate3D(x: 0.0, y: 0.0, projection: .noSRID), - northEast: Coordinate3D(x: 4096, y: 4096, projection: .noSRID)) - - case .epsg3857, .epsg4326: - self.boundingBox = MapTile(x: x, y: y, z: z).boundingBox(projection: projection) - } - - if let parsedLayers = MVTDecoder.layers( - from: data, - x: x, - y: y, - z: z, - projection: projection, - layerWhitelist: layerWhitelistSet, - logger: logger) - { - self.layers = parsedLayers - self.layerNames = Array(layers.keys) - self.origin = .mvt - } - else if let featureCollection = FeatureCollection(jsonData: data) { - self.layers = [:] - self.layerNames = [] - self.origin = .geoJson + self.layers = [:] + self.layerNames = [] + self.origin = .geoJson - setGeoJson(geoJson: featureCollection, propertyName: "vt_layer") - } - else { return nil } + setGeoJson( + geoJson: featureCollection, + propertyName: "vt_layer", + layerAllowList: layerWhitelistSet) if let sortOption { createIndex(sortOption: sortOption) } } - /// Create a vector tile by reading it from `url`, which must be in MVT format, at some tile coordinate. + /// Create a vector tile by reading it from `url`, which must be some GeoJSON object. public init?( - contentsOf url: URL, - tile: MapTile, - projection: Projection = .epsg4326, + contentsOfGeoJson url: URL, indexed sortOption: RTreeSortOption? = nil, layerWhitelist: [String]? = nil, logger: Logger? = nil) { + guard let data = try? Data(contentsOf: url) else { + (logger ?? VectorTile.logger)?.warning("Failed to import GeoJSON from \(url)") + return nil + } + self.init( - contentsOf: url, - x: tile.x, - y: tile.y, - z: tile.z, - projection: projection, + geoJsonData: data, indexed: sortOption, layerWhitelist: layerWhitelist, logger: logger) diff --git a/Tests/MVTToolsTests/VectorTileTests.swift b/Tests/MVTToolsTests/VectorTileTests.swift index 8b5ea8e..4e10eb3 100644 --- a/Tests/MVTToolsTests/VectorTileTests.swift +++ b/Tests/MVTToolsTests/VectorTileTests.swift @@ -59,14 +59,7 @@ final class VectorTileTests: XCTestCase { let mvt = TestData.dataFromFile(name: tileName) XCTAssertFalse(mvt.isEmpty) - let info = try XCTUnwrap(VectorTile.tileInfo(from: mvt)) - - XCTAssertFalse(info.isEmpty) - - let layers = try XCTUnwrap(info["layers"] as? [[String: Any]]) - let errors = try XCTUnwrap(info["errors"] as? Bool) - - XCTAssertEqual(errors, false) + let layers = try XCTUnwrap(VectorTile.tileInfo(from: mvt)) XCTAssertEqual(layers.count, 23) }