Add OSM area preflight
This commit is contained in:
@@ -67,6 +67,7 @@ function parseOsm(xml) {
|
||||
const nodePattern = /<node\b([^>]*?)\/>|<node\b([^>]*?)>([\s\S]*?)<\/node>/g;
|
||||
for (const match of xml.matchAll(nodePattern)) {
|
||||
const attrs = xmlAttrs(match[1] || match[2] || "");
|
||||
if (attrs.action === "delete") continue;
|
||||
if (attrs.id) nodeIds.add(attrs.id);
|
||||
nodeStats.total += 1;
|
||||
const tags = parseTags(match[3] || "");
|
||||
@@ -80,6 +81,8 @@ function parseOsm(xml) {
|
||||
buildingsWithHeight: 0,
|
||||
buildingsWithLevels: 0,
|
||||
buildingsWithBadHeight: 0,
|
||||
buildingsWithBadLevels: 0,
|
||||
buildingGeometryIssues: [],
|
||||
missingNodeRefs: 0,
|
||||
grass: 0,
|
||||
scrub: 0,
|
||||
@@ -87,6 +90,7 @@ function parseOsm(xml) {
|
||||
};
|
||||
for (const match of xml.matchAll(/<way\b([^>]*)>([\s\S]*?)<\/way>/g)) {
|
||||
const attrs = xmlAttrs(match[1]);
|
||||
if (attrs.action === "delete") continue;
|
||||
const body = match[2];
|
||||
const tags = parseTags(body);
|
||||
const refs = [];
|
||||
@@ -110,6 +114,14 @@ function parseOsm(xml) {
|
||||
if (isExplicitHeight(tags)) wayStats.buildingsWithHeight += 1;
|
||||
if (tags["building:levels"]) wayStats.buildingsWithLevels += 1;
|
||||
if (tags.height && !parseHeightMeters(tags.height)) wayStats.buildingsWithBadHeight += 1;
|
||||
if (tags["building:levels"] && !parseBuildingLevels(tags["building:levels"])) {
|
||||
wayStats.buildingsWithBadLevels += 1;
|
||||
}
|
||||
if (refs.length < 4) {
|
||||
wayStats.buildingGeometryIssues.push({ id: way.id, issue: "has fewer than 4 node refs" });
|
||||
} else if (!way.closed) {
|
||||
wayStats.buildingGeometryIssues.push({ id: way.id, issue: "is not closed" });
|
||||
}
|
||||
}
|
||||
if (tags.landuse === "grass") wayStats.grass += 1;
|
||||
if (tags.natural === "scrub") wayStats.scrub += 1;
|
||||
@@ -122,11 +134,13 @@ function parseOsm(xml) {
|
||||
buildingsWithHeight: 0,
|
||||
buildingsWithLevels: 0,
|
||||
buildingsWithBadHeight: 0,
|
||||
buildingsWithBadLevels: 0,
|
||||
healthyBuildingMultipolygons: 0,
|
||||
issues: [],
|
||||
};
|
||||
for (const match of xml.matchAll(/<relation\b([^>]*)>([\s\S]*?)<\/relation>/g)) {
|
||||
const attrs = xmlAttrs(match[1]);
|
||||
if (attrs.action === "delete") continue;
|
||||
const body = match[2];
|
||||
const tags = parseTags(body);
|
||||
const members = [];
|
||||
@@ -139,6 +153,9 @@ function parseOsm(xml) {
|
||||
if (isExplicitHeight(tags)) relationStats.buildingsWithHeight += 1;
|
||||
if (tags["building:levels"]) relationStats.buildingsWithLevels += 1;
|
||||
if (tags.height && !parseHeightMeters(tags.height)) relationStats.buildingsWithBadHeight += 1;
|
||||
if (tags["building:levels"] && !parseBuildingLevels(tags["building:levels"])) {
|
||||
relationStats.buildingsWithBadLevels += 1;
|
||||
}
|
||||
|
||||
const health = buildingRelationHealth(attrs.id || "", members, ways);
|
||||
if (health.ok) relationStats.healthyBuildingMultipolygons += 1;
|
||||
@@ -163,7 +180,10 @@ function parseBounds(xml) {
|
||||
maxLon: Number(attrs.maxlon),
|
||||
maxLat: Number(attrs.maxlat),
|
||||
};
|
||||
return Object.values(bounds).every(Number.isFinite) ? bounds : null;
|
||||
return Object.values(bounds).every(Number.isFinite) &&
|
||||
bounds.minLon < bounds.maxLon && bounds.minLat < bounds.maxLat
|
||||
? bounds
|
||||
: null;
|
||||
}
|
||||
|
||||
function isExplicitHeight(tags) {
|
||||
@@ -177,6 +197,48 @@ function parseHeightMeters(value) {
|
||||
return Number.isFinite(height) && height > 0 ? height : null;
|
||||
}
|
||||
|
||||
function parseBuildingLevels(value) {
|
||||
const levels = Number(String(value).trim());
|
||||
return Number.isFinite(levels) && levels > 0;
|
||||
}
|
||||
|
||||
function analyzeOsmPreflight(osm) {
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
if (!osm.bounds) errors.push("OSM has no valid <bounds>.");
|
||||
if (osm.ways.missingNodeRefs) {
|
||||
errors.push(`OSM ways reference ${osm.ways.missingNodeRefs} missing node(s).`);
|
||||
}
|
||||
for (const issue of osm.ways.buildingGeometryIssues) {
|
||||
errors.push(`Building way ${issue.id}: ${issue.issue}.`);
|
||||
}
|
||||
if (osm.ways.buildingsWithBadHeight || osm.relations.buildingsWithBadHeight) {
|
||||
errors.push("Some building height tags could not be parsed as positive meters.");
|
||||
}
|
||||
for (const issue of osm.relations.issues) {
|
||||
errors.push(`Building relation ${issue.id}: ${issue.issues.join("; ")}.`);
|
||||
}
|
||||
const badLevels = osm.ways.buildingsWithBadLevels + osm.relations.buildingsWithBadLevels;
|
||||
if (badLevels) {
|
||||
warnings.push(`${badLevels} building:levels tag(s) could not be parsed as positive numbers.`);
|
||||
}
|
||||
return {
|
||||
errors: uniqueLines(errors),
|
||||
warnings: uniqueLines(warnings),
|
||||
summary: {
|
||||
bounds: osm.bounds,
|
||||
nodes: osm.nodes.total,
|
||||
ways: osm.ways.total,
|
||||
relations: osm.relations.total,
|
||||
buildingWays: osm.ways.buildings,
|
||||
buildingMultipolygons: osm.relations.buildingMultipolygons,
|
||||
missingNodeRefs: osm.ways.missingNodeRefs,
|
||||
buildingWayIssues: osm.ways.buildingGeometryIssues.length,
|
||||
buildingRelationIssues: osm.relations.issues.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildingRelationHealth(id, members, ways) {
|
||||
const issues = [];
|
||||
const outerMembers = members.filter((member) => member.type === "way" && member.role === "outer");
|
||||
@@ -307,6 +369,15 @@ function stageManifestStatus(area, configPath = null) {
|
||||
const hasReimportManifest = fs.existsSync(reimportManifest);
|
||||
const derivedConfig = path.join(area.outputs.pipelineDir, "osm2streets-qgis.config.json");
|
||||
const stages = [
|
||||
{
|
||||
stage: "preflight",
|
||||
expected: fs.existsSync(stageManifestPath(area, "preflight")),
|
||||
inputs: {
|
||||
...(configPath ? { config: configPath } : {}),
|
||||
osm: area.input,
|
||||
},
|
||||
outputs: {},
|
||||
},
|
||||
{
|
||||
stage: "intermediates",
|
||||
expected: (
|
||||
@@ -763,6 +834,7 @@ function formatBytes(bytes) {
|
||||
|
||||
module.exports = {
|
||||
BUDGETS,
|
||||
analyzeOsmPreflight,
|
||||
analyzeArea,
|
||||
artifactStatus,
|
||||
classifyAreaQuality,
|
||||
|
||||
99
scripts/preflight-area.js
Normal file
99
scripts/preflight-area.js
Normal file
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { readAreaConfig } = require("./lib/area-config");
|
||||
const {
|
||||
analyzeOsmPreflight,
|
||||
defaultConfigPath,
|
||||
parseOsm,
|
||||
} = require("./lib/area-diagnostics");
|
||||
const { fileRecord, writeStageManifest } = require("./lib/stage-manifest");
|
||||
|
||||
const repoRoot = path.resolve(__dirname, "..");
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (!arg.startsWith("--")) continue;
|
||||
const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase());
|
||||
const next = argv[i + 1];
|
||||
if (!next || next.startsWith("--")) {
|
||||
out[key] = "true";
|
||||
} else {
|
||||
out[key] = next;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const startedAt = new Date().toISOString();
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const configPath = path.resolve(args.config || defaultConfigPath(repoRoot));
|
||||
const area = readAreaConfig(configPath, { repoRoot });
|
||||
const osm = parseOsm(fs.readFileSync(area.input, "utf8"));
|
||||
const preflight = analyzeOsmPreflight(osm);
|
||||
|
||||
printPreflightReport(area, configPath, preflight);
|
||||
if (preflight.errors.length) {
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const finishedAt = new Date().toISOString();
|
||||
writeStageManifest(area, {
|
||||
stage: "preflight",
|
||||
status: "ok",
|
||||
config: configPath,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
durationMs: Date.parse(finishedAt) - Date.parse(startedAt),
|
||||
inputs: {
|
||||
config: fileRecord(configPath),
|
||||
osm: fileRecord(area.input),
|
||||
},
|
||||
outputs: {},
|
||||
summary: preflight.summary,
|
||||
errors: preflight.errors,
|
||||
warnings: preflight.warnings,
|
||||
});
|
||||
}
|
||||
|
||||
function printPreflightReport(area, configPath, preflight) {
|
||||
console.log("OSM area preflight");
|
||||
console.log(`Area: ${area.id}`);
|
||||
console.log(`Config: ${configPath}`);
|
||||
console.log(`Input: ${area.input}`);
|
||||
console.log("");
|
||||
console.log(
|
||||
`OSM: ${preflight.summary.nodes} nodes, ${preflight.summary.ways} ways, ` +
|
||||
`${preflight.summary.relations} relations`,
|
||||
);
|
||||
console.log(
|
||||
`Buildings: ${preflight.summary.buildingWays} way(s), ` +
|
||||
`${preflight.summary.buildingMultipolygons} multipolygon relation(s)`,
|
||||
);
|
||||
console.log(`Bounds: ${preflight.summary.bounds ? "valid" : "invalid or missing"}`);
|
||||
console.log("");
|
||||
const status = preflight.errors.length ? "FAIL" : "PASS";
|
||||
console.log(`${status}: ${preflight.errors.length} error(s), ${preflight.warnings.length} warning(s)`);
|
||||
console.log("");
|
||||
printLines("Errors", preflight.errors);
|
||||
console.log("");
|
||||
printLines("Warnings", preflight.warnings);
|
||||
}
|
||||
|
||||
function printLines(label, lines) {
|
||||
console.log(`${label} (${lines.length})`);
|
||||
if (!lines.length) {
|
||||
console.log(" none");
|
||||
return;
|
||||
}
|
||||
for (const line of lines) console.log(` - ${line}`);
|
||||
}
|
||||
|
||||
main();
|
||||
34
scripts/test-area-preflight.js
Normal file
34
scripts/test-area-preflight.js
Normal file
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const assert = require("assert");
|
||||
const { analyzeOsmPreflight, parseOsm } = require("./lib/area-diagnostics");
|
||||
|
||||
function osm({ bounds = '<bounds minlon="0" minlat="0" maxlon="1" maxlat="1"/>', nodes, ways, relations = "" }) {
|
||||
return `<osm>${bounds}${nodes}${ways}${relations}</osm>`;
|
||||
}
|
||||
|
||||
function nodes(ids) {
|
||||
return ids.map((id) => `<node id="${id}" lat="0" lon="0"/>`).join("");
|
||||
}
|
||||
|
||||
function way(id, refs, tags = '<tag k="building" v="yes"/>') {
|
||||
return `<way id="${id}">${refs.map((ref) => `<nd ref="${ref}"/>`).join("")}${tags}</way>`;
|
||||
}
|
||||
|
||||
function preflight(input) {
|
||||
return analyzeOsmPreflight(parseOsm(input));
|
||||
}
|
||||
|
||||
assert.equal(preflight(osm({ nodes: nodes([1, 2, 3]), ways: way(10, [1, 2, 3, 1]) })).errors.length, 0);
|
||||
assert.match(preflight(osm({ bounds: "", nodes: nodes([1, 2, 3]), ways: way(10, [1, 2, 3, 1]) })).errors.join("\n"), /bounds/);
|
||||
assert.match(preflight(osm({ bounds: '<bounds minlon="1" minlat="0" maxlon="0" maxlat="1"/>', nodes: nodes([1, 2, 3]), ways: way(10, [1, 2, 3, 1]) })).errors.join("\n"), /bounds/);
|
||||
assert.match(preflight(osm({ nodes: nodes([1, 2, 3]), ways: way(10, [1, 2, 9, 1]) })).errors.join("\n"), /missing node/);
|
||||
assert.match(preflight(osm({ nodes: nodes([1, 2, 3]), ways: way(10, [1, 2, 3, 2]) })).errors.join("\n"), /not closed/);
|
||||
assert.match(preflight(osm({ nodes: nodes([1, 2, 3]), ways: way(10, [1, 2, 3, 1], '<tag k="building" v="yes"/><tag k="height" v="many"/>') })).errors.join("\n"), /height/);
|
||||
assert.equal(preflight(osm({ nodes: nodes([1, 2, 3]), ways: '<way id="10" action="delete"><tag k="building" v="yes"/></way>' })).errors.length, 0);
|
||||
|
||||
const brokenRelation = '<relation id="20"><member type="way" ref="99" role="outer"/><tag k="type" v="multipolygon"/><tag k="building" v="yes"/></relation>';
|
||||
assert.match(preflight(osm({ nodes: nodes([1, 2, 3]), ways: "", relations: brokenRelation })).errors.join("\n"), /Building relation 20/);
|
||||
|
||||
console.log("OSM preflight tests passed.");
|
||||
Reference in New Issue
Block a user