Add area quality gate

This commit is contained in:
2026-08-04 10:42:10 +08:00
parent b791c4350e
commit 7fcc4ee8cc
9 changed files with 908 additions and 550 deletions

View File

@@ -1,19 +1,14 @@
#!/usr/bin/env node
"use strict";
const fs = require("fs");
const path = require("path");
const { readAreaConfig } = require("./lib/area-config");
const { digest: glbDigest } = require("./glb-digest");
const { fileRecord, readStageManifest, stageManifestPath } = require("./lib/stage-manifest");
const {
analyzeArea,
defaultConfigPath,
printDiagnosticsReport,
} = require("./lib/area-diagnostics");
const repoRoot = path.resolve(__dirname, "..");
const DEFAULT_CONFIG = path.join(repoRoot, "config", "areas", "nantaizi-lake-innovation-valley.json");
const BUDGETS = {
glbBytes: 25 * 1024 * 1024,
glbNodes: 1000,
glbImages: 24,
};
function parseArgs(argv) {
const out = {};
@@ -32,545 +27,10 @@ function parseArgs(argv) {
return out;
}
function xmlAttrs(text) {
const attrs = {};
for (const match of text.matchAll(/([:\w-]+)\s*=\s*("([^"]*)"|'([^']*)')/g)) {
attrs[match[1]] = match[3] !== undefined ? match[3] : match[4];
}
return attrs;
}
function parseTags(body) {
const tags = {};
for (const match of body.matchAll(/<tag\b([^>]*)\/?>/g)) {
const attrs = xmlAttrs(match[1]);
if (attrs.k) tags[attrs.k] = attrs.v || "";
}
return tags;
}
function parseOsm(xml) {
const bounds = parseBounds(xml);
const nodeIds = new Set();
const nodeStats = { total: 0, naturalTree: 0 };
const nodePattern = /<node\b([^>]*?)\/>|<node\b([^>]*?)>([\s\S]*?)<\/node>/g;
for (const match of xml.matchAll(nodePattern)) {
const attrs = xmlAttrs(match[1] || match[2] || "");
if (attrs.id) nodeIds.add(attrs.id);
nodeStats.total += 1;
const tags = parseTags(match[3] || "");
if (tags.natural === "tree") nodeStats.naturalTree += 1;
}
const ways = new Map();
const wayStats = {
total: 0,
buildings: 0,
buildingsWithHeight: 0,
buildingsWithLevels: 0,
buildingsWithBadHeight: 0,
missingNodeRefs: 0,
grass: 0,
scrub: 0,
treeRows: 0,
};
for (const match of xml.matchAll(/<way\b([^>]*)>([\s\S]*?)<\/way>/g)) {
const attrs = xmlAttrs(match[1]);
const body = match[2];
const tags = parseTags(body);
const refs = [];
for (const ndMatch of body.matchAll(/<nd\b([^>]*)\/?>/g)) {
const nd = xmlAttrs(ndMatch[1]);
if (nd.ref) refs.push(nd.ref);
}
const missingNodeRefs = refs.filter((ref) => !nodeIds.has(ref)).length;
const way = {
id: attrs.id || "",
refs,
tags,
closed: refs.length > 1 && refs[0] === refs[refs.length - 1],
missingNodeRefs,
};
if (way.id) ways.set(way.id, way);
wayStats.total += 1;
wayStats.missingNodeRefs += missingNodeRefs;
if (tags.building) {
wayStats.buildings += 1;
if (isExplicitHeight(tags)) wayStats.buildingsWithHeight += 1;
if (tags["building:levels"]) wayStats.buildingsWithLevels += 1;
if (tags.height && !parseHeightMeters(tags.height)) wayStats.buildingsWithBadHeight += 1;
}
if (tags.landuse === "grass") wayStats.grass += 1;
if (tags.natural === "scrub") wayStats.scrub += 1;
if (tags.natural === "tree_row") wayStats.treeRows += 1;
}
const relationStats = {
total: 0,
buildingMultipolygons: 0,
buildingsWithHeight: 0,
buildingsWithLevels: 0,
buildingsWithBadHeight: 0,
healthyBuildingMultipolygons: 0,
issues: [],
};
for (const match of xml.matchAll(/<relation\b([^>]*)>([\s\S]*?)<\/relation>/g)) {
const attrs = xmlAttrs(match[1]);
const body = match[2];
const tags = parseTags(body);
const members = [];
for (const memberMatch of body.matchAll(/<member\b([^>]*)\/?>/g)) {
members.push(xmlAttrs(memberMatch[1]));
}
relationStats.total += 1;
if (tags.type !== "multipolygon" || !tags.building) continue;
relationStats.buildingMultipolygons += 1;
if (isExplicitHeight(tags)) relationStats.buildingsWithHeight += 1;
if (tags["building:levels"]) relationStats.buildingsWithLevels += 1;
if (tags.height && !parseHeightMeters(tags.height)) relationStats.buildingsWithBadHeight += 1;
const health = buildingRelationHealth(attrs.id || "", members, ways);
if (health.ok) {
relationStats.healthyBuildingMultipolygons += 1;
} else {
relationStats.issues.push(health);
}
}
return {
bounds,
nodes: nodeStats,
ways: wayStats,
relations: relationStats,
};
}
function parseBounds(xml) {
const match = xml.match(/<bounds\b([^>]*)\/?>/);
if (!match) return null;
const attrs = xmlAttrs(match[1]);
const bounds = {
minLon: Number(attrs.minlon),
minLat: Number(attrs.minlat),
maxLon: Number(attrs.maxlon),
maxLat: Number(attrs.maxlat),
};
return Object.values(bounds).every(Number.isFinite) ? bounds : null;
}
function isExplicitHeight(tags) {
return Boolean(tags.height && parseHeightMeters(tags.height));
}
function parseHeightMeters(value) {
const match = String(value).trim().match(/^(-?\d+(?:\.\d+)?)/);
if (!match) return null;
const height = Number(match[1]);
return Number.isFinite(height) && height > 0 ? height : null;
}
function buildingRelationHealth(id, members, ways) {
const issues = [];
const outerMembers = members.filter((member) => member.type === "way" && member.role === "outer");
const innerMembers = members.filter((member) => member.type === "way" && member.role === "inner");
const nonWayMembers = members.filter((member) => member.type && member.type !== "way");
if (!outerMembers.length) issues.push("no outer way members");
if (nonWayMembers.length) issues.push(`${nonWayMembers.length} non-way member(s)`);
const outerHealth = ringGroupHealth(outerMembers, ways);
const innerHealth = ringGroupHealth(innerMembers, ways);
issues.push(...outerHealth.issues.map((issue) => `outer ${issue}`));
issues.push(...innerHealth.issues.map((issue) => `inner ${issue}`));
return {
id,
ok: issues.length === 0,
outerMembers: outerMembers.length,
innerMembers: innerMembers.length,
unresolvedMembers: outerHealth.unresolved + innerHealth.unresolved,
openRings: outerHealth.open + innerHealth.open,
issues,
};
}
function ringGroupHealth(members, ways) {
if (!members.length) return { issues: [], unresolved: 0, open: 0 };
const issues = [];
let unresolved = 0;
let open = 0;
const fragments = [];
for (const member of members) {
const way = ways.get(member.ref);
if (!way) {
unresolved += 1;
continue;
}
if (way.refs.length < 4) {
issues.push(`member ${member.ref} has fewer than 4 node refs`);
continue;
}
fragments.push(way.refs);
}
if (unresolved) issues.push(`${unresolved} unresolved member way(s)`);
if (!fragments.length) return { issues, unresolved, open };
if (fragments.length === 1) {
if (!isClosedRefs(fragments[0])) {
open += 1;
issues.push(`member ${members[0].ref} is not closed`);
}
return { issues, unresolved, open };
}
const endpointDegrees = new Map();
for (const refs of fragments) {
if (isClosedRefs(refs)) continue;
open += 1;
addEndpoint(endpointDegrees, refs[0]);
addEndpoint(endpointDegrees, refs[refs.length - 1]);
}
const badEndpoints = [...endpointDegrees.values()].filter((count) => count !== 2).length;
if (badEndpoints) {
issues.push(`${open} open member fragment(s) do not stitch into closed rings`);
}
return { issues, unresolved, open: badEndpoints ? open : 0 };
}
function isClosedRefs(refs) {
return refs.length > 1 && refs[0] === refs[refs.length - 1];
}
function addEndpoint(map, ref) {
map.set(ref, (map.get(ref) || 0) + 1);
}
function artifactStatus(area) {
const entries = [
["GeoJSON dir", area.outputs.geojsonDir, true, "dir"],
["GeoPackage", area.outputs.gpkg, true, "file"],
["QGIS project", area.outputs.qgisProject, true, "file"],
["QGIS preview", area.outputs.qgisPreview, true, "file"],
["Blend scene", area.outputs.blend, true, "file"],
["Render PNG", area.outputs.render, true, "file"],
["Cesium GLB", area.outputs.glb, true, "file"],
["Cesium metadata", area.outputs.metadata, true, "file"],
["Cesium preview", area.outputs.cesiumPreview, true, "file"],
["Compressed GLB", area.outputs.compressedGlb, false, "file"],
["Compressed metadata", area.outputs.compressedMetadata, false, "file"],
["Compressed preview", area.outputs.compressedCesiumPreview, false, "file"],
];
return entries.map(([label, file, expected, type]) => {
const exists = fs.existsSync(file);
const stat = exists ? fs.statSync(file) : null;
const validType = !exists || (type === "dir" ? stat.isDirectory() : stat.isFile());
const geojsonFiles = exists && type === "dir"
? fs.readdirSync(file).filter((entry) => entry.endsWith(".geojson")).length
: null;
return {
label,
path: file,
expected,
type,
exists,
validType,
bytes: stat && stat.isFile() ? stat.size : null,
geojsonFiles,
};
});
}
function metadataSummary(file, warnings) {
if (!fs.existsSync(file)) return null;
try {
const metadata = JSON.parse(fs.readFileSync(file, "utf8"));
return {
asset: metadata.asset || null,
assets: Array.isArray(metadata.assets) ? metadata.assets.length : 0,
origin: metadata.origin || metadata.center || null,
};
} catch (error) {
warnings.push(`Cesium metadata is not valid JSON: ${error.message}`);
return null;
}
}
function stageManifestStatus(area) {
const stages = [
{
stage: "cesium",
expected: fs.existsSync(area.outputs.glb),
inputs: {
blend: area.outputs.blend,
},
outputs: {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
cesiumPreview: area.outputs.cesiumPreview,
},
},
{
stage: "compress",
expected: fs.existsSync(area.outputs.compressedGlb),
inputs: {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
cesiumPreview: area.outputs.cesiumPreview,
},
outputs: {
compressedGlb: area.outputs.compressedGlb,
compressedMetadata: area.outputs.compressedMetadata,
compressedCesiumPreview: area.outputs.compressedCesiumPreview,
},
},
];
return stages.map((entry) => {
const file = stageManifestPath(area, entry.stage);
try {
const manifest = readStageManifest(area, entry.stage);
if (!manifest) {
return {
stage: entry.stage,
path: file,
expected: entry.expected,
exists: false,
valid: false,
fresh: false,
manifestWarnings: [],
issues: entry.expected ? ["manifest missing"] : [],
};
}
const issues = [
...manifestFileIssues(manifest.inputs || {}, entry.inputs, "input"),
...manifestFileIssues(manifest.outputs || {}, entry.outputs, "output"),
];
return {
stage: entry.stage,
path: file,
expected: entry.expected,
exists: true,
valid: true,
fresh: issues.length === 0,
finishedAt: manifest.finishedAt || null,
durationMs: manifest.durationMs ?? null,
summary: manifest.summary || null,
manifestWarnings: Array.isArray(manifest.warnings) ? manifest.warnings : [],
issues,
};
} catch (error) {
return {
stage: entry.stage,
path: file,
expected: entry.expected,
exists: fs.existsSync(file),
valid: false,
fresh: false,
manifestWarnings: [],
issues: [`manifest unreadable: ${error.message}`],
};
}
});
}
function manifestFileIssues(records, expectedFiles, label) {
const issues = [];
for (const [key, file] of Object.entries(expectedFiles)) {
const recorded = records[key];
if (!recorded) {
issues.push(`${label} ${key} not recorded`);
continue;
}
if (!fs.existsSync(file)) {
issues.push(`${label} ${key} file missing`);
continue;
}
const current = fileRecord(file);
if (recorded.bytes !== current.bytes) {
issues.push(`${label} ${key} bytes changed`);
} else if (recorded.sha256 && recorded.sha256 !== current.sha256) {
issues.push(`${label} ${key} sha256 changed`);
}
}
return issues;
}
function collectWarnings(area, osm, artifacts, manifests, glb, metadata) {
const warnings = [];
if (!osm.bounds) warnings.push("OSM has no valid <bounds>; scene extent may be wrong.");
if (osm.ways.missingNodeRefs) {
warnings.push(`OSM ways reference ${osm.ways.missingNodeRefs} missing node(s).`);
}
if (osm.ways.buildingsWithBadHeight || osm.relations.buildingsWithBadHeight) {
warnings.push("Some building height tags could not be parsed as positive meters.");
}
for (const issue of osm.relations.issues) {
warnings.push(`Building relation ${issue.id}: ${issue.issues.join("; ")}.`);
}
for (const artifact of artifacts) {
if (artifact.expected && !artifact.exists) {
warnings.push(`Expected artifact missing: ${artifact.label} (${artifact.path}).`);
} else if (artifact.exists && !artifact.validType) {
warnings.push(`Artifact has wrong type: ${artifact.label} (${artifact.path}).`);
}
}
for (const manifest of manifests) {
if (manifest.expected && !manifest.exists) {
warnings.push(`Expected stage manifest missing: ${manifest.stage} (${manifest.path}).`);
} else if (manifest.exists && !manifest.valid) {
warnings.push(`Stage manifest invalid: ${manifest.stage} (${manifest.issues.join("; ")}).`);
} else if (manifest.exists && !manifest.fresh) {
warnings.push(`Stage manifest stale: ${manifest.stage} (${manifest.issues.join("; ")}).`);
}
for (const warning of manifest.manifestWarnings) {
warnings.push(`Stage manifest warning (${manifest.stage}): ${warning}.`);
}
}
if (glb) {
if (glb.fileBytes > BUDGETS.glbBytes) {
warnings.push(`GLB size ${mb(glb.fileBytes)} MB exceeds budget ${mb(BUDGETS.glbBytes)} MB.`);
}
if (glb.counts.nodes > BUDGETS.glbNodes) {
warnings.push(`GLB nodes ${glb.counts.nodes} exceed budget ${BUDGETS.glbNodes}.`);
}
if (glb.counts.images > BUDGETS.glbImages) {
warnings.push(`GLB images ${glb.counts.images} exceed budget ${BUDGETS.glbImages}.`);
}
}
if (metadata && metadata.assets < 1) {
warnings.push("Cesium metadata has no assets entries.");
}
if (!area.blender.treeStyle) {
warnings.push("No Blender tree style configured.");
}
return warnings;
}
function printReport(area, configPath, osm, artifacts, manifests, glb, metadata, warnings) {
console.log("Area diagnostics");
console.log(`Area: ${area.id}`);
console.log(`Config: ${configPath}`);
console.log(`Input: ${area.input}`);
console.log(`Output: ${area.outputs.areaDir}`);
console.log("");
console.log("OSM");
console.log(` Bounds: ${osm.bounds ? formatBounds(osm.bounds) : "missing"}`);
console.log(` Nodes: ${formatNumber(osm.nodes.total)} (${formatNumber(osm.nodes.naturalTree)} natural=tree)`);
console.log(` Ways: ${formatNumber(osm.ways.total)}`);
console.log(` Relations: ${formatNumber(osm.relations.total)}`);
console.log(
` Buildings: ${formatNumber(osm.ways.buildings)} way(s), ` +
`${formatNumber(osm.relations.buildingMultipolygons)} multipolygon relation(s)`,
);
console.log(
` Building heights: ${formatNumber(osm.ways.buildingsWithHeight + osm.relations.buildingsWithHeight)} ` +
`height tag(s), ${formatNumber(osm.ways.buildingsWithLevels + osm.relations.buildingsWithLevels)} ` +
"building:levels tag(s)",
);
console.log(
` Building relations healthy: ${formatNumber(osm.relations.healthyBuildingMultipolygons)} / ` +
`${formatNumber(osm.relations.buildingMultipolygons)}`,
);
console.log(
` Vegetation ways: ${formatNumber(osm.ways.grass)} grass, ` +
`${formatNumber(osm.ways.scrub)} scrub, ${formatNumber(osm.ways.treeRows)} tree_row`,
);
if (osm.relations.issues.length) {
console.log(" Relation issues:");
for (const issue of osm.relations.issues) {
console.log(` - ${issue.id}: ${issue.issues.join("; ")}`);
}
}
console.log("");
console.log("Artifacts");
for (const artifact of artifacts) {
const state = artifact.exists && artifact.validType ? "ok" : (artifact.expected ? "missing" : "absent");
const suffix = artifact.geojsonFiles !== null
? `, ${artifact.geojsonFiles} GeoJSON file(s)`
: artifact.bytes !== null ? `, ${formatBytes(artifact.bytes)}` : "";
console.log(` ${state.padEnd(7)} ${artifact.label}: ${artifact.path}${suffix}`);
}
if (metadata) {
console.log(` metadata asset: ${metadata.asset || "missing"}, assets: ${metadata.assets}`);
}
console.log("");
console.log("Stage manifests");
for (const manifest of manifests) {
let state = "absent";
if (manifest.exists && !manifest.valid) state = "invalid";
else if (manifest.exists && !manifest.fresh) state = "stale";
else if (manifest.exists) state = "ok";
else if (manifest.expected) state = "missing";
const timing = manifest.finishedAt
? `, finished ${manifest.finishedAt}, ${manifest.durationMs ?? "?"} ms`
: "";
const issues = manifest.issues.length ? `, ${manifest.issues.join("; ")}` : "";
console.log(` ${state.padEnd(7)} ${manifest.stage}: ${manifest.path}${timing}${issues}`);
for (const warning of manifest.manifestWarnings) {
console.log(` warning: ${warning}`);
}
const glbSummary = manifest.summary?.glb || manifest.summary?.compressedGlb;
if (glbSummary?.counts) {
console.log(
` GLB: ${formatBytes(glbSummary.fileBytes)}, ${glbSummary.counts.nodes} nodes, ` +
`${glbSummary.counts.meshes} meshes, ${glbSummary.counts.images} images`,
);
}
}
if (glb) {
console.log("");
console.log("GLB digest");
console.log(` Size: ${mb(glb.fileBytes)} MB`);
console.log(
` Counts: ${glb.counts.nodes} nodes, ${glb.counts.meshes} meshes, ` +
`${glb.counts.materials} materials, ${glb.counts.images} images, ${glb.counts.accessors} accessors`,
);
console.log(` Extensions: ${glb.extensionsUsed.length ? glb.extensionsUsed.join(", ") : "none"}`);
}
console.log("");
console.log("Warnings");
if (!warnings.length) {
console.log(" none");
} else {
for (const warning of warnings) console.log(` - ${warning}`);
}
}
function formatBounds(bounds) {
return `${bounds.minLon},${bounds.minLat} -> ${bounds.maxLon},${bounds.maxLat}`;
}
function formatNumber(value) {
return new Intl.NumberFormat("en-US").format(value);
}
function mb(bytes) {
return Number((bytes / 1024 / 1024).toFixed(2));
}
function formatBytes(bytes) {
if (bytes >= 1024 * 1024) return `${mb(bytes)} MB`;
if (bytes >= 1024) return `${Number((bytes / 1024).toFixed(1))} KB`;
return `${bytes} B`;
}
function main() {
const args = parseArgs(process.argv.slice(2));
const configPath = path.resolve(args.config || DEFAULT_CONFIG);
const area = readAreaConfig(configPath, { repoRoot });
const osm = parseOsm(fs.readFileSync(area.input, "utf8"));
const artifacts = artifactStatus(area);
const manifests = stageManifestStatus(area);
const metadataWarnings = [];
const metadata = metadataSummary(area.outputs.metadata, metadataWarnings);
const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null;
const warnings = [
...metadataWarnings,
...collectWarnings(area, osm, artifacts, manifests, glb, metadata),
];
printReport(area, configPath, osm, artifacts, manifests, glb, metadata, warnings);
const configPath = path.resolve(args.config || defaultConfigPath(repoRoot));
printDiagnosticsReport(analyzeArea(configPath, { repoRoot }));
}
main();