Files
osmWorkflow/scripts/build-area.js

588 lines
20 KiB
JavaScript
Executable File

#!/usr/bin/env node
const fs = require("fs");
const path = require("path");
const { spawnSync } = require("child_process");
const { readAreaConfig } = require("./lib/area-config");
const { blenderExecutable: resolveBlenderExecutable } = require("./lib/tool-paths");
const {
SCENE_LAYERS,
SCENE_FILE,
SCENE_STYLE_FILE,
layerFile,
} = require("./lib/scene-layers");
const { digest: glbDigest } = require("./glb-digest");
const { buildVehicleRoute: buildPreviewVehicleRoute } = require("./lib/vehicle-route");
const { writePreviewVehicleLibrary } = require("./lib/vehicle-library");
const { readTrafficSignals } = require("./lib/traffic-signals");
const {
cesiumPreviewHtml,
previewSummary,
writeCesiumPreviewSupportFiles,
} = require("./lib/area-preview");
const {
fileRecord,
evaluateGlbBudget,
glbBudgetWarnings,
glbSummary,
optionalFileRecord,
stageManifestPath,
writeStageManifest,
} = require("./lib/stage-manifest");
const repoRoot = path.resolve(__dirname, "..");
const args = parseArgs(process.argv.slice(2));
const configPath = path.resolve(
args.config || path.join(repoRoot, "config", "areas", "nantaizi-lake-innovation-valley.json"),
);
const area = readAreaConfig(configPath, { repoRoot });
const requestedStages = args.stages
? splitList(args.stages)
: null;
const stages = resolveStages(area.stages, requestedStages);
// intermediates deletes and rebuilds the GeoPackage from OSM, which is exactly
// the manual work reimport exists to recover. Refuse the combination instead of
// silently letting one undo the other.
if (stages.intermediates && stages.reimport) {
throw new Error(
"Stages 'intermediates' and 'reimport' are mutually exclusive: " +
"intermediates rebuilds the GeoPackage from OSM and would discard the QGIS edits reimport reads back.",
);
}
console.log(`Area: ${area.id}`);
console.log(`Config: ${configPath}`);
console.log(`Output: ${area.outputs.areaDir}`);
if (stages.intermediates) {
buildIntermediates(area);
}
if (stages.reimport) {
reimportGpkg(area);
}
if (stages.blender) {
buildBlenderScene(area);
}
if (stages.cesium) {
exportCesium(area);
}
if (stages.preview) {
writeCesiumPreview(area);
}
if (stages.compress) {
compressCesiumGlb(area);
}
console.log("Done.");
function parseArgs(argv) {
const out = {};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (!arg.startsWith("--")) continue;
const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase());
const next = argv[i + 1];
if (!next || next.startsWith("--")) {
out[key] = "true";
} else {
out[key] = next;
i += 1;
}
}
return out;
}
function splitList(value) {
return String(value)
.split(",")
.map((item) => item.trim())
.filter(Boolean);
}
function resolveStages(defaults, requested) {
if (!requested) return defaults;
// 'reimport' is deliberately absent from 'all': it is a recovery step for
// hand-edited GeoPackages, never part of a full build.
// 'compress' is also absent from 'all': it creates an alternate Cesium GLB,
// not the baseline asset.
const aliases = {
all: ["intermediates", "blender", "cesium"],
qgis: ["intermediates"],
osm2streets: ["intermediates"],
geojson: ["intermediates"],
intermediate: ["intermediates"],
intermediates: ["intermediates"],
reimport: ["reimport"],
gpkg: ["reimport"],
blender: ["blender"],
scene: ["blender"],
cesium: ["cesium"],
glb: ["cesium"],
preview: ["preview"],
html: ["preview"],
cesiumPreview: ["preview"],
compress: ["compress"],
compression: ["compress"],
compressedCesium: ["compress"],
};
const out = {
intermediates: false,
reimport: false,
blender: false,
cesium: false,
preview: false,
compress: false,
};
for (const stage of requested) {
const mapped = aliases[stage];
if (!mapped) {
throw new Error(`Unknown stage '${stage}'. Use intermediates, reimport, blender, cesium, preview, compress, or all.`);
}
for (const key of mapped) out[key] = true;
}
return out;
}
function writeDerivedConfig(area) {
fs.mkdirSync(area.outputs.pipelineDir, { recursive: true });
const derivedConfig = {
qgisApp: area.qgisApp,
input: area.input,
outDir: area.outputs.geojsonDir,
gpkg: area.outputs.gpkg,
project: area.outputs.qgisProject,
preview: area.outputs.qgisPreview,
trafficSignalAssemblies: area.outputs.trafficSignalAssemblies,
arrowScale: area.qgis.arrowScale,
arrowMergeTriangles: area.qgis.arrowMergeTriangles,
arrowOutlineSimplifyMeters: area.qgis.arrowOutlineSimplifyMeters,
intersectionCornerSourceMaxDimensionMeters: area.qgis.intersectionCornerSourceMaxDimensionMeters,
clipPad: area.qgis.clipPad,
canvasPad: area.qgis.canvasPad,
previewPad: area.qgis.previewPad,
canvasExtent: area.qgis.canvasExtent,
previewExtent: area.qgis.previewExtent,
layerPrefix: area.qgis.layerPrefix,
turnLaneArrows: area.turnLaneArrows,
osm2streets: area.osm2streets,
};
const derivedConfigPath = path.join(area.outputs.pipelineDir, "osm2streets-qgis.config.json");
fs.writeFileSync(derivedConfigPath, `${JSON.stringify(derivedConfig, null, 2)}\n`);
return derivedConfigPath;
}
function buildIntermediates(area) {
const started = Date.now();
const startedAt = new Date(started).toISOString();
const derivedConfigPath = writeDerivedConfig(area);
console.log("Stage: intermediates (osm2streets GeoJSON + QGIS)");
runCommand(process.execPath, [
path.join(repoRoot, "scripts", "build-osm2streets-qgis.js"),
"--config",
derivedConfigPath,
], "intermediates");
writeTrafficSignals(area);
fs.rmSync(stageManifestPath(area, "reimport"), { force: true });
const finished = Date.now();
writeStageManifest(area, {
stage: "intermediates",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
},
outputs: {
derivedConfig: fileRecord(derivedConfigPath),
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
trafficSignalAssemblies: fileRecord(area.outputs.trafficSignalAssemblies),
trafficSignals: fileRecord(area.outputs.trafficSignals),
gpkg: fileRecord(area.outputs.gpkg),
qgisProject: fileRecord(area.outputs.qgisProject),
qgisPreview: optionalFileRecord(area.outputs.qgisPreview),
},
summary: {
geojson: geojsonFeatureCounts(area),
trafficSignalAssemblies: featureCount(area.outputs.trafficSignalAssemblies),
},
warnings: [],
});
}
function reimportGpkg(area) {
const started = Date.now();
const startedAt = new Date(started).toISOString();
const derivedConfigPath = writeDerivedConfig(area);
console.log("Stage: reimport (GeoPackage -> GeoJSON)");
runCommand(process.execPath, [
path.join(repoRoot, "scripts", "reimport-gpkg.js"),
"--config",
derivedConfigPath,
], "reimport");
writeTrafficSignals(area);
fs.rmSync(stageManifestPath(area, "intermediates"), { force: true });
const finished = Date.now();
writeStageManifest(area, {
stage: "reimport",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
derivedConfig: fileRecord(derivedConfigPath),
gpkg: fileRecord(area.outputs.gpkg),
},
outputs: {
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
trafficSignalAssemblies: fileRecord(area.outputs.trafficSignalAssemblies),
trafficSignals: fileRecord(area.outputs.trafficSignals),
},
summary: {
geojson: geojsonFeatureCounts(area),
trafficSignalAssemblies: featureCount(area.outputs.trafficSignalAssemblies),
},
warnings: [],
});
}
function buildBlenderScene(area) {
ensureFile(blenderExecutable(area), "Blender executable");
ensureFile(path.join(repoRoot, "blender", "generate_scene.py"), "Blender scene generator");
ensureFile(area.outputs.trafficSignalAssemblies, "Editable traffic signal assemblies");
// Blender consumes the editable assembly layer; OSM only initializes it in
// intermediates, so QGIS edits remain authoritative across later stages.
writeTrafficSignals(area);
ensureFile(area.outputs.trafficSignals, "Traffic signal anchors");
fs.mkdirSync(path.dirname(area.outputs.blend), { recursive: true });
fs.mkdirSync(path.dirname(area.outputs.render), { recursive: true });
const blenderArgs = [
"--background",
"--factory-startup",
"--python",
path.join(repoRoot, "blender", "generate_scene.py"),
"--",
"--osm",
area.input,
"--geojson",
area.outputs.geojsonDir,
"--output",
area.outputs.blend,
"--render",
area.outputs.render,
"--tree-style",
area.blender.treeStyle,
];
if (area.blender.officeOverrides) {
blenderArgs.push("--office-overrides", area.blender.officeOverrides);
}
console.log("Stage: blender");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(blenderExecutable(area), blenderArgs, "blender");
const finished = Date.now();
writeStageManifest(area, {
stage: "blender",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
trafficSignalAssemblies: fileRecord(area.outputs.trafficSignalAssemblies),
trafficSignals: fileRecord(area.outputs.trafficSignals),
},
outputs: {
blend: fileRecord(area.outputs.blend),
render: fileRecord(area.outputs.render),
},
summary: {
geojson: geojsonFeatureCounts(area),
blendBytes: fileRecord(area.outputs.blend).bytes,
renderBytes: fileRecord(area.outputs.render).bytes,
},
warnings: [],
});
}
function exportCesium(area) {
ensureFile(blenderExecutable(area), "Blender executable");
ensureFile(area.outputs.blend, "Blend scene");
ensureFile(path.join(repoRoot, "blender", "export_cesium.py"), "Cesium exporter");
fs.mkdirSync(path.dirname(area.outputs.glb), { recursive: true });
fs.mkdirSync(path.dirname(area.outputs.metadata), { recursive: true });
console.log("Stage: cesium");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(blenderExecutable(area), [
"--background",
"--factory-startup",
"--python",
path.join(repoRoot, "blender", "export_cesium.py"),
"--",
"--blend",
area.outputs.blend,
"--glb",
area.outputs.glb,
"--dynamic-glb",
area.outputs.trafficSignalsDynamicGlb,
"--countdown-0-glb",
area.outputs.trafficSignalsCountdown0Glb,
"--countdown-1-glb",
area.outputs.trafficSignalsCountdown1Glb,
"--metadata",
area.outputs.metadata,
], "cesium");
ensureFile(area.outputs.trafficSignalsDynamicGlb, "Dynamic traffic signal GLB");
ensureFile(area.outputs.trafficSignalsCountdown0Glb, "Traffic countdown group 0 GLB");
ensureFile(area.outputs.trafficSignalsCountdown1Glb, "Traffic countdown group 1 GLB");
const semanticAssets = semanticAssetRecords(area);
writeCesiumPreview(area);
const finished = Date.now();
const digest = glbDigest(area.outputs.glb);
writeStageManifest(area, {
stage: "cesium",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
blend: fileRecord(area.outputs.blend),
},
outputs: {
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
trafficSignalsDynamicGlb: fileRecord(area.outputs.trafficSignalsDynamicGlb),
semanticAssets,
},
summary: {
glb: glbSummary(digest),
budget: evaluateGlbBudget(digest, area.budget),
},
warnings: glbBudgetWarnings("Cesium", digest, area.budget),
});
}
function semanticAssetRecords(area) {
const metadata = JSON.parse(fs.readFileSync(area.outputs.metadata, "utf8"));
const assets = Array.isArray(metadata.assets) ? metadata.assets : [];
const records = {};
for (const asset of assets) {
if (asset.category !== "semantic") continue;
const file = path.join(area.outputs.areaDir, asset.url);
ensureFile(file, `Cesium semantic asset '${asset.id}'`);
records[asset.id] = fileRecord(file);
}
return records;
}
function compressCesiumGlb(area) {
ensureFile(area.outputs.glb, "Cesium GLB");
ensureFile(area.outputs.metadata, "Cesium metadata");
ensureFile(area.outputs.cesiumPreview, "Cesium preview");
ensureFile(path.join(repoRoot, "scripts", "compress-glb.js"), "GLB compressor");
fs.mkdirSync(path.dirname(area.outputs.compressedGlb), { recursive: true });
fs.mkdirSync(path.dirname(area.outputs.compressedMetadata), { recursive: true });
fs.mkdirSync(path.dirname(area.outputs.compressedCesiumPreview), { recursive: true });
const compressArgs = [
path.join(repoRoot, "scripts", "compress-glb.js"),
"--input",
area.outputs.glb,
"--output",
area.outputs.compressedGlb,
"--texture-size",
String(area.compress.textureSize),
"--quality",
String(area.compress.quality),
"--effort",
String(area.compress.effort),
"--metadata",
area.outputs.metadata,
"--metadata-output",
area.outputs.compressedMetadata,
"--preview",
area.outputs.cesiumPreview,
"--preview-output",
area.outputs.compressedCesiumPreview,
];
if (area.compress.meshopt) {
compressArgs.push("--meshopt");
}
console.log("Stage: compress (Cesium GLB texture resize + WebP)");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(process.execPath, compressArgs, "compress");
const finished = Date.now();
const sourceDigest = glbDigest(area.outputs.glb);
const compressedDigest = glbDigest(area.outputs.compressedGlb);
writeStageManifest(area, {
stage: "compress",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
},
outputs: {
compressedGlb: fileRecord(area.outputs.compressedGlb),
compressedMetadata: fileRecord(area.outputs.compressedMetadata),
compressedCesiumPreview: fileRecord(area.outputs.compressedCesiumPreview),
},
summary: {
sourceGlb: glbSummary(sourceDigest),
compressedGlb: glbSummary(compressedDigest),
budget: evaluateGlbBudget(compressedDigest, area.budget),
options: {
textureSize: area.compress.textureSize,
quality: area.compress.quality,
effort: area.compress.effort,
meshopt: area.compress.meshopt,
},
compressionRatio: Number((compressedDigest.fileBytes / sourceDigest.fileBytes).toFixed(4)),
savedBytes: sourceDigest.fileBytes - compressedDigest.fileBytes,
},
warnings: glbBudgetWarnings("Compressed", compressedDigest, area.budget),
});
}
function blenderExecutable(area) {
return resolveBlenderExecutable(area.blenderApp);
}
function ensureFile(file, label) {
if (!fs.existsSync(file)) {
throw new Error(`${label} not found: ${file}`);
}
}
function runCommand(command, commandArgs, stage) {
const result = spawnSync(command, commandArgs, { stdio: "inherit" });
if (result.error) {
throw result.error;
}
if (result.status !== 0) {
const signal = result.signal ? ` signal=${result.signal}` : "";
throw new Error(`Stage '${stage}' failed with status=${result.status}${signal}`);
}
}
function writeCesiumPreview(area) {
ensureFile(area.outputs.glb, "Cesium GLB");
ensureFile(area.outputs.metadata, "Cesium metadata");
ensureFile(area.outputs.trafficSignals, "Traffic signal anchors");
const htmlPath = area.outputs.cesiumPreview;
const started = Date.now();
const startedAt = new Date(started).toISOString();
fs.mkdirSync(path.dirname(htmlPath), { recursive: true });
writeVehicleRoute(area);
const vehicleModelNames = writeVehicleModel(area);
writeCesiumPreviewSupportFiles(path.dirname(htmlPath));
const glbName = path.basename(area.outputs.glb);
const metadataName = path.basename(area.outputs.metadata);
const routeName = path.basename(area.outputs.vehicleRoute);
const vehicleModelName = path.basename(area.outputs.vehicleModel);
fs.writeFileSync(htmlPath, cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, area.id, vehicleModelNames, previewRelativePath(area.outputs.areaDir, area.outputs.trafficSignals)));
console.log(`Cesium preview: ${htmlPath}`);
const finished = Date.now();
writeStageManifest(area, {
stage: "preview",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
previewCss: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.css")),
previewJs: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.js")),
},
outputs: {
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
vehicleRoute: fileRecord(area.outputs.vehicleRoute),
vehicleModel: fileRecord(area.outputs.vehicleModel),
trafficSignals: fileRecord(area.outputs.trafficSignals),
},
summary: previewSummary(area),
warnings: [],
});
}
function writeTrafficSignals(area) {
const signals = readTrafficSignals(area.outputs.trafficSignalAssemblies, area.input);
fs.writeFileSync(area.outputs.trafficSignals, `${JSON.stringify(signals, null, 2)}\n`);
console.log(`Traffic signals: ${signals.signals.length} anchors in ${area.outputs.trafficSignals}`);
}
function previewRelativePath(fromDir, target) {
return path.relative(fromDir, target).split(path.sep).join("/");
}
function writeVehicleRoute(area) {
const route = buildPreviewVehicleRoute(area.input);
fs.mkdirSync(path.dirname(area.outputs.vehicleRoute), { recursive: true });
fs.writeFileSync(area.outputs.vehicleRoute, `${JSON.stringify(route, null, 2)}\n`);
console.log(`Vehicle route: ${area.outputs.vehicleRoute}`);
}
function writeVehicleModel(area) {
fs.mkdirSync(path.dirname(area.outputs.vehicleModel), { recursive: true });
const outDir = path.dirname(area.outputs.vehicleModel);
const fileStem = path.basename(area.outputs.vehicleModel, ".gltf").replace(/-vehicle-car$/, "");
const models = writePreviewVehicleLibrary(outDir, fileStem);
// Preserve the existing output/manifest contract for old preview HTML.
fs.copyFileSync(path.join(outDir, models[0]), area.outputs.vehicleModel);
console.log(`Vehicle models: ${models.length} candidates in ${path.dirname(area.outputs.vehicleModel)}`);
return models;
}
function sceneGeojsonRecords(area) {
const records = {};
for (const layer of SCENE_LAYERS) {
records[layer.id] = fileRecord(path.join(area.outputs.geojsonDir, layerFile(layer)));
}
records[SCENE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_FILE));
records[SCENE_STYLE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_STYLE_FILE));
return records;
}
function geojsonFeatureCounts(area) {
const out = {};
for (const layer of SCENE_LAYERS) {
const file = path.join(area.outputs.geojsonDir, layerFile(layer));
out[layer.id] = featureCount(file);
}
out[SCENE_FILE] = featureCount(path.join(area.outputs.geojsonDir, SCENE_FILE));
return out;
}
function featureCount(file) {
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
return Array.isArray(parsed.features) ? parsed.features.length : null;
}