que01/osmWorkflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add full stage manifests

Browse Source
This commit is contained in:
que01
2026-08-04 11:37:20 +08:00
parent 0b7e69770c
commit 5874cf9379
9 changed files with 516 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
scripts/build-area.js
View File

@@ -4,12 +4,19 @@ const fs = require("fs");
const path = require("path");
const { spawnSync } = require("child_process");
const { readAreaConfig } = require("./lib/area-config");
const {
SCENE_LAYERS,
SCENE_FILE,
SCENE_STYLE_FILE,
layerFile,
} = require("./lib/scene-layers");
const { digest: glbDigest } = require("./glb-digest");
const {
fileRecord,
glbBudgetWarnings,
glbSummary,
optionalFileRecord,
stageManifestPath,
writeStageManifest,
} = require("./lib/stage-manifest");
@@ -154,6 +161,8 @@ function writeDerivedConfig(area) {
}
function buildIntermediates(area) {
const started = Date.now();
const startedAt = new Date(started).toISOString();
const derivedConfigPath = writeDerivedConfig(area);
console.log("Stage: intermediates (osm2streets GeoJSON + QGIS)");
@@ -162,9 +171,37 @@ function buildIntermediates(area) {
"--config",
derivedConfigPath,
], "intermediates");
fs.rmSync(stageManifestPath(area, "reimport"), { force: true });
const finished = Date.now();
writeStageManifest(area, {
stage: "intermediates",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
},
outputs: {
derivedConfig: fileRecord(derivedConfigPath),
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
gpkg: fileRecord(area.outputs.gpkg),
qgisProject: fileRecord(area.outputs.qgisProject),
qgisPreview: optionalFileRecord(area.outputs.qgisPreview),
},
summary: {
geojson: geojsonFeatureCounts(area),
},
warnings: [],
});
}
function reimportGpkg(area) {
const started = Date.now();
const startedAt = new Date(started).toISOString();
const derivedConfigPath = writeDerivedConfig(area);
console.log("Stage: reimport (GeoPackage -> GeoJSON)");
@@ -173,6 +210,29 @@ function reimportGpkg(area) {
"--config",
derivedConfigPath,
], "reimport");
fs.rmSync(stageManifestPath(area, "intermediates"), { force: true });
const finished = Date.now();
writeStageManifest(area, {
stage: "reimport",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
derivedConfig: fileRecord(derivedConfigPath),
gpkg: fileRecord(area.outputs.gpkg),
},
outputs: {
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
},
summary: {
geojson: geojsonFeatureCounts(area),
},
warnings: [],
});
}
function buildBlenderScene(area) {
@@ -203,7 +263,34 @@ function buildBlenderScene(area) {
}
console.log("Stage: blender");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(blenderExecutable(area), blenderArgs, "blender");
const finished = Date.now();
writeStageManifest(area, {
stage: "blender",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
},
outputs: {
blend: fileRecord(area.outputs.blend),
render: fileRecord(area.outputs.render),
},
summary: {
geojson: geojsonFeatureCounts(area),
blendBytes: fileRecord(area.outputs.blend).bytes,
renderBytes: fileRecord(area.outputs.render).bytes,
},
warnings: [],
});
}
function exportCesium(area) {
@@ -244,9 +331,6 @@ function exportCesium(area) {
outputs: {
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
vehicleRoute: optionalFileRecord(area.outputs.vehicleRoute),
vehicleModel: optionalFileRecord(area.outputs.vehicleModel),
},
summary: {
glb: glbSummary(digest),
@@ -354,6 +438,8 @@ function writeCesiumPreview(area) {
ensureFile(area.outputs.glb, "Cesium GLB");
ensureFile(area.outputs.metadata, "Cesium metadata");
const htmlPath = area.outputs.cesiumPreview;
const started = Date.now();
const startedAt = new Date(started).toISOString();
fs.mkdirSync(path.dirname(htmlPath), { recursive: true });
writeVehicleRoute(area);
writeVehicleModel(area);
@@ -364,6 +450,30 @@ function writeCesiumPreview(area) {
const vehicleModelName = path.basename(area.outputs.vehicleModel);
fs.writeFileSync(htmlPath, cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, area.id));
console.log(`Cesium preview: ${htmlPath}`);
const finished = Date.now();
writeStageManifest(area, {
stage: "preview",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
previewCss: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.css")),
previewJs: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.js")),
},
outputs: {
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
vehicleRoute: fileRecord(area.outputs.vehicleRoute),
vehicleModel: fileRecord(area.outputs.vehicleModel),
},
summary: previewSummary(area),
warnings: [],
});
}
function writeCesiumPreviewSupportFiles(outDir) {
@@ -813,3 +923,39 @@ function escapeScriptJson(value) {
.replaceAll("\u2028", "\\u2028")
.replaceAll("\u2029", "\\u2029");
}
function sceneGeojsonRecords(area) {
const records = {};
for (const layer of SCENE_LAYERS) {
records[layer.id] = fileRecord(path.join(area.outputs.geojsonDir, layerFile(layer)));
}
records[SCENE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_FILE));
records[SCENE_STYLE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_STYLE_FILE));
return records;
}
function geojsonFeatureCounts(area) {
const out = {};
for (const layer of SCENE_LAYERS) {
const file = path.join(area.outputs.geojsonDir, layerFile(layer));
out[layer.id] = featureCount(file);
}
out[SCENE_FILE] = featureCount(path.join(area.outputs.geojsonDir, SCENE_FILE));
return out;
}
function featureCount(file) {
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
return Array.isArray(parsed.features) ? parsed.features.length : null;
}
function previewSummary(area) {
const route = JSON.parse(fs.readFileSync(area.outputs.vehicleRoute, "utf8"));
return {
glbName: path.basename(area.outputs.glb),
metadataName: path.basename(area.outputs.metadata),
routeName: path.basename(area.outputs.vehicleRoute),
vehicleModelName: path.basename(area.outputs.vehicleModel),
routeSegments: Array.isArray(route.segments) ? route.segments.length : null,
};
}

112
scripts/lib/area-diagnostics.js
View File

@@ -3,6 +3,12 @@
const fs = require("fs");
const path = require("path");
const { readAreaConfig } = require("./area-config");
const {
SCENE_LAYERS,
SCENE_FILE,
SCENE_STYLE_FILE,
layerFile,
} = require("./scene-layers");
const { digest: glbDigest } = require("../glb-digest");
const { BUDGETS, fileRecord, readStageManifest, stageManifestPath } = require("./stage-manifest");
@@ -16,7 +22,7 @@ function analyzeArea(configPath, options = {}) {
const area = readAreaConfig(resolvedConfig, { repoRoot });
const osm = parseOsm(fs.readFileSync(area.input, "utf8"));
const artifacts = artifactStatus(area);
const manifests = stageManifestStatus(area);
const manifests = stageManifestStatus(area, resolvedConfig);
const metadataWarnings = [];
const metadata = metadataSummary(area.outputs.metadata, metadataWarnings);
const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null;
@@ -296,8 +302,58 @@ function metadataSummary(file, warnings) {
}
}
function stageManifestStatus(area) {
function stageManifestStatus(area, configPath = null) {
const reimportManifest = stageManifestPath(area, "reimport");
const hasReimportManifest = fs.existsSync(reimportManifest);
const derivedConfig = path.join(area.outputs.pipelineDir, "osm2streets-qgis.config.json");
const stages = [
{
stage: "intermediates",
expected: (
fs.existsSync(area.outputs.geojsonDir) ||
fs.existsSync(area.outputs.gpkg) ||
fs.existsSync(area.outputs.qgisProject)
) && !hasReimportManifest,
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
},
outputs: {
derivedConfig,
geojsonDir: area.outputs.geojsonDir,
...sceneGeojsonFiles(area),
gpkg: area.outputs.gpkg,
qgisProject: area.outputs.qgisProject,
qgisPreview: optionalExpectedFile(area.outputs.qgisPreview),
},
},
{
stage: "reimport",
expected: hasReimportManifest,
inputs: {
...(configPath ? { config: configPath } : {}),
derivedConfig,
gpkg: area.outputs.gpkg,
},
outputs: {
geojsonDir: area.outputs.geojsonDir,
...sceneGeojsonFiles(area),
},
},
{
stage: "blender",
expected: fs.existsSync(area.outputs.blend),
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
geojsonDir: area.outputs.geojsonDir,
...sceneGeojsonFiles(area),
},
outputs: {
blend: area.outputs.blend,
render: area.outputs.render,
},
},
{
stage: "cesium",
expected: fs.existsSync(area.outputs.glb),
@@ -307,7 +363,23 @@ function stageManifestStatus(area) {
outputs: {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
},
},
{
stage: "preview",
expected: fs.existsSync(area.outputs.cesiumPreview),
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
glb: area.outputs.glb,
metadata: area.outputs.metadata,
previewCss: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.css"),
previewJs: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.js"),
},
outputs: {
cesiumPreview: area.outputs.cesiumPreview,
vehicleRoute: area.outputs.vehicleRoute,
vehicleModel: area.outputs.vehicleModel,
},
},
{
@@ -375,12 +447,19 @@ function stageManifestStatus(area) {
function manifestFileIssues(records, expectedFiles, label) {
const issues = [];
for (const [key, file] of Object.entries(expectedFiles)) {
for (const [key, expected] of Object.entries(expectedFiles)) {
const { file, required } = normalizeExpectedFile(expected);
const recorded = records[key];
if (!required && !recorded && !fs.existsSync(file)) {
continue;
}
if (!recorded) {
issues.push(`${label} ${key} not recorded`);
continue;
}
if (!required && recorded === null && !fs.existsSync(file)) {
continue;
}
if (!fs.existsSync(file)) {
issues.push(`${label} ${key} file missing`);
continue;
@@ -395,6 +474,33 @@ function manifestFileIssues(records, expectedFiles, label) {
return issues;
}
function normalizeExpectedFile(expected) {
if (expected && typeof expected === "object" && expected.path) {
return {
file: expected.path,
required: expected.required !== false,
};
}
return {
file: expected,
required: true,
};
}
function optionalExpectedFile(file) {
return { path: file, required: false };
}
function sceneGeojsonFiles(area) {
const files = {};
for (const layer of SCENE_LAYERS) {
files[layer.id] = path.join(area.outputs.geojsonDir, layerFile(layer));
}
files[SCENE_FILE] = path.join(area.outputs.geojsonDir, SCENE_FILE);
files[SCENE_STYLE_FILE] = path.join(area.outputs.geojsonDir, SCENE_STYLE_FILE);
return files;
}
function collectWarnings(area, osm, artifacts, manifests, glb, metadata) {
const warnings = [];
if (!osm.bounds) warnings.push("OSM has no valid <bounds>; scene extent may be wrong.");