From 5874cf9379bebdc1af6386f806ee6bfe85e045e3 Mon Sep 17 00:00:00 2001 From: que01 Date: Tue, 4 Aug 2026 11:37:20 +0800 Subject: [PATCH] Add full stage manifests --- .trellis/spec/pipeline/cli-and-stages.md | 82 ++++++++-- .trellis/spec/pipeline/index.md | 8 +- .../08-04-add-full-stage-manifests/design.md | 77 +++++++++ .../implement.md | 30 ++++ .../08-04-add-full-stage-manifests/prd.md | 50 ++++++ README.md | 18 ++- docs/changelog.md | 8 + scripts/build-area.js | 152 +++++++++++++++++- scripts/lib/area-diagnostics.js | 112 ++++++++++++- 9 files changed, 516 insertions(+), 21 deletions(-) create mode 100644 .trellis/tasks/08-04-add-full-stage-manifests/design.md create mode 100644 .trellis/tasks/08-04-add-full-stage-manifests/implement.md create mode 100644 .trellis/tasks/08-04-add-full-stage-manifests/prd.md diff --git a/.trellis/spec/pipeline/cli-and-stages.md b/.trellis/spec/pipeline/cli-and-stages.md index 51418e3..df099ce 100644 --- a/.trellis/spec/pipeline/cli-and-stages.md +++ b/.trellis/spec/pipeline/cli-and-stages.md @@ -300,9 +300,10 @@ const gate = classifyAreaQuality(result); ### 1. Scope / Trigger -Stage manifest 是构建阶段完成后的机器可读产物契约。第一版覆盖 GLB 相关阶段： -`cesium` 和 `compress`。它用于诊断产物是否存在、是否 stale、体量是否超预算，以及后续 -`check:area` / 增量构建判断。 +Stage manifest 是构建阶段完成后的机器可读产物契约。它覆盖完整区域链： +`intermediates`、`reimport`、`blender`、`cesium`、`preview` 和 `compress`。 +它用于诊断产物是否存在、是否 stale、体量是否超预算，以及后续 `check:area` / +增量构建判断。 ### 2. Signatures @@ -315,7 +316,11 @@ Manifest 路径固定： 当前会写： ```text +/_pipeline/stages/intermediates.manifest.json +/_pipeline/stages/reimport.manifest.json +/_pipeline/stages/blender.manifest.json /_pipeline/stages/cesium.manifest.json +/_pipeline/stages/preview.manifest.json /_pipeline/stages/compress.manifest.json ``` @@ -357,18 +362,65 @@ File records use this shape: } ``` +`intermediates` manifest: + +- `inputs.config` +- `inputs.osm` +- `outputs.derivedConfig` +- `outputs.geojsonDir` +- `outputs.`、`outputs.osm2streets_scene.geojson`、 + `outputs.osm2streets_scene_style.json` +- `outputs.gpkg`、`outputs.qgisProject`、可选 `outputs.qgisPreview` +- `summary.geojson`：各 layer / merged scene 的 feature count + +`reimport` manifest: + +- `inputs.config` +- `inputs.derivedConfig` +- `inputs.gpkg` +- `outputs.geojsonDir` +- `outputs.`、merged scene/style +- `summary.geojson` + +`intermediates` 和 `reimport` 是同一批 GeoJSON 的互斥 ownership： +成功运行 `intermediates` 删除旧 `reimport.manifest.json`；成功运行 `reimport` 删除旧 +`intermediates.manifest.json`。诊断只要求当前 ownership 路径的 manifest，不要求两者同时存在。 + +`blender` manifest: + +- `inputs.config` +- `inputs.osm` +- `inputs.geojsonDir`、`inputs.`、merged scene/style +- `outputs.blend` +- `outputs.render` +- `summary.geojson`、`summary.blendBytes`、`summary.renderBytes` + `cesium` manifest: - `inputs.blend` - `outputs.glb` - `outputs.metadata` -- `outputs.cesiumPreview` -- `outputs.vehicleRoute` -- `outputs.vehicleModel` - `summary.glb.fileBytes` - `summary.glb.counts` - `summary.glb.extensionsUsed` +`cesium` 会调用 preview 生成函数，但 preview HTML / route / vehicle model 的 freshness +所有权属于独立 `preview` manifest。否则单跑 `--stages preview` 会把 Cesium manifest +错误判 stale。 + +`preview` manifest: + +- `inputs.config` +- `inputs.osm` +- `inputs.glb` +- `inputs.metadata` +- `inputs.previewCss` +- `inputs.previewJs` +- `outputs.cesiumPreview` +- `outputs.vehicleRoute` +- `outputs.vehicleModel` +- `summary.glbName`、`summary.metadataName`、`summary.routeSegments` + `compress` manifest: - `inputs.glb` @@ -395,12 +447,19 @@ Manifest files are written atomically via `*.tmp` then `renameSync`. | Manifest JSON 无法解析 | 诊断报告 invalid warning | | Manifest 记录的 input / output sha/bytes 与当前文件不同 | 诊断报告 stale warning | | GLB size / nodes / images 超保守预算 | manifest `warnings[]` 记录，诊断继续独立报警 | +| `intermediates` 后留有旧 reimport manifest | 成功后删除旧 reimport manifest | +| `reimport` 后留有旧 intermediates manifest | 成功后删除旧 intermediates manifest | ### 5. Good/Base/Bad Cases -- Good: `--stages cesium` 成功后写 `cesium.manifest.json`，诊断显示 `ok cesium`。 +- Good: `--stages intermediates` 成功后写 `intermediates.manifest.json`，诊断显示 + `ok intermediates`。 +- Good: `--stages blender` 成功后写 `blender.manifest.json`，诊断显示 `ok blender`。 +- Good: `--stages cesium` 成功后写 `cesium.manifest.json` 和 `preview.manifest.json`。 +- Good: `--stages preview` 只更新 preview manifest，不让 cesium manifest stale。 - Good: `--stages compress` 成功后写 `compress.manifest.json`，summary 记录压缩比和节省字节。 -- Base: 旧产物没有 manifest，诊断显示 expected manifest missing，提示重跑对应阶段。 +- Base: 旧产物没有当前 ownership 路径的 manifest，诊断显示 expected manifest missing， + 提示重跑对应阶段。 - Bad: 手工编辑 GLB 后不重跑阶段，诊断显示 manifest stale。 ### 6. Tests Required @@ -408,10 +467,15 @@ Manifest files are written atomically via `*.tmp` then `renameSync`. - `node --check scripts/lib/stage-manifest.js` - `node --check scripts/build-area.js` - `node --check scripts/diagnose-area.js` +- `node --check scripts/check-area.js` +- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages intermediates` +- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages blender` - `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages cesium` +- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages preview` - `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress` - `npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.json` -- JSON parse both generated manifests and assert `manifestVersion`, `stage`, `inputs`, +- `npm run check:area -- --config config/areas/nantaizi-lake-innovation-valley.json` +- JSON parse generated manifests and assert `manifestVersion`, `stage`, `inputs`, `outputs`, `summary`, and `warnings` fields exist. ### 7. Wrong vs Correct diff --git a/.trellis/spec/pipeline/index.md b/.trellis/spec/pipeline/index.md index 93b8f81..d8d1ad9 100644 --- a/.trellis/spec/pipeline/index.md +++ b/.trellis/spec/pipeline/index.md @@ -38,14 +38,17 @@ config/areas/.json │ → osm2streets_scene.geojson + _scene_style.json │ → ogr2ogr 导入 .gpkg │ → QGIS 生成 .qgz + -preview.png + │ → _pipeline/stages/intermediates.manifest.json │ ├─[reimport]──────▶ reimport-gpkg.js （反向，与 intermediates 互斥） │ ogr2ogr 从 .gpkg 导出 → 校验 → 覆写 *.geojson │ → 重建 scene.geojson + scene_style.json + │ → _pipeline/stages/reimport.manifest.json │ ├─[blender]───────▶ Blender + blender/generate_scene.py │ 读 .osm + osm2streets_web_out/ │ → .blend + .png + │ → _pipeline/stages/blender.manifest.json │ ├─[cesium]────────▶ Blender + blender/export_cesium.py │ 读 .blend → .glb + .json @@ -55,6 +58,7 @@ config/areas/.json ├─[preview]───────▶ 生成 -cesium-preview.html │ + 拷贝 lib/cesium-preview.{js,css} │ + 车辆巡航路线与模型 + │ → _pipeline/stages/preview.manifest.json │ └─[compress]──────▶ 生成并列压缩 GLB / metadata / preview → _pipeline/stages/compress.manifest.json @@ -83,10 +87,10 @@ config/areas/.json | 文件 | 行数 | 职责 | |---|---|---| -| `build-area.js` | 815 | 主入口：区域配置读取、阶段调度、Cesium 预览页、车辆巡航和 stage manifest 写入 | +| `build-area.js` | 961 | 主入口：区域配置读取、阶段调度、Cesium 预览页、车辆巡航和 stage manifest 写入 | | `diagnose-area.js` | 36 | 快速诊断入口：调用共享 area diagnostics 并打印完整报告 | | `check-area.js` | 74 | 区域质量门入口：调用共享 area diagnostics，输出 PASS/FAIL 并设置退出码 | -| `lib/area-diagnostics.js` | 660 | 共享区域诊断事实源：OSM、产物、metadata、stage manifest、GLB digest 和质量门分类 | +| `lib/area-diagnostics.js` | 776 | 共享区域诊断事实源：OSM、产物、metadata、stage manifest、GLB digest 和质量门分类 | | `lib/area-config.js` | 135 | 区域配置归一化与输出路径推导，供 build / diagnose 复用 | | `lib/stage-manifest.js` | 100 | stage manifest 路径、文件记录、GLB budget warning 和原子 JSON 写入 | | `build-osm2streets-qgis.js` | 1468 | intermediates：osm2streets 解析、图层拆分、人行道转角合成、GeoPackage 与 QGIS 工程生成 | diff --git a/.trellis/tasks/08-04-add-full-stage-manifests/design.md b/.trellis/tasks/08-04-add-full-stage-manifests/design.md new file mode 100644 index 0000000..449dab8 --- /dev/null +++ b/.trellis/tasks/08-04-add-full-stage-manifests/design.md @@ -0,0 +1,77 @@ +# Design + +## Scope + +The existing manifest infrastructure already supports file records, atomic JSON +write, digest summaries, and diagnostics-side freshness checks. This task +extends that same contract to the earlier and read-only/patched stages without +adding a new manifest format. + +## Stage Boundaries + +`intermediates` + +- Written by `build-area.js` after `build-osm2streets-qgis.js` succeeds. +- Inputs: area config, source OSM XML. +- Outputs: derived config, GeoJSON directory, all scene-layer GeoJSON files, + merged scene/style files, GeoPackage, QGIS project, QGIS preview if present. +- Summary: GeoJSON feature counts by file. + +`reimport` + +- Written by `build-area.js` after `reimport-gpkg.js` succeeds. +- Inputs: area config, GeoPackage. +- Outputs: GeoJSON directory, all scene-layer GeoJSON files, merged scene/style + files. +- Summary: GeoJSON feature counts by file. + +`blender` + +- Written by `build-area.js` after Blender scene generation succeeds. +- Inputs: area config, source OSM XML, GeoJSON directory, all scene-layer + GeoJSON files, merged scene/style files. +- Outputs: `.blend`, render PNG. +- Summary: parsed `SCENE_DONE` payload when available, plus blend/render bytes. + +`preview` + +- Written by `build-area.js` after preview generation succeeds. Since `cesium` + automatically calls preview generation, this manifest should also be written + during `--stages cesium`. +- Inputs: area config, source OSM XML, GLB, metadata, preview runtime JS/CSS. +- Outputs: Cesium preview HTML, vehicle route JSON, vehicle model glTF. +- Summary: vehicle route count and asset filenames. + +## Diagnostics + +`scripts/lib/area-diagnostics.js` will extend `stageManifestStatus()` from two +hard-coded GLB stages to the full ordered stage list. Expected status should +derive from stage outputs: + +- `intermediates`: expected when GeoJSON dir, GeoPackage, or QGIS project exists. +- `reimport`: expected when GeoJSON dir exists and `reimport.manifest.json` + already exists; it is an alternate path and should not be required for normal + intermediates builds. +- `blender`: expected when blend exists. +- `cesium`: expected when baseline GLB exists. +- `preview`: expected when Cesium preview exists. +- `compress`: expected when compressed GLB exists. + +Only expected manifests are fatal in `check:area`. Optional absent manifests +remain ignored. + +## Compatibility + +- Existing `cesium.manifest.json` and `compress.manifest.json` remain valid. +- Existing outputs without new manifests will show missing expected manifests + until their stages are rerun. +- No stage should overwrite its primary outputs differently just to write a + manifest. + +## Tradeoffs + +- Manifest writing lives in `build-area.js` for this task because it already + owns normalized area paths and stage timing. Moving preview generation and + vehicle assets out of `build-area.js` is a later refactor. +- Reimport remains optional/alternate to avoid making normal `check:area` fail + for users who never use the reimport workflow. diff --git a/.trellis/tasks/08-04-add-full-stage-manifests/implement.md b/.trellis/tasks/08-04-add-full-stage-manifests/implement.md new file mode 100644 index 0000000..825b80f --- /dev/null +++ b/.trellis/tasks/08-04-add-full-stage-manifests/implement.md @@ -0,0 +1,30 @@ +# Implementation Plan + +1. Read current manifest and stage code. +2. Add reusable helpers in `build-area.js` for: + - file record collection + - GeoJSON feature-count summaries + - manifest duration/timestamps + - writing `intermediates`, `reimport`, `blender`, and `preview` manifests +3. Wire manifest writes after successful stage functions only. +4. Extend `area-diagnostics.js` manifest expected/input/output definitions to + cover all stages. +5. Update README, changelog, and `.trellis/spec/pipeline/*`. +6. Validate with: + - `node --check scripts/build-area.js` + - `node --check scripts/lib/area-diagnostics.js` + - `node --check scripts/check-area.js` + - `node --check scripts/diagnose-area.js` + - `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages preview` + - `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages blender` + - `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages intermediates` + - `npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.json` + - `npm run check:area -- --config config/areas/nantaizi-lake-innovation-valley.json` + - controlled stale/missing manifest failure + - `git diff --check` + +## Rollback + +- Remove added manifest write calls and diagnostics stage definitions. +- Delete generated `_pipeline/stages/{intermediates,reimport,blender,preview}.manifest.json` + from outputs if needed; outputs are ignored and not committed. diff --git a/.trellis/tasks/08-04-add-full-stage-manifests/prd.md b/.trellis/tasks/08-04-add-full-stage-manifests/prd.md new file mode 100644 index 0000000..12931e6 --- /dev/null +++ b/.trellis/tasks/08-04-add-full-stage-manifests/prd.md @@ -0,0 +1,50 @@ +# Add full stage manifests + +## Goal + +Extend the stage manifest contract from GLB-only stages to the full area +pipeline so `diagnose:area` and `check:area` can reason about stage freshness +across intermediates, reimport, blender, preview, cesium, and compress. + +## Requirements + +- Write stage manifests for: + - `intermediates` + - `reimport` + - `blender` + - `preview` +- Preserve existing `cesium` and `compress` manifest behavior. +- Do not change default build outputs except for adding/updating manifest files + under `/_pipeline/stages/`. +- Stage manifests must record enough inputs and outputs to detect stale files + using the existing bytes/sha freshness logic. +- `diagnose:area` must display these additional manifests. +- `check:area` must fail when an expected full-pipeline manifest is missing, + invalid, or stale. +- `reimport` and `intermediates` remain mutually exclusive. +- Missing optional `compress` artifacts still must not make `check:area` fail. +- README, changelog, and Trellis pipeline spec must document the expanded + manifest coverage. + +## Acceptance Criteria + +- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages preview` + writes a fresh `preview.manifest.json`. +- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages blender` + writes a fresh `blender.manifest.json`. +- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages intermediates` + writes a fresh `intermediates.manifest.json`. +- [x] `diagnose:area` shows `intermediates`, `blender`, `cesium`, `preview`, and + optional `compress` manifest status. +- [x] `check:area` passes for current nantaizi outputs after the required + manifests are generated. +- [x] A controlled stale/missing manifest case makes `check:area` exit non-zero. +- [x] Syntax checks pass for changed Node scripts. +- [x] README, changelog, and pipeline spec describe full stage manifest coverage. + +## Notes + +- Validate only `nantaizi-lake-innovation-valley` unless explicitly asked. +- Ignore `hanyang-block`. +- This task is about pipeline contracts and diagnostics, not lane-arrow visual + behavior. diff --git a/README.md b/README.md index f4a9f4d..797d3e3 100644 --- a/README.md +++ b/README.md @@ -76,16 +76,26 @@ npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.j 现有产物状态，以及 GLB 的 size / nodes / meshes / materials / images / extensions。 缺少已期望的基线产物、异常 building relation、GLB 超过保守预算等会进入 `Warnings`。 -`cesium` 和 `compress` 阶段成功后会写机器可读的阶段 manifest： +每个成功的区域构建阶段都会写机器可读的 stage manifest： ```text +outputs//_pipeline/stages/intermediates.manifest.json +outputs//_pipeline/stages/reimport.manifest.json +outputs//_pipeline/stages/blender.manifest.json outputs//_pipeline/stages/cesium.manifest.json +outputs//_pipeline/stages/preview.manifest.json outputs//_pipeline/stages/compress.manifest.json ``` -manifest 记录阶段输入/输出文件的 bytes、mtime、sha256、耗时、GLB 结构摘要和 warning。 -`diagnose:area` 会读取这些 manifest；如果已有 GLB 但 manifest 缺失，或者 manifest -记录的输入/输出 sha/bytes 和当前文件不一致，会在 `Stage manifests` 和 `Warnings` 里标出来。 +`intermediates` 和 `reimport` 是同一批 GeoJSON 的两种所有权路径：前者从 OSM 重建， +后者从手工编辑的 GeoPackage 回导。成功运行其中一个会清掉另一个的 manifest，避免旧路径 +造成假 stale。 + +manifest 记录阶段输入/输出文件的 bytes、mtime、sha256、耗时和结构摘要：前段记录 OSM / +GeoJSON feature counts，Blender 记录 `.blend` / render，Cesium/压缩记录 GLB digest， +preview 记录 GLB、metadata、车辆路线和 runtime 文件。`diagnose:area` 会读取这些 +manifest；缺失或当前输入/输出 sha/bytes 不一致会在 `Stage manifests` 和 `Warnings` +里标出来。 ## 区域质量门 diff --git a/docs/changelog.md b/docs/changelog.md index cd03d5c..e0fc86e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,14 @@ ## 2026-08-04 +- 扩展 stage manifest 到完整区域链：`intermediates`、`reimport`、`blender` 和 + `preview` 现在和既有 `cesium` / `compress` 一样，在成功后写 + `_pipeline/stages/.manifest.json`。前段 manifest 记录 config / OSM / + GeoJSON layer 文件与 feature count，Blender 记录 OSM / GeoJSON 输入和 `.blend` / + render，preview 记录 GLB / metadata / OSM / runtime 以及 HTML / route / vehicle + model。`intermediates` 与 `reimport` 采用互斥 ownership：成功运行一方会删除另一方的 + manifest；Cesium 和 preview 也拆开 freshness ownership，单跑 preview 不会把 + Cesium manifest 判 stale。`diagnose:area` / `check:area` 现在覆盖完整阶段链。 - 新增区域质量门入口：`npm run check:area -- --config config/areas/.json`。 它复用 `scripts/lib/area-diagnostics.js` 的 OSM、产物、metadata、stage manifest 和 GLB digest 检查，只读已有文件，不调用 QGIS / Blender / Cesium / 压缩 / 重建；报告 diff --git a/scripts/build-area.js b/scripts/build-area.js index f294f35..51f9d0f 100755 --- a/scripts/build-area.js +++ b/scripts/build-area.js @@ -4,12 +4,19 @@ const fs = require("fs"); const path = require("path"); const { spawnSync } = require("child_process"); const { readAreaConfig } = require("./lib/area-config"); +const { + SCENE_LAYERS, + SCENE_FILE, + SCENE_STYLE_FILE, + layerFile, +} = require("./lib/scene-layers"); const { digest: glbDigest } = require("./glb-digest"); const { fileRecord, glbBudgetWarnings, glbSummary, optionalFileRecord, + stageManifestPath, writeStageManifest, } = require("./lib/stage-manifest"); @@ -154,6 +161,8 @@ function writeDerivedConfig(area) { } function buildIntermediates(area) { + const started = Date.now(); + const startedAt = new Date(started).toISOString(); const derivedConfigPath = writeDerivedConfig(area); console.log("Stage: intermediates (osm2streets GeoJSON + QGIS)"); @@ -162,9 +171,37 @@ function buildIntermediates(area) { "--config", derivedConfigPath, ], "intermediates"); + fs.rmSync(stageManifestPath(area, "reimport"), { force: true }); + const finished = Date.now(); + writeStageManifest(area, { + stage: "intermediates", + status: "ok", + config: configPath, + startedAt, + finishedAt: new Date(finished).toISOString(), + durationMs: finished - started, + inputs: { + config: fileRecord(configPath), + osm: fileRecord(area.input), + }, + outputs: { + derivedConfig: fileRecord(derivedConfigPath), + geojsonDir: fileRecord(area.outputs.geojsonDir), + ...sceneGeojsonRecords(area), + gpkg: fileRecord(area.outputs.gpkg), + qgisProject: fileRecord(area.outputs.qgisProject), + qgisPreview: optionalFileRecord(area.outputs.qgisPreview), + }, + summary: { + geojson: geojsonFeatureCounts(area), + }, + warnings: [], + }); } function reimportGpkg(area) { + const started = Date.now(); + const startedAt = new Date(started).toISOString(); const derivedConfigPath = writeDerivedConfig(area); console.log("Stage: reimport (GeoPackage -> GeoJSON)"); @@ -173,6 +210,29 @@ function reimportGpkg(area) { "--config", derivedConfigPath, ], "reimport"); + fs.rmSync(stageManifestPath(area, "intermediates"), { force: true }); + const finished = Date.now(); + writeStageManifest(area, { + stage: "reimport", + status: "ok", + config: configPath, + startedAt, + finishedAt: new Date(finished).toISOString(), + durationMs: finished - started, + inputs: { + config: fileRecord(configPath), + derivedConfig: fileRecord(derivedConfigPath), + gpkg: fileRecord(area.outputs.gpkg), + }, + outputs: { + geojsonDir: fileRecord(area.outputs.geojsonDir), + ...sceneGeojsonRecords(area), + }, + summary: { + geojson: geojsonFeatureCounts(area), + }, + warnings: [], + }); } function buildBlenderScene(area) { @@ -203,7 +263,34 @@ function buildBlenderScene(area) { } console.log("Stage: blender"); + const started = Date.now(); + const startedAt = new Date(started).toISOString(); runCommand(blenderExecutable(area), blenderArgs, "blender"); + const finished = Date.now(); + writeStageManifest(area, { + stage: "blender", + status: "ok", + config: configPath, + startedAt, + finishedAt: new Date(finished).toISOString(), + durationMs: finished - started, + inputs: { + config: fileRecord(configPath), + osm: fileRecord(area.input), + geojsonDir: fileRecord(area.outputs.geojsonDir), + ...sceneGeojsonRecords(area), + }, + outputs: { + blend: fileRecord(area.outputs.blend), + render: fileRecord(area.outputs.render), + }, + summary: { + geojson: geojsonFeatureCounts(area), + blendBytes: fileRecord(area.outputs.blend).bytes, + renderBytes: fileRecord(area.outputs.render).bytes, + }, + warnings: [], + }); } function exportCesium(area) { @@ -244,9 +331,6 @@ function exportCesium(area) { outputs: { glb: fileRecord(area.outputs.glb), metadata: fileRecord(area.outputs.metadata), - cesiumPreview: fileRecord(area.outputs.cesiumPreview), - vehicleRoute: optionalFileRecord(area.outputs.vehicleRoute), - vehicleModel: optionalFileRecord(area.outputs.vehicleModel), }, summary: { glb: glbSummary(digest), @@ -354,6 +438,8 @@ function writeCesiumPreview(area) { ensureFile(area.outputs.glb, "Cesium GLB"); ensureFile(area.outputs.metadata, "Cesium metadata"); const htmlPath = area.outputs.cesiumPreview; + const started = Date.now(); + const startedAt = new Date(started).toISOString(); fs.mkdirSync(path.dirname(htmlPath), { recursive: true }); writeVehicleRoute(area); writeVehicleModel(area); @@ -364,6 +450,30 @@ function writeCesiumPreview(area) { const vehicleModelName = path.basename(area.outputs.vehicleModel); fs.writeFileSync(htmlPath, cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, area.id)); console.log(`Cesium preview: ${htmlPath}`); + const finished = Date.now(); + writeStageManifest(area, { + stage: "preview", + status: "ok", + config: configPath, + startedAt, + finishedAt: new Date(finished).toISOString(), + durationMs: finished - started, + inputs: { + config: fileRecord(configPath), + osm: fileRecord(area.input), + glb: fileRecord(area.outputs.glb), + metadata: fileRecord(area.outputs.metadata), + previewCss: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.css")), + previewJs: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.js")), + }, + outputs: { + cesiumPreview: fileRecord(area.outputs.cesiumPreview), + vehicleRoute: fileRecord(area.outputs.vehicleRoute), + vehicleModel: fileRecord(area.outputs.vehicleModel), + }, + summary: previewSummary(area), + warnings: [], + }); } function writeCesiumPreviewSupportFiles(outDir) { @@ -813,3 +923,39 @@ function escapeScriptJson(value) { .replaceAll("\u2028", "\\u2028") .replaceAll("\u2029", "\\u2029"); } + +function sceneGeojsonRecords(area) { + const records = {}; + for (const layer of SCENE_LAYERS) { + records[layer.id] = fileRecord(path.join(area.outputs.geojsonDir, layerFile(layer))); + } + records[SCENE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_FILE)); + records[SCENE_STYLE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_STYLE_FILE)); + return records; +} + +function geojsonFeatureCounts(area) { + const out = {}; + for (const layer of SCENE_LAYERS) { + const file = path.join(area.outputs.geojsonDir, layerFile(layer)); + out[layer.id] = featureCount(file); + } + out[SCENE_FILE] = featureCount(path.join(area.outputs.geojsonDir, SCENE_FILE)); + return out; +} + +function featureCount(file) { + const parsed = JSON.parse(fs.readFileSync(file, "utf8")); + return Array.isArray(parsed.features) ? parsed.features.length : null; +} + +function previewSummary(area) { + const route = JSON.parse(fs.readFileSync(area.outputs.vehicleRoute, "utf8")); + return { + glbName: path.basename(area.outputs.glb), + metadataName: path.basename(area.outputs.metadata), + routeName: path.basename(area.outputs.vehicleRoute), + vehicleModelName: path.basename(area.outputs.vehicleModel), + routeSegments: Array.isArray(route.segments) ? route.segments.length : null, + }; +} diff --git a/scripts/lib/area-diagnostics.js b/scripts/lib/area-diagnostics.js index 71023c3..ad96e8a 100644 --- a/scripts/lib/area-diagnostics.js +++ b/scripts/lib/area-diagnostics.js @@ -3,6 +3,12 @@ const fs = require("fs"); const path = require("path"); const { readAreaConfig } = require("./area-config"); +const { + SCENE_LAYERS, + SCENE_FILE, + SCENE_STYLE_FILE, + layerFile, +} = require("./scene-layers"); const { digest: glbDigest } = require("../glb-digest"); const { BUDGETS, fileRecord, readStageManifest, stageManifestPath } = require("./stage-manifest"); @@ -16,7 +22,7 @@ function analyzeArea(configPath, options = {}) { const area = readAreaConfig(resolvedConfig, { repoRoot }); const osm = parseOsm(fs.readFileSync(area.input, "utf8")); const artifacts = artifactStatus(area); - const manifests = stageManifestStatus(area); + const manifests = stageManifestStatus(area, resolvedConfig); const metadataWarnings = []; const metadata = metadataSummary(area.outputs.metadata, metadataWarnings); const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null; @@ -296,8 +302,58 @@ function metadataSummary(file, warnings) { } } -function stageManifestStatus(area) { +function stageManifestStatus(area, configPath = null) { + const reimportManifest = stageManifestPath(area, "reimport"); + const hasReimportManifest = fs.existsSync(reimportManifest); + const derivedConfig = path.join(area.outputs.pipelineDir, "osm2streets-qgis.config.json"); const stages = [ + { + stage: "intermediates", + expected: ( + fs.existsSync(area.outputs.geojsonDir) || + fs.existsSync(area.outputs.gpkg) || + fs.existsSync(area.outputs.qgisProject) + ) && !hasReimportManifest, + inputs: { + ...(configPath ? { config: configPath } : {}), + osm: area.input, + }, + outputs: { + derivedConfig, + geojsonDir: area.outputs.geojsonDir, + ...sceneGeojsonFiles(area), + gpkg: area.outputs.gpkg, + qgisProject: area.outputs.qgisProject, + qgisPreview: optionalExpectedFile(area.outputs.qgisPreview), + }, + }, + { + stage: "reimport", + expected: hasReimportManifest, + inputs: { + ...(configPath ? { config: configPath } : {}), + derivedConfig, + gpkg: area.outputs.gpkg, + }, + outputs: { + geojsonDir: area.outputs.geojsonDir, + ...sceneGeojsonFiles(area), + }, + }, + { + stage: "blender", + expected: fs.existsSync(area.outputs.blend), + inputs: { + ...(configPath ? { config: configPath } : {}), + osm: area.input, + geojsonDir: area.outputs.geojsonDir, + ...sceneGeojsonFiles(area), + }, + outputs: { + blend: area.outputs.blend, + render: area.outputs.render, + }, + }, { stage: "cesium", expected: fs.existsSync(area.outputs.glb), @@ -307,7 +363,23 @@ function stageManifestStatus(area) { outputs: { glb: area.outputs.glb, metadata: area.outputs.metadata, + }, + }, + { + stage: "preview", + expected: fs.existsSync(area.outputs.cesiumPreview), + inputs: { + ...(configPath ? { config: configPath } : {}), + osm: area.input, + glb: area.outputs.glb, + metadata: area.outputs.metadata, + previewCss: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.css"), + previewJs: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.js"), + }, + outputs: { cesiumPreview: area.outputs.cesiumPreview, + vehicleRoute: area.outputs.vehicleRoute, + vehicleModel: area.outputs.vehicleModel, }, }, { @@ -375,12 +447,19 @@ function stageManifestStatus(area) { function manifestFileIssues(records, expectedFiles, label) { const issues = []; - for (const [key, file] of Object.entries(expectedFiles)) { + for (const [key, expected] of Object.entries(expectedFiles)) { + const { file, required } = normalizeExpectedFile(expected); const recorded = records[key]; + if (!required && !recorded && !fs.existsSync(file)) { + continue; + } if (!recorded) { issues.push(`${label} ${key} not recorded`); continue; } + if (!required && recorded === null && !fs.existsSync(file)) { + continue; + } if (!fs.existsSync(file)) { issues.push(`${label} ${key} file missing`); continue; @@ -395,6 +474,33 @@ function manifestFileIssues(records, expectedFiles, label) { return issues; } +function normalizeExpectedFile(expected) { + if (expected && typeof expected === "object" && expected.path) { + return { + file: expected.path, + required: expected.required !== false, + }; + } + return { + file: expected, + required: true, + }; +} + +function optionalExpectedFile(file) { + return { path: file, required: false }; +} + +function sceneGeojsonFiles(area) { + const files = {}; + for (const layer of SCENE_LAYERS) { + files[layer.id] = path.join(area.outputs.geojsonDir, layerFile(layer)); + } + files[SCENE_FILE] = path.join(area.outputs.geojsonDir, SCENE_FILE); + files[SCENE_STYLE_FILE] = path.join(area.outputs.geojsonDir, SCENE_STYLE_FILE); + return files; +} + function collectWarnings(area, osm, artifacts, manifests, glb, metadata) { const warnings = []; if (!osm.bounds) warnings.push("OSM has no valid ; scene extent may be wrong.");