Add full stage manifests

This commit is contained in:
2026-08-04 11:37:20 +08:00
parent 0b7e69770c
commit 5874cf9379
9 changed files with 516 additions and 21 deletions

View File

@@ -300,9 +300,10 @@ const gate = classifyAreaQuality(result);
### 1. Scope / Trigger ### 1. Scope / Trigger
Stage manifest 是构建阶段完成后的机器可读产物契约。第一版覆盖 GLB 相关阶段 Stage manifest 是构建阶段完成后的机器可读产物契约。它覆盖完整区域链
`cesium``compress`。它用于诊断产物是否存在、是否 stale、体量是否超预算以及后续 `intermediates``reimport``blender``cesium``preview``compress`
`check:area` / 增量构建判断。 它用于诊断产物是否存在、是否 stale、体量是否超预算以及后续 `check:area` /
增量构建判断。
### 2. Signatures ### 2. Signatures
@@ -315,7 +316,11 @@ Manifest 路径固定:
当前会写: 当前会写:
```text ```text
<areaDir>/_pipeline/stages/intermediates.manifest.json
<areaDir>/_pipeline/stages/reimport.manifest.json
<areaDir>/_pipeline/stages/blender.manifest.json
<areaDir>/_pipeline/stages/cesium.manifest.json <areaDir>/_pipeline/stages/cesium.manifest.json
<areaDir>/_pipeline/stages/preview.manifest.json
<areaDir>/_pipeline/stages/compress.manifest.json <areaDir>/_pipeline/stages/compress.manifest.json
``` ```
@@ -357,18 +362,65 @@ File records use this shape:
} }
``` ```
`intermediates` manifest:
- `inputs.config`
- `inputs.osm`
- `outputs.derivedConfig`
- `outputs.geojsonDir`
- `outputs.<nine scene layers>``outputs.osm2streets_scene.geojson`
`outputs.osm2streets_scene_style.json`
- `outputs.gpkg``outputs.qgisProject`、可选 `outputs.qgisPreview`
- `summary.geojson`:各 layer / merged scene 的 feature count
`reimport` manifest:
- `inputs.config`
- `inputs.derivedConfig`
- `inputs.gpkg`
- `outputs.geojsonDir`
- `outputs.<nine scene layers>`、merged scene/style
- `summary.geojson`
`intermediates``reimport` 是同一批 GeoJSON 的互斥 ownership
成功运行 `intermediates` 删除旧 `reimport.manifest.json`;成功运行 `reimport` 删除旧
`intermediates.manifest.json`。诊断只要求当前 ownership 路径的 manifest不要求两者同时存在。
`blender` manifest:
- `inputs.config`
- `inputs.osm`
- `inputs.geojsonDir``inputs.<nine scene layers>`、merged scene/style
- `outputs.blend`
- `outputs.render`
- `summary.geojson``summary.blendBytes``summary.renderBytes`
`cesium` manifest: `cesium` manifest:
- `inputs.blend` - `inputs.blend`
- `outputs.glb` - `outputs.glb`
- `outputs.metadata` - `outputs.metadata`
- `outputs.cesiumPreview`
- `outputs.vehicleRoute`
- `outputs.vehicleModel`
- `summary.glb.fileBytes` - `summary.glb.fileBytes`
- `summary.glb.counts` - `summary.glb.counts`
- `summary.glb.extensionsUsed` - `summary.glb.extensionsUsed`
`cesium` 会调用 preview 生成函数,但 preview HTML / route / vehicle model 的 freshness
所有权属于独立 `preview` manifest。否则单跑 `--stages preview` 会把 Cesium manifest
错误判 stale。
`preview` manifest:
- `inputs.config`
- `inputs.osm`
- `inputs.glb`
- `inputs.metadata`
- `inputs.previewCss`
- `inputs.previewJs`
- `outputs.cesiumPreview`
- `outputs.vehicleRoute`
- `outputs.vehicleModel`
- `summary.glbName``summary.metadataName``summary.routeSegments`
`compress` manifest: `compress` manifest:
- `inputs.glb` - `inputs.glb`
@@ -395,12 +447,19 @@ Manifest files are written atomically via `*.tmp` then `renameSync`.
| Manifest JSON 无法解析 | 诊断报告 invalid warning | | Manifest JSON 无法解析 | 诊断报告 invalid warning |
| Manifest 记录的 input / output sha/bytes 与当前文件不同 | 诊断报告 stale warning | | Manifest 记录的 input / output sha/bytes 与当前文件不同 | 诊断报告 stale warning |
| GLB size / nodes / images 超保守预算 | manifest `warnings[]` 记录,诊断继续独立报警 | | GLB size / nodes / images 超保守预算 | manifest `warnings[]` 记录,诊断继续独立报警 |
| `intermediates` 后留有旧 reimport manifest | 成功后删除旧 reimport manifest |
| `reimport` 后留有旧 intermediates manifest | 成功后删除旧 intermediates manifest |
### 5. Good/Base/Bad Cases ### 5. Good/Base/Bad Cases
- Good: `--stages cesium` 成功后写 `cesium.manifest.json`,诊断显示 `ok cesium` - Good: `--stages intermediates` 成功后写 `intermediates.manifest.json`,诊断显示
`ok intermediates`
- Good: `--stages blender` 成功后写 `blender.manifest.json`,诊断显示 `ok blender`
- Good: `--stages cesium` 成功后写 `cesium.manifest.json``preview.manifest.json`
- Good: `--stages preview` 只更新 preview manifest不让 cesium manifest stale。
- Good: `--stages compress` 成功后写 `compress.manifest.json`summary 记录压缩比和节省字节。 - Good: `--stages compress` 成功后写 `compress.manifest.json`summary 记录压缩比和节省字节。
- Base: 旧产物没有 manifest诊断显示 expected manifest missing提示重跑对应阶段。 - Base: 旧产物没有当前 ownership 路径的 manifest诊断显示 expected manifest missing
提示重跑对应阶段。
- Bad: 手工编辑 GLB 后不重跑阶段,诊断显示 manifest stale。 - Bad: 手工编辑 GLB 后不重跑阶段,诊断显示 manifest stale。
### 6. Tests Required ### 6. Tests Required
@@ -408,10 +467,15 @@ Manifest files are written atomically via `*.tmp` then `renameSync`.
- `node --check scripts/lib/stage-manifest.js` - `node --check scripts/lib/stage-manifest.js`
- `node --check scripts/build-area.js` - `node --check scripts/build-area.js`
- `node --check scripts/diagnose-area.js` - `node --check scripts/diagnose-area.js`
- `node --check scripts/check-area.js`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages intermediates`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages blender`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages cesium` - `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages cesium`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages preview`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress` - `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress`
- `npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.json` - `npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.json`
- JSON parse both generated manifests and assert `manifestVersion`, `stage`, `inputs`, - `npm run check:area -- --config config/areas/nantaizi-lake-innovation-valley.json`
- JSON parse generated manifests and assert `manifestVersion`, `stage`, `inputs`,
`outputs`, `summary`, and `warnings` fields exist. `outputs`, `summary`, and `warnings` fields exist.
### 7. Wrong vs Correct ### 7. Wrong vs Correct

View File

@@ -38,14 +38,17 @@ config/areas/<id>.json
│ → osm2streets_scene.geojson + _scene_style.json │ → osm2streets_scene.geojson + _scene_style.json
│ → ogr2ogr 导入 <id>.gpkg │ → ogr2ogr 导入 <id>.gpkg
│ → QGIS 生成 .qgz + -preview.png │ → QGIS 生成 .qgz + -preview.png
│ → _pipeline/stages/intermediates.manifest.json
├─[reimport]──────▶ reimport-gpkg.js (反向,与 intermediates 互斥) ├─[reimport]──────▶ reimport-gpkg.js (反向,与 intermediates 互斥)
│ ogr2ogr 从 .gpkg 导出 → 校验 → 覆写 *.geojson │ ogr2ogr 从 .gpkg 导出 → 校验 → 覆写 *.geojson
│ → 重建 scene.geojson + scene_style.json │ → 重建 scene.geojson + scene_style.json
│ → _pipeline/stages/reimport.manifest.json
├─[blender]───────▶ Blender + blender/generate_scene.py ├─[blender]───────▶ Blender + blender/generate_scene.py
│ 读 .osm + osm2streets_web_out/ │ 读 .osm + osm2streets_web_out/
│ → <id>.blend + <id>.png │ → <id>.blend + <id>.png
│ → _pipeline/stages/blender.manifest.json
├─[cesium]────────▶ Blender + blender/export_cesium.py ├─[cesium]────────▶ Blender + blender/export_cesium.py
│ 读 .blend → <id>.glb + <id>.json │ 读 .blend → <id>.glb + <id>.json
@@ -55,6 +58,7 @@ config/areas/<id>.json
├─[preview]───────▶ 生成 <id>-cesium-preview.html ├─[preview]───────▶ 生成 <id>-cesium-preview.html
│ + 拷贝 lib/cesium-preview.{js,css} │ + 拷贝 lib/cesium-preview.{js,css}
│ + 车辆巡航路线与模型 │ + 车辆巡航路线与模型
│ → _pipeline/stages/preview.manifest.json
└─[compress]──────▶ 生成并列压缩 GLB / metadata / preview └─[compress]──────▶ 生成并列压缩 GLB / metadata / preview
→ _pipeline/stages/compress.manifest.json → _pipeline/stages/compress.manifest.json
@@ -83,10 +87,10 @@ config/areas/<id>.json
| 文件 | 行数 | 职责 | | 文件 | 行数 | 职责 |
|---|---|---| |---|---|---|
| `build-area.js` | 815 | 主入口区域配置读取、阶段调度、Cesium 预览页、车辆巡航和 stage manifest 写入 | | `build-area.js` | 961 | 主入口区域配置读取、阶段调度、Cesium 预览页、车辆巡航和 stage manifest 写入 |
| `diagnose-area.js` | 36 | 快速诊断入口:调用共享 area diagnostics 并打印完整报告 | | `diagnose-area.js` | 36 | 快速诊断入口:调用共享 area diagnostics 并打印完整报告 |
| `check-area.js` | 74 | 区域质量门入口:调用共享 area diagnostics输出 PASS/FAIL 并设置退出码 | | `check-area.js` | 74 | 区域质量门入口:调用共享 area diagnostics输出 PASS/FAIL 并设置退出码 |
| `lib/area-diagnostics.js` | 660 | 共享区域诊断事实源OSM、产物、metadata、stage manifest、GLB digest 和质量门分类 | | `lib/area-diagnostics.js` | 776 | 共享区域诊断事实源OSM、产物、metadata、stage manifest、GLB digest 和质量门分类 |
| `lib/area-config.js` | 135 | 区域配置归一化与输出路径推导,供 build / diagnose 复用 | | `lib/area-config.js` | 135 | 区域配置归一化与输出路径推导,供 build / diagnose 复用 |
| `lib/stage-manifest.js` | 100 | stage manifest 路径、文件记录、GLB budget warning 和原子 JSON 写入 | | `lib/stage-manifest.js` | 100 | stage manifest 路径、文件记录、GLB budget warning 和原子 JSON 写入 |
| `build-osm2streets-qgis.js` | 1468 | intermediatesosm2streets 解析、图层拆分、人行道转角合成、GeoPackage 与 QGIS 工程生成 | | `build-osm2streets-qgis.js` | 1468 | intermediatesosm2streets 解析、图层拆分、人行道转角合成、GeoPackage 与 QGIS 工程生成 |

View File

@@ -0,0 +1,77 @@
# Design
## Scope
The existing manifest infrastructure already supports file records, atomic JSON
write, digest summaries, and diagnostics-side freshness checks. This task
extends that same contract to the earlier and read-only/patched stages without
adding a new manifest format.
## Stage Boundaries
`intermediates`
- Written by `build-area.js` after `build-osm2streets-qgis.js` succeeds.
- Inputs: area config, source OSM XML.
- Outputs: derived config, GeoJSON directory, all scene-layer GeoJSON files,
merged scene/style files, GeoPackage, QGIS project, QGIS preview if present.
- Summary: GeoJSON feature counts by file.
`reimport`
- Written by `build-area.js` after `reimport-gpkg.js` succeeds.
- Inputs: area config, GeoPackage.
- Outputs: GeoJSON directory, all scene-layer GeoJSON files, merged scene/style
files.
- Summary: GeoJSON feature counts by file.
`blender`
- Written by `build-area.js` after Blender scene generation succeeds.
- Inputs: area config, source OSM XML, GeoJSON directory, all scene-layer
GeoJSON files, merged scene/style files.
- Outputs: `.blend`, render PNG.
- Summary: parsed `SCENE_DONE` payload when available, plus blend/render bytes.
`preview`
- Written by `build-area.js` after preview generation succeeds. Since `cesium`
automatically calls preview generation, this manifest should also be written
during `--stages cesium`.
- Inputs: area config, source OSM XML, GLB, metadata, preview runtime JS/CSS.
- Outputs: Cesium preview HTML, vehicle route JSON, vehicle model glTF.
- Summary: vehicle route count and asset filenames.
## Diagnostics
`scripts/lib/area-diagnostics.js` will extend `stageManifestStatus()` from two
hard-coded GLB stages to the full ordered stage list. Expected status should
derive from stage outputs:
- `intermediates`: expected when GeoJSON dir, GeoPackage, or QGIS project exists.
- `reimport`: expected when GeoJSON dir exists and `reimport.manifest.json`
already exists; it is an alternate path and should not be required for normal
intermediates builds.
- `blender`: expected when blend exists.
- `cesium`: expected when baseline GLB exists.
- `preview`: expected when Cesium preview exists.
- `compress`: expected when compressed GLB exists.
Only expected manifests are fatal in `check:area`. Optional absent manifests
remain ignored.
## Compatibility
- Existing `cesium.manifest.json` and `compress.manifest.json` remain valid.
- Existing outputs without new manifests will show missing expected manifests
until their stages are rerun.
- No stage should overwrite its primary outputs differently just to write a
manifest.
## Tradeoffs
- Manifest writing lives in `build-area.js` for this task because it already
owns normalized area paths and stage timing. Moving preview generation and
vehicle assets out of `build-area.js` is a later refactor.
- Reimport remains optional/alternate to avoid making normal `check:area` fail
for users who never use the reimport workflow.

View File

@@ -0,0 +1,30 @@
# Implementation Plan
1. Read current manifest and stage code.
2. Add reusable helpers in `build-area.js` for:
- file record collection
- GeoJSON feature-count summaries
- manifest duration/timestamps
- writing `intermediates`, `reimport`, `blender`, and `preview` manifests
3. Wire manifest writes after successful stage functions only.
4. Extend `area-diagnostics.js` manifest expected/input/output definitions to
cover all stages.
5. Update README, changelog, and `.trellis/spec/pipeline/*`.
6. Validate with:
- `node --check scripts/build-area.js`
- `node --check scripts/lib/area-diagnostics.js`
- `node --check scripts/check-area.js`
- `node --check scripts/diagnose-area.js`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages preview`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages blender`
- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages intermediates`
- `npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.json`
- `npm run check:area -- --config config/areas/nantaizi-lake-innovation-valley.json`
- controlled stale/missing manifest failure
- `git diff --check`
## Rollback
- Remove added manifest write calls and diagnostics stage definitions.
- Delete generated `_pipeline/stages/{intermediates,reimport,blender,preview}.manifest.json`
from outputs if needed; outputs are ignored and not committed.

View File

@@ -0,0 +1,50 @@
# Add full stage manifests
## Goal
Extend the stage manifest contract from GLB-only stages to the full area
pipeline so `diagnose:area` and `check:area` can reason about stage freshness
across intermediates, reimport, blender, preview, cesium, and compress.
## Requirements
- Write stage manifests for:
- `intermediates`
- `reimport`
- `blender`
- `preview`
- Preserve existing `cesium` and `compress` manifest behavior.
- Do not change default build outputs except for adding/updating manifest files
under `<areaDir>/_pipeline/stages/`.
- Stage manifests must record enough inputs and outputs to detect stale files
using the existing bytes/sha freshness logic.
- `diagnose:area` must display these additional manifests.
- `check:area` must fail when an expected full-pipeline manifest is missing,
invalid, or stale.
- `reimport` and `intermediates` remain mutually exclusive.
- Missing optional `compress` artifacts still must not make `check:area` fail.
- README, changelog, and Trellis pipeline spec must document the expanded
manifest coverage.
## Acceptance Criteria
- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages preview`
writes a fresh `preview.manifest.json`.
- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages blender`
writes a fresh `blender.manifest.json`.
- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages intermediates`
writes a fresh `intermediates.manifest.json`.
- [x] `diagnose:area` shows `intermediates`, `blender`, `cesium`, `preview`, and
optional `compress` manifest status.
- [x] `check:area` passes for current nantaizi outputs after the required
manifests are generated.
- [x] A controlled stale/missing manifest case makes `check:area` exit non-zero.
- [x] Syntax checks pass for changed Node scripts.
- [x] README, changelog, and pipeline spec describe full stage manifest coverage.
## Notes
- Validate only `nantaizi-lake-innovation-valley` unless explicitly asked.
- Ignore `hanyang-block`.
- This task is about pipeline contracts and diagnostics, not lane-arrow visual
behavior.

View File

@@ -76,16 +76,26 @@ npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.j
现有产物状态,以及 GLB 的 size / nodes / meshes / materials / images / extensions。 现有产物状态,以及 GLB 的 size / nodes / meshes / materials / images / extensions。
缺少已期望的基线产物、异常 building relation、GLB 超过保守预算等会进入 `Warnings` 缺少已期望的基线产物、异常 building relation、GLB 超过保守预算等会进入 `Warnings`
`cesium``compress` 阶段成功后会写机器可读的阶段 manifest 每个成功的区域构建阶段都会写机器可读的 stage manifest
```text ```text
outputs/<area-id>/_pipeline/stages/intermediates.manifest.json
outputs/<area-id>/_pipeline/stages/reimport.manifest.json
outputs/<area-id>/_pipeline/stages/blender.manifest.json
outputs/<area-id>/_pipeline/stages/cesium.manifest.json outputs/<area-id>/_pipeline/stages/cesium.manifest.json
outputs/<area-id>/_pipeline/stages/preview.manifest.json
outputs/<area-id>/_pipeline/stages/compress.manifest.json outputs/<area-id>/_pipeline/stages/compress.manifest.json
``` ```
manifest 记录阶段输入/输出文件的 bytes、mtime、sha256、耗时、GLB 结构摘要和 warning。 `intermediates``reimport` 是同一批 GeoJSON 的两种所有权路径:前者从 OSM 重建,
`diagnose:area` 会读取这些 manifest如果已有 GLB 但 manifest 缺失,或者 manifest 后者从手工编辑的 GeoPackage 回导。成功运行其中一个会清掉另一个的 manifest避免旧路径
记录的输入/输出 sha/bytes 和当前文件不一致,会在 `Stage manifests``Warnings` 里标出来 造成假 stale
manifest 记录阶段输入/输出文件的 bytes、mtime、sha256、耗时和结构摘要前段记录 OSM /
GeoJSON feature countsBlender 记录 `.blend` / renderCesium/压缩记录 GLB digest
preview 记录 GLB、metadata、车辆路线和 runtime 文件。`diagnose:area` 会读取这些
manifest缺失或当前输入/输出 sha/bytes 不一致会在 `Stage manifests``Warnings`
里标出来。
## 区域质量门 ## 区域质量门

View File

@@ -2,6 +2,14 @@
## 2026-08-04 ## 2026-08-04
- 扩展 stage manifest 到完整区域链:`intermediates``reimport``blender`
`preview` 现在和既有 `cesium` / `compress` 一样,在成功后写
`_pipeline/stages/<stage>.manifest.json`。前段 manifest 记录 config / OSM /
GeoJSON layer 文件与 feature countBlender 记录 OSM / GeoJSON 输入和 `.blend` /
renderpreview 记录 GLB / metadata / OSM / runtime 以及 HTML / route / vehicle
model。`intermediates``reimport` 采用互斥 ownership成功运行一方会删除另一方的
manifestCesium 和 preview 也拆开 freshness ownership单跑 preview 不会把
Cesium manifest 判 stale。`diagnose:area` / `check:area` 现在覆盖完整阶段链。
- 新增区域质量门入口:`npm run check:area -- --config config/areas/<area>.json` - 新增区域质量门入口:`npm run check:area -- --config config/areas/<area>.json`
它复用 `scripts/lib/area-diagnostics.js` 的 OSM、产物、metadata、stage manifest 和 它复用 `scripts/lib/area-diagnostics.js` 的 OSM、产物、metadata、stage manifest 和
GLB digest 检查,只读已有文件,不调用 QGIS / Blender / Cesium / 压缩 / 重建;报告 GLB digest 检查,只读已有文件,不调用 QGIS / Blender / Cesium / 压缩 / 重建;报告

View File

@@ -4,12 +4,19 @@ const fs = require("fs");
const path = require("path"); const path = require("path");
const { spawnSync } = require("child_process"); const { spawnSync } = require("child_process");
const { readAreaConfig } = require("./lib/area-config"); const { readAreaConfig } = require("./lib/area-config");
const {
SCENE_LAYERS,
SCENE_FILE,
SCENE_STYLE_FILE,
layerFile,
} = require("./lib/scene-layers");
const { digest: glbDigest } = require("./glb-digest"); const { digest: glbDigest } = require("./glb-digest");
const { const {
fileRecord, fileRecord,
glbBudgetWarnings, glbBudgetWarnings,
glbSummary, glbSummary,
optionalFileRecord, optionalFileRecord,
stageManifestPath,
writeStageManifest, writeStageManifest,
} = require("./lib/stage-manifest"); } = require("./lib/stage-manifest");
@@ -154,6 +161,8 @@ function writeDerivedConfig(area) {
} }
function buildIntermediates(area) { function buildIntermediates(area) {
const started = Date.now();
const startedAt = new Date(started).toISOString();
const derivedConfigPath = writeDerivedConfig(area); const derivedConfigPath = writeDerivedConfig(area);
console.log("Stage: intermediates (osm2streets GeoJSON + QGIS)"); console.log("Stage: intermediates (osm2streets GeoJSON + QGIS)");
@@ -162,9 +171,37 @@ function buildIntermediates(area) {
"--config", "--config",
derivedConfigPath, derivedConfigPath,
], "intermediates"); ], "intermediates");
fs.rmSync(stageManifestPath(area, "reimport"), { force: true });
const finished = Date.now();
writeStageManifest(area, {
stage: "intermediates",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
},
outputs: {
derivedConfig: fileRecord(derivedConfigPath),
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
gpkg: fileRecord(area.outputs.gpkg),
qgisProject: fileRecord(area.outputs.qgisProject),
qgisPreview: optionalFileRecord(area.outputs.qgisPreview),
},
summary: {
geojson: geojsonFeatureCounts(area),
},
warnings: [],
});
} }
function reimportGpkg(area) { function reimportGpkg(area) {
const started = Date.now();
const startedAt = new Date(started).toISOString();
const derivedConfigPath = writeDerivedConfig(area); const derivedConfigPath = writeDerivedConfig(area);
console.log("Stage: reimport (GeoPackage -> GeoJSON)"); console.log("Stage: reimport (GeoPackage -> GeoJSON)");
@@ -173,6 +210,29 @@ function reimportGpkg(area) {
"--config", "--config",
derivedConfigPath, derivedConfigPath,
], "reimport"); ], "reimport");
fs.rmSync(stageManifestPath(area, "intermediates"), { force: true });
const finished = Date.now();
writeStageManifest(area, {
stage: "reimport",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
derivedConfig: fileRecord(derivedConfigPath),
gpkg: fileRecord(area.outputs.gpkg),
},
outputs: {
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
},
summary: {
geojson: geojsonFeatureCounts(area),
},
warnings: [],
});
} }
function buildBlenderScene(area) { function buildBlenderScene(area) {
@@ -203,7 +263,34 @@ function buildBlenderScene(area) {
} }
console.log("Stage: blender"); console.log("Stage: blender");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(blenderExecutable(area), blenderArgs, "blender"); runCommand(blenderExecutable(area), blenderArgs, "blender");
const finished = Date.now();
writeStageManifest(area, {
stage: "blender",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
geojsonDir: fileRecord(area.outputs.geojsonDir),
...sceneGeojsonRecords(area),
},
outputs: {
blend: fileRecord(area.outputs.blend),
render: fileRecord(area.outputs.render),
},
summary: {
geojson: geojsonFeatureCounts(area),
blendBytes: fileRecord(area.outputs.blend).bytes,
renderBytes: fileRecord(area.outputs.render).bytes,
},
warnings: [],
});
} }
function exportCesium(area) { function exportCesium(area) {
@@ -244,9 +331,6 @@ function exportCesium(area) {
outputs: { outputs: {
glb: fileRecord(area.outputs.glb), glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata), metadata: fileRecord(area.outputs.metadata),
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
vehicleRoute: optionalFileRecord(area.outputs.vehicleRoute),
vehicleModel: optionalFileRecord(area.outputs.vehicleModel),
}, },
summary: { summary: {
glb: glbSummary(digest), glb: glbSummary(digest),
@@ -354,6 +438,8 @@ function writeCesiumPreview(area) {
ensureFile(area.outputs.glb, "Cesium GLB"); ensureFile(area.outputs.glb, "Cesium GLB");
ensureFile(area.outputs.metadata, "Cesium metadata"); ensureFile(area.outputs.metadata, "Cesium metadata");
const htmlPath = area.outputs.cesiumPreview; const htmlPath = area.outputs.cesiumPreview;
const started = Date.now();
const startedAt = new Date(started).toISOString();
fs.mkdirSync(path.dirname(htmlPath), { recursive: true }); fs.mkdirSync(path.dirname(htmlPath), { recursive: true });
writeVehicleRoute(area); writeVehicleRoute(area);
writeVehicleModel(area); writeVehicleModel(area);
@@ -364,6 +450,30 @@ function writeCesiumPreview(area) {
const vehicleModelName = path.basename(area.outputs.vehicleModel); const vehicleModelName = path.basename(area.outputs.vehicleModel);
fs.writeFileSync(htmlPath, cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, area.id)); fs.writeFileSync(htmlPath, cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, area.id));
console.log(`Cesium preview: ${htmlPath}`); console.log(`Cesium preview: ${htmlPath}`);
const finished = Date.now();
writeStageManifest(area, {
stage: "preview",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
config: fileRecord(configPath),
osm: fileRecord(area.input),
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
previewCss: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.css")),
previewJs: fileRecord(path.join(repoRoot, "scripts", "lib", "cesium-preview.js")),
},
outputs: {
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
vehicleRoute: fileRecord(area.outputs.vehicleRoute),
vehicleModel: fileRecord(area.outputs.vehicleModel),
},
summary: previewSummary(area),
warnings: [],
});
} }
function writeCesiumPreviewSupportFiles(outDir) { function writeCesiumPreviewSupportFiles(outDir) {
@@ -813,3 +923,39 @@ function escapeScriptJson(value) {
.replaceAll("\u2028", "\\u2028") .replaceAll("\u2028", "\\u2028")
.replaceAll("\u2029", "\\u2029"); .replaceAll("\u2029", "\\u2029");
} }
function sceneGeojsonRecords(area) {
const records = {};
for (const layer of SCENE_LAYERS) {
records[layer.id] = fileRecord(path.join(area.outputs.geojsonDir, layerFile(layer)));
}
records[SCENE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_FILE));
records[SCENE_STYLE_FILE] = fileRecord(path.join(area.outputs.geojsonDir, SCENE_STYLE_FILE));
return records;
}
function geojsonFeatureCounts(area) {
const out = {};
for (const layer of SCENE_LAYERS) {
const file = path.join(area.outputs.geojsonDir, layerFile(layer));
out[layer.id] = featureCount(file);
}
out[SCENE_FILE] = featureCount(path.join(area.outputs.geojsonDir, SCENE_FILE));
return out;
}
function featureCount(file) {
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
return Array.isArray(parsed.features) ? parsed.features.length : null;
}
function previewSummary(area) {
const route = JSON.parse(fs.readFileSync(area.outputs.vehicleRoute, "utf8"));
return {
glbName: path.basename(area.outputs.glb),
metadataName: path.basename(area.outputs.metadata),
routeName: path.basename(area.outputs.vehicleRoute),
vehicleModelName: path.basename(area.outputs.vehicleModel),
routeSegments: Array.isArray(route.segments) ? route.segments.length : null,
};
}

View File

@@ -3,6 +3,12 @@
const fs = require("fs"); const fs = require("fs");
const path = require("path"); const path = require("path");
const { readAreaConfig } = require("./area-config"); const { readAreaConfig } = require("./area-config");
const {
SCENE_LAYERS,
SCENE_FILE,
SCENE_STYLE_FILE,
layerFile,
} = require("./scene-layers");
const { digest: glbDigest } = require("../glb-digest"); const { digest: glbDigest } = require("../glb-digest");
const { BUDGETS, fileRecord, readStageManifest, stageManifestPath } = require("./stage-manifest"); const { BUDGETS, fileRecord, readStageManifest, stageManifestPath } = require("./stage-manifest");
@@ -16,7 +22,7 @@ function analyzeArea(configPath, options = {}) {
const area = readAreaConfig(resolvedConfig, { repoRoot }); const area = readAreaConfig(resolvedConfig, { repoRoot });
const osm = parseOsm(fs.readFileSync(area.input, "utf8")); const osm = parseOsm(fs.readFileSync(area.input, "utf8"));
const artifacts = artifactStatus(area); const artifacts = artifactStatus(area);
const manifests = stageManifestStatus(area); const manifests = stageManifestStatus(area, resolvedConfig);
const metadataWarnings = []; const metadataWarnings = [];
const metadata = metadataSummary(area.outputs.metadata, metadataWarnings); const metadata = metadataSummary(area.outputs.metadata, metadataWarnings);
const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null; const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null;
@@ -296,8 +302,58 @@ function metadataSummary(file, warnings) {
} }
} }
function stageManifestStatus(area) { function stageManifestStatus(area, configPath = null) {
const reimportManifest = stageManifestPath(area, "reimport");
const hasReimportManifest = fs.existsSync(reimportManifest);
const derivedConfig = path.join(area.outputs.pipelineDir, "osm2streets-qgis.config.json");
const stages = [ const stages = [
{
stage: "intermediates",
expected: (
fs.existsSync(area.outputs.geojsonDir) ||
fs.existsSync(area.outputs.gpkg) ||
fs.existsSync(area.outputs.qgisProject)
) && !hasReimportManifest,
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
},
outputs: {
derivedConfig,
geojsonDir: area.outputs.geojsonDir,
...sceneGeojsonFiles(area),
gpkg: area.outputs.gpkg,
qgisProject: area.outputs.qgisProject,
qgisPreview: optionalExpectedFile(area.outputs.qgisPreview),
},
},
{
stage: "reimport",
expected: hasReimportManifest,
inputs: {
...(configPath ? { config: configPath } : {}),
derivedConfig,
gpkg: area.outputs.gpkg,
},
outputs: {
geojsonDir: area.outputs.geojsonDir,
...sceneGeojsonFiles(area),
},
},
{
stage: "blender",
expected: fs.existsSync(area.outputs.blend),
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
geojsonDir: area.outputs.geojsonDir,
...sceneGeojsonFiles(area),
},
outputs: {
blend: area.outputs.blend,
render: area.outputs.render,
},
},
{ {
stage: "cesium", stage: "cesium",
expected: fs.existsSync(area.outputs.glb), expected: fs.existsSync(area.outputs.glb),
@@ -307,7 +363,23 @@ function stageManifestStatus(area) {
outputs: { outputs: {
glb: area.outputs.glb, glb: area.outputs.glb,
metadata: area.outputs.metadata, metadata: area.outputs.metadata,
},
},
{
stage: "preview",
expected: fs.existsSync(area.outputs.cesiumPreview),
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
glb: area.outputs.glb,
metadata: area.outputs.metadata,
previewCss: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.css"),
previewJs: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.js"),
},
outputs: {
cesiumPreview: area.outputs.cesiumPreview, cesiumPreview: area.outputs.cesiumPreview,
vehicleRoute: area.outputs.vehicleRoute,
vehicleModel: area.outputs.vehicleModel,
}, },
}, },
{ {
@@ -375,12 +447,19 @@ function stageManifestStatus(area) {
function manifestFileIssues(records, expectedFiles, label) { function manifestFileIssues(records, expectedFiles, label) {
const issues = []; const issues = [];
for (const [key, file] of Object.entries(expectedFiles)) { for (const [key, expected] of Object.entries(expectedFiles)) {
const { file, required } = normalizeExpectedFile(expected);
const recorded = records[key]; const recorded = records[key];
if (!required && !recorded && !fs.existsSync(file)) {
continue;
}
if (!recorded) { if (!recorded) {
issues.push(`${label} ${key} not recorded`); issues.push(`${label} ${key} not recorded`);
continue; continue;
} }
if (!required && recorded === null && !fs.existsSync(file)) {
continue;
}
if (!fs.existsSync(file)) { if (!fs.existsSync(file)) {
issues.push(`${label} ${key} file missing`); issues.push(`${label} ${key} file missing`);
continue; continue;
@@ -395,6 +474,33 @@ function manifestFileIssues(records, expectedFiles, label) {
return issues; return issues;
} }
function normalizeExpectedFile(expected) {
if (expected && typeof expected === "object" && expected.path) {
return {
file: expected.path,
required: expected.required !== false,
};
}
return {
file: expected,
required: true,
};
}
function optionalExpectedFile(file) {
return { path: file, required: false };
}
function sceneGeojsonFiles(area) {
const files = {};
for (const layer of SCENE_LAYERS) {
files[layer.id] = path.join(area.outputs.geojsonDir, layerFile(layer));
}
files[SCENE_FILE] = path.join(area.outputs.geojsonDir, SCENE_FILE);
files[SCENE_STYLE_FILE] = path.join(area.outputs.geojsonDir, SCENE_STYLE_FILE);
return files;
}
function collectWarnings(area, osm, artifacts, manifests, glb, metadata) { function collectWarnings(area, osm, artifacts, manifests, glb, metadata) {
const warnings = []; const warnings = [];
if (!osm.bounds) warnings.push("OSM has no valid <bounds>; scene extent may be wrong."); if (!osm.bounds) warnings.push("OSM has no valid <bounds>; scene extent may be wrong.");