From 2832383f6e07679aed778e6a14d2521b0bb0f4ba Mon Sep 17 00:00:00 2001 From: que01 Date: Tue, 4 Aug 2026 09:45:51 +0800 Subject: [PATCH] Add GLB stage manifests --- .trellis/spec/config/index.md | 13 +- .../spec/guides/code-reuse-thinking-guide.md | 2 +- .../spec/guides/cross-layer-thinking-guide.md | 4 +- .trellis/spec/guides/index.md | 2 +- .trellis/spec/index.md | 3 +- .trellis/spec/pipeline/cli-and-stages.md | 155 +++++++++++++++++- .trellis/spec/pipeline/index.md | 20 ++- .../08-04-add-stage-manifests/check.jsonl | 1 + .../08-04-add-stage-manifests/implement.jsonl | 1 + .../tasks/08-04-add-stage-manifests/prd.md | 41 +++++ .../tasks/08-04-add-stage-manifests/task.json | 26 +++ README.md | 11 ++ docs/changelog.md | 7 + scripts/build-area.js | 70 ++++++++ scripts/diagnose-area.js | 146 ++++++++++++++++- scripts/lib/area-config.js | 4 +- scripts/lib/stage-manifest.js | 100 +++++++++++ 17 files changed, 579 insertions(+), 27 deletions(-) create mode 100644 .trellis/tasks/08-04-add-stage-manifests/check.jsonl create mode 100644 .trellis/tasks/08-04-add-stage-manifests/implement.jsonl create mode 100644 .trellis/tasks/08-04-add-stage-manifests/prd.md create mode 100644 .trellis/tasks/08-04-add-stage-manifests/task.json create mode 100644 scripts/lib/stage-manifest.js diff --git a/.trellis/spec/config/index.md b/.trellis/spec/config/index.md index 74b42ae..445799c 100644 --- a/.trellis/spec/config/index.md +++ b/.trellis/spec/config/index.md @@ -12,7 +12,8 @@ ``` config/areas/.json ← 你写的 - │ build-area.js: normalizeAreaConfig() 补默认值 + 推导输出路径 + │ scripts/lib/area-config.js: normalizeAreaConfig() + │ 补默认值 + 推导输出路径 ▼ /_pipeline/osm2streets-qgis.config.json ← 生成的，不要手改 │ @@ -109,7 +110,7 @@ cp config/examples/template.json config/areas/my-area.json } ``` -⚠️ **给了就整体替换，不做逐字段合并**（`build-area.js:132`：`raw.osm2streets || {...}`）。 +⚠️ **给了就整体替换，不做逐字段合并**（`raw.osm2streets || {...}`）。 只想改一个开关也必须把五个字段全写上，否则其余四个会退到 osm2streets 自己的默认值。 ### `blender` @@ -146,10 +147,10 @@ cp config/examples/template.json config/areas/my-area.json } ``` -可覆盖的键（`build-area.js:87-102`）：`areaDir`、`fileStem`、`geojsonDir`、`gpkg`、 +可覆盖的键（`scripts/lib/area-config.js`）：`areaDir`、`fileStem`、`geojsonDir`、`gpkg`、 `qgisProject`、`qgisPreview`、`blend`、`render`、`glb`、`metadata`、`cesiumPreview`、 `compressedFileStem`、`compressedGlb`、`compressedMetadata`、`compressedCesiumPreview`、 -`vehicleRoute`、`vehicleModel`、`pipelineDir`。 +`vehicleRoute`、`vehicleModel`、`pipelineDir`、`stageManifestDir`。 **优先改 `fileStem` 或 `areaDir`**——它们能一次性影响全部派生路径。逐个覆盖容易漏。 @@ -157,11 +158,11 @@ cp config/examples/template.json config/areas/my-area.json ## 加一个配置字段 -1. `normalizeAreaConfig`（`build-area.js:74`）里加进对应的分组，**用 `??` 不用 `||`** +1. `normalizeAreaConfig`（`scripts/lib/area-config.js`）里加进对应的分组，**用 `??` 不用 `||`** （`false` / `0` 可能是合法值） 2. 只写两级 fallback：`raw.?. ?? 默认值`。 **不要**制造新的顶层平铺别名——那三级写法是历史兼容，不是模式 -3. 若要传给低层脚本，加进 `writeDerivedConfig`（`:189`）的 `derivedConfig` 对象 +3. 若要传给低层脚本，加进 `writeDerivedConfig` 的 `derivedConfig` 对象 4. 若是数值，在消费侧加 `Number.isFinite` + 范围校验，**在任何副作用之前** 5. 更新 `config/examples/template.json` 6. 更新本文档的字段表 diff --git a/.trellis/spec/guides/code-reuse-thinking-guide.md b/.trellis/spec/guides/code-reuse-thinking-guide.md index befbe46..5e5cb32 100644 --- a/.trellis/spec/guides/code-reuse-thinking-guide.md +++ b/.trellis/spec/guides/code-reuse-thinking-guide.md @@ -118,7 +118,7 @@ Cesium 导出调色也属于同一个材质声明：新场景把 `catalog.MATERI - OSM XML → `osmassets/osm.py:parse_osm()` - 米制几何 → `osmassets/geom.py` - GeoJSON 场景合并 → `scene-layers.js:mergeScene(getCollection)` -- 区域配置 → `build-area.js:normalizeAreaConfig()` +- 区域配置 → `scripts/lib/area-config.js:normalizeAreaConfig()` 如果确实需要新解析器，把输入格式、容错语义和调用者写清楚，并给纯 Python 逻辑补测试。 diff --git a/.trellis/spec/guides/cross-layer-thinking-guide.md b/.trellis/spec/guides/cross-layer-thinking-guide.md index 774b5b4..4e674e7 100644 --- a/.trellis/spec/guides/cross-layer-thinking-guide.md +++ b/.trellis/spec/guides/cross-layer-thinking-guide.md @@ -11,6 +11,8 @@ ``` config/areas/*.json JSON 数据 ↓ ① +lib/area-config.js 区域配置归一化 + ↓ build-area.js Node（宿主机） ↓ ② 派生配置 JSON build-osm2streets-qgis.js Node + osm2streets WASM @@ -26,7 +28,7 @@ cesium-preview.js 浏览器 | # | 边界 | 常见问题 | |---|---|---| -| ① | 用户配置 → 归一化 | `??` vs `\|\|`、相对路径、字段整体替换 | +| ① | 用户配置 → 归一化 | `??` vs `\|\|`、相对路径、字段整体替换、绕开共享归一化 | | ② | 两层配置 | 低层脚本读错配置源 | | ③ | Node → 外部进程 | 环境变量缺失、退出码与信号、0 字节产物 | | ④ | 文件交换 | 图层集合/顺序漂移、精度丢失 | diff --git a/.trellis/spec/guides/index.md b/.trellis/spec/guides/index.md index 56a457a..bd85b5e 100644 --- a/.trellis/spec/guides/index.md +++ b/.trellis/spec/guides/index.md @@ -23,7 +23,7 @@ - [ ] 改 `blender/osmassets/catalog.py:28` 的 `ROAD_LAYERS` 或 `catalog.py:56` 的 `MATERIALS` - [ ] 改 `catalog.MATERIALS[*]["cesium"]`、`material["cesium_export"]` 或 `export_cesium.py` 的旧材质名回退表 -- [ ] 改 `build-area.js:74` 的 `normalizeAreaConfig()` 或 `config/examples/template.json` +- [ ] 改 `scripts/lib/area-config.js` 的 `normalizeAreaConfig()` 或 `config/examples/template.json` - [ ] 改任何 `execFileSync` / `spawnSync` 调起的脚本或参数 - [ ] 改 `SCENE_DONE` / `CESIUM_EXPORT_DONE` 的 stdout 标记 diff --git a/.trellis/spec/index.md b/.trellis/spec/index.md index 232a737..964d758 100644 --- a/.trellis/spec/index.md +++ b/.trellis/spec/index.md @@ -9,7 +9,8 @@ 本仓库不是前端应用，而是 **OSM → QGIS/Blender/Cesium 的资产生成管线**： -- `scripts/build-area.js:74` 的 `normalizeAreaConfig()` 归一化区域配置并调度阶段 +- `scripts/lib/area-config.js` 的 `normalizeAreaConfig()` 归一化区域配置和输出路径 +- `scripts/build-area.js` 调度阶段并写入 GLB 相关 stage manifest - `scripts/lib/scene-layers.js:15` 的 `SCENE_LAYERS` 是 osm2streets 九个 2D 图层的 JS 侧事实源 - `blender/osmassets/catalog.py:28` 的 `ROAD_LAYERS` 是 Blender 侧道路图层与材质顺序事实源 - `scripts/lib/cesium-preview.js:1` 是无构建步骤的浏览器预览 IIFE diff --git a/.trellis/spec/pipeline/cli-and-stages.md b/.trellis/spec/pipeline/cli-and-stages.md index 93b1cd3..c3da593 100644 --- a/.trellis/spec/pipeline/cli-and-stages.md +++ b/.trellis/spec/pipeline/cli-and-stages.md @@ -151,9 +151,10 @@ node scripts/diagnose-area.js [--config config/areas/.json] - 配置里的 OSM XML - 已存在的 `area.outputs.*` 产物 - 已存在的默认 GLB（通过 `scripts/glb-digest.js` 导出的 `digest()`） + - 已存在的 stage manifests（通过 `scripts/lib/stage-manifest.js`） - 输出为 text report，包含 OSM bounds、节点/way/relation 数量、building way、 building multipolygon relation、显式 `height` / `building:levels`、植被数量、 - 产物存在性、metadata 摘要、GLB size/counts/extensions 和 warnings。 + 产物存在性、stage manifest 状态、metadata 摘要、GLB size/counts/extensions 和 warnings。 - warnings 不改变退出码；配置缺失、输入 OSM 缺失、GLB 文件损坏这类无法继续读取的错误才 非零退出。 @@ -167,6 +168,8 @@ node scripts/diagnose-area.js [--config config/areas/.json] | OSM 缺 `` | warning，不中断 | | building multipolygon 缺 outer / unresolved way / open ring | warning，不中断 | | baseline 产物缺失 | warning，不中断 | +| expected stage manifest 缺失 | warning，不中断 | +| stage manifest inputs / outputs 与当前文件 sha/bytes 不一致 | stale warning，不中断 | | metadata JSON 损坏 | warning，不中断 | | GLB 存在但不是合法 GLB | 抛出 `glb-digest` 错误，非零 | | GLB 超过保守预算 | warning，不中断 | @@ -207,6 +210,147 @@ const glb = area.outputs.glb; --- +## Stage Manifest 契约 + +### 1. Scope / Trigger + +Stage manifest 是构建阶段完成后的机器可读产物契约。第一版覆盖 GLB 相关阶段： +`cesium` 和 `compress`。它用于诊断产物是否存在、是否 stale、体量是否超预算，以及后续 +`check:area` / 增量构建判断。 + +### 2. Signatures + +Manifest 路径固定： + +```text +/_pipeline/stages/.manifest.json +``` + +当前会写： + +```text +/_pipeline/stages/cesium.manifest.json +/_pipeline/stages/compress.manifest.json +``` + +代码入口： + +```js +const { writeStageManifest, readStageManifest } = require("./lib/stage-manifest"); +``` + +### 3. Contracts + +通用字段： + +```json +{ + "manifestVersion": 1, + "area": "nantaizi-lake-innovation-valley", + "stage": "cesium", + "status": "ok", + "config": "/absolute/config.json", + "startedAt": "2026-08-04T01:33:40.241Z", + "finishedAt": "2026-08-04T01:34:31.065Z", + "durationMs": 50824, + "inputs": {}, + "outputs": {}, + "summary": {}, + "warnings": [] +} +``` + +File records use this shape: + +```json +{ + "path": "/absolute/file.glb", + "bytes": 22381996, + "modifiedAt": "2026-08-04T01:34:30.646Z", + "sha256": "..." +} +``` + +`cesium` manifest: + +- `inputs.blend` +- `outputs.glb` +- `outputs.metadata` +- `outputs.cesiumPreview` +- `outputs.vehicleRoute` +- `outputs.vehicleModel` +- `summary.glb.fileBytes` +- `summary.glb.counts` +- `summary.glb.extensionsUsed` + +`compress` manifest: + +- `inputs.glb` +- `inputs.metadata` +- `inputs.cesiumPreview` +- `outputs.compressedGlb` +- `outputs.compressedMetadata` +- `outputs.compressedCesiumPreview` +- `summary.sourceGlb` +- `summary.compressedGlb` +- `summary.options` +- `summary.compressionRatio` +- `summary.savedBytes` + +Manifest files are written atomically via `*.tmp` then `renameSync`. + +### 4. Validation & Error Matrix + +| 条件 | 结果 | +|---|---| +| Stage command fails | 不写成功 manifest；原 stage 错误继续抛出 | +| GLB digest 失败 | stage 失败，manifest 不写 | +| Manifest output 文件缺失 | 写入时抛错；诊断时报告 stale/missing | +| Manifest JSON 无法解析 | 诊断报告 invalid warning | +| Manifest 记录的 input / output sha/bytes 与当前文件不同 | 诊断报告 stale warning | +| GLB size / nodes / images 超保守预算 | manifest `warnings[]` 记录，诊断继续独立报警 | + +### 5. Good/Base/Bad Cases + +- Good: `--stages cesium` 成功后写 `cesium.manifest.json`，诊断显示 `ok cesium`。 +- Good: `--stages compress` 成功后写 `compress.manifest.json`，summary 记录压缩比和节省字节。 +- Base: 旧产物没有 manifest，诊断显示 expected manifest missing，提示重跑对应阶段。 +- Bad: 手工编辑 GLB 后不重跑阶段，诊断显示 manifest stale。 + +### 6. Tests Required + +- `node --check scripts/lib/stage-manifest.js` +- `node --check scripts/build-area.js` +- `node --check scripts/diagnose-area.js` +- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages cesium` +- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress` +- `npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.json` +- JSON parse both generated manifests and assert `manifestVersion`, `stage`, `inputs`, + `outputs`, `summary`, and `warnings` fields exist. + +### 7. Wrong vs Correct + +Wrong: + +```js +fs.writeFileSync(path.join(area.outputs.areaDir, "cesium.json"), JSON.stringify(data)); +``` + +Correct: + +```js +writeStageManifest(area, { + stage: "cesium", + status: "ok", + inputs, + outputs, + summary, + warnings, +}); +``` + +--- + ## CLI 参数解析 独立入口脚本各有一份同语义的 `parseArgs`： @@ -239,7 +383,7 @@ const glb = area.outputs.glb; ``` config/areas/.json 用户写的区域配置（面向人） - │ build-area.js: normalizeAreaConfig() —— 补默认值、推导全部输出路径 + │ lib/area-config.js: normalizeAreaConfig() —— 补默认值、推导全部输出路径 ▼ area（内存中的归一化对象） │ writeDerivedConfig() @@ -251,14 +395,15 @@ build-osm2streets-qgis.js / reimport-gpkg.js ``` **低层脚本从不读区域配置**，只读派生配置。这条边界让低层脚本能被独立调试，也让 -"输出路径怎么算出来的"只有一处答案（`normalizeAreaConfig`，`build-area.js:74`）。 +"输出路径怎么算出来的"只有一处答案（`scripts/lib/area-config.js` 的 +`normalizeAreaConfig`）。 派生配置**落在 `_pipeline/` 目录里而不是临时目录**——构建失败时它还在，可以直接拿去 -复现（`writeDerivedConfig`，`build-area.js:189`）。 +复现。 ### 输出路径全部从 `id` 推导 -`normalizeAreaConfig` 一次性算出 14 个输出路径（`build-area.js:87-102`），规则统一是 +`normalizeAreaConfig` 一次性算出输出路径，规则统一是 `//.`，`fileStem` 默认等于 `id`。 每一项都可以被 `outputs.*` 单独覆盖，写法固定： diff --git a/.trellis/spec/pipeline/index.md b/.trellis/spec/pipeline/index.md index 3acc039..075415a 100644 --- a/.trellis/spec/pipeline/index.md +++ b/.trellis/spec/pipeline/index.md @@ -14,6 +14,7 @@ | 调 QGIS / GDAL / Blender 子进程 | [外部工具调用](./external-tools.md) | | 加阶段、加 CLI 参数、改配置字段 | [CLI 与阶段](./cli-and-stages.md) | | 改区域诊断命令或共享区域配置归一化 | [CLI 与阶段](./cli-and-stages.md#区域诊断命令) | +| 改 stage manifest 写入、读取或 stale 判断 | [CLI 与阶段](./cli-and-stages.md#stage-manifest-契约) | | 改预览页生成 | [../preview/](../preview/index.md) | | 声称"纯重构，产物不变" | [产物一致性指南](../guides/artifact-parity-guide.md) | @@ -49,10 +50,14 @@ config/areas/.json ├─[cesium]────────▶ Blender + blender/export_cesium.py │ 读 .blend → .glb + .json │ → 并自动执行 preview + │ → _pipeline/stages/cesium.manifest.json │ - └─[preview]───────▶ 生成 -cesium-preview.html - + 拷贝 lib/cesium-preview.{js,css} - + 车辆巡航路线与模型 + ├─[preview]───────▶ 生成 -cesium-preview.html + │ + 拷贝 lib/cesium-preview.{js,css} + │ + 车辆巡航路线与模型 + │ + └─[compress]──────▶ 生成并列压缩 GLB / metadata / preview + → _pipeline/stages/compress.manifest.json ``` **阶段之间只通过磁盘产物耦合**，不传内存状态。这是单跑任意阶段能work 的前提。 @@ -78,16 +83,17 @@ config/areas/.json | 文件 | 行数 | 职责 | |---|---|---| -| `build-area.js` | 745 | 主入口：区域配置读取、阶段调度、Cesium 预览页与车辆巡航生成 | -| `diagnose-area.js` | 438 | 快速诊断：OSM building relation、植被统计、现有产物和 GLB digest | -| `lib/area-config.js` | 133 | 区域配置归一化与输出路径推导，供 build / diagnose 复用 | +| `build-area.js` | 815 | 主入口：区域配置读取、阶段调度、Cesium 预览页、车辆巡航和 stage manifest 写入 | +| `diagnose-area.js` | 557 | 快速诊断：OSM building relation、植被统计、现有产物、stage manifest 和 GLB digest | +| `lib/area-config.js` | 135 | 区域配置归一化与输出路径推导，供 build / diagnose 复用 | +| `lib/stage-manifest.js` | 100 | stage manifest 路径、文件记录、GLB budget warning 和原子 JSON 写入 | | `build-osm2streets-qgis.js` | 1468 | intermediates：osm2streets 解析、图层拆分、人行道转角合成、GeoPackage 与 QGIS 工程生成 | | `reimport-gpkg.js` | 179 | reimport：GeoPackage → GeoJSON 反向导出 | | `lib/scene-layers.js` | 164 | 九个图层的单一事实源 + 四个派生函数 | | `lib/cesium-preview.js` / `.css` | 672 / 230 | 预览页运行时，见 [../preview/](../preview/index.md) | | `normalize-lane-arrows.py` | 182 | 合并 osm2streets 的三角网箭头（跑在 QGIS Python 里） | | `parity.js` | 270 | 产物一致性校验驱动 | -| `glb-digest.js` | 132 | GLB 结构摘要，CLI 和诊断脚本共用 | +| `glb-digest.js` | 132 | GLB 结构摘要，CLI、build manifest 和诊断脚本共用 | --- diff --git a/.trellis/tasks/08-04-add-stage-manifests/check.jsonl b/.trellis/tasks/08-04-add-stage-manifests/check.jsonl new file mode 100644 index 0000000..9dd3234 --- /dev/null +++ b/.trellis/tasks/08-04-add-stage-manifests/check.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/08-04-add-stage-manifests/implement.jsonl b/.trellis/tasks/08-04-add-stage-manifests/implement.jsonl new file mode 100644 index 0000000..9dd3234 --- /dev/null +++ b/.trellis/tasks/08-04-add-stage-manifests/implement.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/08-04-add-stage-manifests/prd.md b/.trellis/tasks/08-04-add-stage-manifests/prd.md new file mode 100644 index 0000000..ac51832 --- /dev/null +++ b/.trellis/tasks/08-04-add-stage-manifests/prd.md @@ -0,0 +1,41 @@ +# Add stage manifests + +## Goal + +Add first-version machine-readable stage manifests for build outputs and diagnostics. + +## Requirements + +- Add a first-version stage manifest contract for build pipeline outputs. +- Manifest files must live under the area `_pipeline/` directory and be JSON. +- The first implementation must cover: + - `cesium` stage: record blend input, GLB / metadata / preview outputs, GLB + digest summary, duration, and warnings. + - `compress` stage: record source GLB / metadata / preview inputs, compressed + GLB / metadata / preview outputs, compression summary, duration, and + warnings. +- Manifest writing must not replace or rename existing baseline artifacts. +- Diagnostics must report discovered stage manifest status so stale/missing + stage contracts become visible. +- Shared helpers should prevent duplicate path/stat/hash/manifest formatting + rules across scripts. +- README and pipeline spec must document the manifest location and contract. + +## Acceptance Criteria + +- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages cesium` + writes a `cesium` manifest under `_pipeline`. +- [x] Running `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress` + writes a `compress` manifest under `_pipeline`. +- [x] Manifest JSON records input/output file stats and machine-readable summary + counts for GLB-producing stages. +- [x] `npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.json` + displays stage manifest status. +- [x] Syntax checks pass for changed Node scripts. +- [x] README and Trellis pipeline spec document the new contract. + +## Notes + +- Keep `prd.md` focused on requirements, constraints, and acceptance criteria. +- Lightweight tasks can remain PRD-only. +- For complex tasks, add `design.md` for technical design and `implement.md` for execution planning before `task.py start`. diff --git a/.trellis/tasks/08-04-add-stage-manifests/task.json b/.trellis/tasks/08-04-add-stage-manifests/task.json new file mode 100644 index 0000000..38b3bda --- /dev/null +++ b/.trellis/tasks/08-04-add-stage-manifests/task.json @@ -0,0 +1,26 @@ +{ + "id": "add-stage-manifests", + "name": "add-stage-manifests", + "title": "Add stage manifests", + "description": "Add first-version machine-readable stage manifests for build outputs and diagnostics.", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "dingkang", + "assignee": "dingkang", + "createdAt": "2026-08-04", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/README.md b/README.md index fd0f73e..f5930a3 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,17 @@ npm run diagnose:area -- --config config/areas/nantaizi-lake-innovation-valley.j 现有产物状态，以及 GLB 的 size / nodes / meshes / materials / images / extensions。 缺少已期望的基线产物、异常 building relation、GLB 超过保守预算等会进入 `Warnings`。 +`cesium` 和 `compress` 阶段成功后会写机器可读的阶段 manifest： + +```text +outputs//_pipeline/stages/cesium.manifest.json +outputs//_pipeline/stages/compress.manifest.json +``` + +manifest 记录阶段输入/输出文件的 bytes、mtime、sha256、耗时、GLB 结构摘要和 warning。 +`diagnose:area` 会读取这些 manifest；如果已有 GLB 但 manifest 缺失，或者 manifest +记录的输入/输出 sha/bytes 和当前文件不一致，会在 `Stage manifests` 和 `Warnings` 里标出来。 + ## 区域配置 新区域从模板复制： diff --git a/docs/changelog.md b/docs/changelog.md index 6c8990e..34e19ab 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,13 @@ ## 2026-08-04 +- 新增第一版 stage manifest 契约：`cesium` 和 `compress` 阶段成功后分别写入 + `outputs//_pipeline/stages/cesium.manifest.json` 与 + `compress.manifest.json`。manifest 记录输入/输出文件的 path / bytes / + modifiedAt / sha256、阶段 started / finished / duration、GLB digest summary、 + 压缩参数 / ratio / savedBytes 和 warnings；写入走 `scripts/lib/stage-manifest.js` + 的临时文件 + rename 原子落盘。`diagnose:area` 现在会展示 stage manifest 状态， + 对旧产物缺 manifest 或输入/输出 sha/bytes 漂移给 warning。 - 新增区域快速诊断入口：`npm run diagnose:area -- --config config/areas/.json`。 诊断只读取区域配置、OSM XML 和已有输出，不启动 QGIS / Blender / Cesium 构建；报告 OSM bounds、building way / multipolygon relation、显式 `height` / diff --git a/scripts/build-area.js b/scripts/build-area.js index e02d968..f294f35 100755 --- a/scripts/build-area.js +++ b/scripts/build-area.js @@ -4,6 +4,14 @@ const fs = require("fs"); const path = require("path"); const { spawnSync } = require("child_process"); const { readAreaConfig } = require("./lib/area-config"); +const { digest: glbDigest } = require("./glb-digest"); +const { + fileRecord, + glbBudgetWarnings, + glbSummary, + optionalFileRecord, + writeStageManifest, +} = require("./lib/stage-manifest"); const repoRoot = path.resolve(__dirname, ".."); const args = parseArgs(process.argv.slice(2)); @@ -206,6 +214,8 @@ function exportCesium(area) { fs.mkdirSync(path.dirname(area.outputs.metadata), { recursive: true }); console.log("Stage: cesium"); + const started = Date.now(); + const startedAt = new Date(started).toISOString(); runCommand(blenderExecutable(area), [ "--background", "--python", @@ -219,6 +229,30 @@ function exportCesium(area) { area.outputs.metadata, ], "cesium"); writeCesiumPreview(area); + const finished = Date.now(); + const digest = glbDigest(area.outputs.glb); + writeStageManifest(area, { + stage: "cesium", + status: "ok", + config: configPath, + startedAt, + finishedAt: new Date(finished).toISOString(), + durationMs: finished - started, + inputs: { + blend: fileRecord(area.outputs.blend), + }, + outputs: { + glb: fileRecord(area.outputs.glb), + metadata: fileRecord(area.outputs.metadata), + cesiumPreview: fileRecord(area.outputs.cesiumPreview), + vehicleRoute: optionalFileRecord(area.outputs.vehicleRoute), + vehicleModel: optionalFileRecord(area.outputs.vehicleModel), + }, + summary: { + glb: glbSummary(digest), + }, + warnings: glbBudgetWarnings("Cesium", digest), + }); } function compressCesiumGlb(area) { @@ -256,7 +290,43 @@ function compressCesiumGlb(area) { } console.log("Stage: compress (Cesium GLB texture resize + WebP)"); + const started = Date.now(); + const startedAt = new Date(started).toISOString(); runCommand(process.execPath, compressArgs, "compress"); + const finished = Date.now(); + const sourceDigest = glbDigest(area.outputs.glb); + const compressedDigest = glbDigest(area.outputs.compressedGlb); + writeStageManifest(area, { + stage: "compress", + status: "ok", + config: configPath, + startedAt, + finishedAt: new Date(finished).toISOString(), + durationMs: finished - started, + inputs: { + glb: fileRecord(area.outputs.glb), + metadata: fileRecord(area.outputs.metadata), + cesiumPreview: fileRecord(area.outputs.cesiumPreview), + }, + outputs: { + compressedGlb: fileRecord(area.outputs.compressedGlb), + compressedMetadata: fileRecord(area.outputs.compressedMetadata), + compressedCesiumPreview: fileRecord(area.outputs.compressedCesiumPreview), + }, + summary: { + sourceGlb: glbSummary(sourceDigest), + compressedGlb: glbSummary(compressedDigest), + options: { + textureSize: area.compress.textureSize, + quality: area.compress.quality, + effort: area.compress.effort, + meshopt: area.compress.meshopt, + }, + compressionRatio: Number((compressedDigest.fileBytes / sourceDigest.fileBytes).toFixed(4)), + savedBytes: sourceDigest.fileBytes - compressedDigest.fileBytes, + }, + warnings: glbBudgetWarnings("Compressed", compressedDigest), + }); } function blenderExecutable(area) { diff --git a/scripts/diagnose-area.js b/scripts/diagnose-area.js index 7a4f4c9..0ba3dab 100644 --- a/scripts/diagnose-area.js +++ b/scripts/diagnose-area.js @@ -5,6 +5,7 @@ const fs = require("fs"); const path = require("path"); const { readAreaConfig } = require("./lib/area-config"); const { digest: glbDigest } = require("./glb-digest"); +const { fileRecord, readStageManifest, stageManifestPath } = require("./lib/stage-manifest"); const repoRoot = path.resolve(__dirname, ".."); const DEFAULT_CONFIG = path.join(repoRoot, "config", "areas", "nantaizi-lake-innovation-valley.json"); @@ -293,7 +294,106 @@ function metadataSummary(file, warnings) { } } -function collectWarnings(area, osm, artifacts, glb, metadata) { +function stageManifestStatus(area) { + const stages = [ + { + stage: "cesium", + expected: fs.existsSync(area.outputs.glb), + inputs: { + blend: area.outputs.blend, + }, + outputs: { + glb: area.outputs.glb, + metadata: area.outputs.metadata, + cesiumPreview: area.outputs.cesiumPreview, + }, + }, + { + stage: "compress", + expected: fs.existsSync(area.outputs.compressedGlb), + inputs: { + glb: area.outputs.glb, + metadata: area.outputs.metadata, + cesiumPreview: area.outputs.cesiumPreview, + }, + outputs: { + compressedGlb: area.outputs.compressedGlb, + compressedMetadata: area.outputs.compressedMetadata, + compressedCesiumPreview: area.outputs.compressedCesiumPreview, + }, + }, + ]; + return stages.map((entry) => { + const file = stageManifestPath(area, entry.stage); + try { + const manifest = readStageManifest(area, entry.stage); + if (!manifest) { + return { + stage: entry.stage, + path: file, + expected: entry.expected, + exists: false, + valid: false, + fresh: false, + manifestWarnings: [], + issues: entry.expected ? ["manifest missing"] : [], + }; + } + const issues = [ + ...manifestFileIssues(manifest.inputs || {}, entry.inputs, "input"), + ...manifestFileIssues(manifest.outputs || {}, entry.outputs, "output"), + ]; + return { + stage: entry.stage, + path: file, + expected: entry.expected, + exists: true, + valid: true, + fresh: issues.length === 0, + finishedAt: manifest.finishedAt || null, + durationMs: manifest.durationMs ?? null, + summary: manifest.summary || null, + manifestWarnings: Array.isArray(manifest.warnings) ? manifest.warnings : [], + issues, + }; + } catch (error) { + return { + stage: entry.stage, + path: file, + expected: entry.expected, + exists: fs.existsSync(file), + valid: false, + fresh: false, + manifestWarnings: [], + issues: [`manifest unreadable: ${error.message}`], + }; + } + }); +} + +function manifestFileIssues(records, expectedFiles, label) { + const issues = []; + for (const [key, file] of Object.entries(expectedFiles)) { + const recorded = records[key]; + if (!recorded) { + issues.push(`${label} ${key} not recorded`); + continue; + } + if (!fs.existsSync(file)) { + issues.push(`${label} ${key} file missing`); + continue; + } + const current = fileRecord(file); + if (recorded.bytes !== current.bytes) { + issues.push(`${label} ${key} bytes changed`); + } else if (recorded.sha256 && recorded.sha256 !== current.sha256) { + issues.push(`${label} ${key} sha256 changed`); + } + } + return issues; +} + +function collectWarnings(area, osm, artifacts, manifests, glb, metadata) { const warnings = []; if (!osm.bounds) warnings.push("OSM has no valid ; scene extent may be wrong."); if (osm.ways.missingNodeRefs) { @@ -312,6 +412,18 @@ function collectWarnings(area, osm, artifacts, glb, metadata) { warnings.push(`Artifact has wrong type: ${artifact.label} (${artifact.path}).`); } } + for (const manifest of manifests) { + if (manifest.expected && !manifest.exists) { + warnings.push(`Expected stage manifest missing: ${manifest.stage} (${manifest.path}).`); + } else if (manifest.exists && !manifest.valid) { + warnings.push(`Stage manifest invalid: ${manifest.stage} (${manifest.issues.join("; ")}).`); + } else if (manifest.exists && !manifest.fresh) { + warnings.push(`Stage manifest stale: ${manifest.stage} (${manifest.issues.join("; ")}).`); + } + for (const warning of manifest.manifestWarnings) { + warnings.push(`Stage manifest warning (${manifest.stage}): ${warning}.`); + } + } if (glb) { if (glb.fileBytes > BUDGETS.glbBytes) { warnings.push(`GLB size ${mb(glb.fileBytes)} MB exceeds budget ${mb(BUDGETS.glbBytes)} MB.`); @@ -332,7 +444,7 @@ function collectWarnings(area, osm, artifacts, glb, metadata) { return warnings; } -function printReport(area, configPath, osm, artifacts, glb, metadata, warnings) { +function printReport(area, configPath, osm, artifacts, manifests, glb, metadata, warnings) { console.log("Area diagnostics"); console.log(`Area: ${area.id}`); console.log(`Config: ${configPath}`); @@ -381,6 +493,31 @@ function printReport(area, configPath, osm, artifacts, glb, metadata, warnings) if (metadata) { console.log(` metadata asset: ${metadata.asset || "missing"}, assets: ${metadata.assets}`); } + console.log(""); + + console.log("Stage manifests"); + for (const manifest of manifests) { + let state = "absent"; + if (manifest.exists && !manifest.valid) state = "invalid"; + else if (manifest.exists && !manifest.fresh) state = "stale"; + else if (manifest.exists) state = "ok"; + else if (manifest.expected) state = "missing"; + const timing = manifest.finishedAt + ? `, finished ${manifest.finishedAt}, ${manifest.durationMs ?? "?"} ms` + : ""; + const issues = manifest.issues.length ? `, ${manifest.issues.join("; ")}` : ""; + console.log(` ${state.padEnd(7)} ${manifest.stage}: ${manifest.path}${timing}${issues}`); + for (const warning of manifest.manifestWarnings) { + console.log(` warning: ${warning}`); + } + const glbSummary = manifest.summary?.glb || manifest.summary?.compressedGlb; + if (glbSummary?.counts) { + console.log( + ` GLB: ${formatBytes(glbSummary.fileBytes)}, ${glbSummary.counts.nodes} nodes, ` + + `${glbSummary.counts.meshes} meshes, ${glbSummary.counts.images} images`, + ); + } + } if (glb) { console.log(""); console.log("GLB digest"); @@ -425,14 +562,15 @@ function main() { const area = readAreaConfig(configPath, { repoRoot }); const osm = parseOsm(fs.readFileSync(area.input, "utf8")); const artifacts = artifactStatus(area); + const manifests = stageManifestStatus(area); const metadataWarnings = []; const metadata = metadataSummary(area.outputs.metadata, metadataWarnings); const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null; const warnings = [ ...metadataWarnings, - ...collectWarnings(area, osm, artifacts, glb, metadata), + ...collectWarnings(area, osm, artifacts, manifests, glb, metadata), ]; - printReport(area, configPath, osm, artifacts, glb, metadata, warnings); + printReport(area, configPath, osm, artifacts, manifests, glb, metadata, warnings); } main(); diff --git a/scripts/lib/area-config.js b/scripts/lib/area-config.js index 1324832..04dfc16 100644 --- a/scripts/lib/area-config.js +++ b/scripts/lib/area-config.js @@ -25,6 +25,7 @@ function normalizeAreaConfig(raw, options = {}) { const compress = normalizeCompressConfig(raw.compress); const compressedFileStem = outputOverrides.compressedFileStem || `${fileStem}-compressed-webp${compress.textureSize}${compress.meshopt ? "-meshopt" : ""}`; + const pipelineDir = path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline")); const outputs = { areaDir, geojsonDir: path.resolve(outputOverrides.geojsonDir || path.join(areaDir, "osm2streets_web_out")), @@ -49,7 +50,8 @@ function normalizeAreaConfig(raw, options = {}) { ), vehicleRoute: path.resolve(outputOverrides.vehicleRoute || path.join(areaDir, `${fileStem}-vehicle-route.json`)), vehicleModel: path.resolve(outputOverrides.vehicleModel || path.join(areaDir, `${fileStem}-vehicle-car.gltf`)), - pipelineDir: path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline")), + pipelineDir, + stageManifestDir: path.resolve(outputOverrides.stageManifestDir || path.join(pipelineDir, "stages")), }; return { diff --git a/scripts/lib/stage-manifest.js b/scripts/lib/stage-manifest.js new file mode 100644 index 0000000..23e330e --- /dev/null +++ b/scripts/lib/stage-manifest.js @@ -0,0 +1,100 @@ +"use strict"; + +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); + +const MANIFEST_VERSION = 1; +const BUDGETS = { + glbBytes: 25 * 1024 * 1024, + glbNodes: 1000, + glbImages: 24, +}; + +function stageManifestPath(area, stage) { + return path.join(area.outputs.stageManifestDir, `${stage}.manifest.json`); +} + +function readStageManifest(area, stage) { + const file = stageManifestPath(area, stage); + if (!fs.existsSync(file)) return null; + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function writeStageManifest(area, manifest) { + const file = stageManifestPath(area, manifest.stage); + const payload = { + manifestVersion: MANIFEST_VERSION, + area: area.id, + ...manifest, + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + const temp = `${file}.tmp`; + fs.writeFileSync(temp, `${JSON.stringify(payload, null, 2)}\n`); + fs.renameSync(temp, file); + console.log(`Stage manifest: ${file}`); + return file; +} + +function fileRecord(file) { + const stat = fs.statSync(file); + const record = { + path: file, + bytes: stat.size, + modifiedAt: stat.mtime.toISOString(), + }; + if (stat.isFile()) { + record.sha256 = sha256(file); + } + return record; +} + +function optionalFileRecord(file) { + if (!fs.existsSync(file)) return null; + return fileRecord(file); +} + +function glbSummary(digest) { + if (!digest) return null; + return { + fileBytes: digest.fileBytes, + counts: digest.counts, + extensionsUsed: digest.extensionsUsed, + }; +} + +function glbBudgetWarnings(label, digest, budgets = BUDGETS) { + if (!digest) return []; + const warnings = []; + if (digest.fileBytes > budgets.glbBytes) { + warnings.push(`${label} GLB size ${mb(digest.fileBytes)} MB exceeds budget ${mb(budgets.glbBytes)} MB`); + } + if (digest.counts.nodes > budgets.glbNodes) { + warnings.push(`${label} GLB nodes ${digest.counts.nodes} exceed budget ${budgets.glbNodes}`); + } + if (digest.counts.images > budgets.glbImages) { + warnings.push(`${label} GLB images ${digest.counts.images} exceed budget ${budgets.glbImages}`); + } + return warnings; +} + +function sha256(file) { + const hash = crypto.createHash("sha256"); + hash.update(fs.readFileSync(file)); + return hash.digest("hex"); +} + +function mb(bytes) { + return Number((bytes / 1024 / 1024).toFixed(2)); +} + +module.exports = { + BUDGETS, + fileRecord, + glbBudgetWarnings, + glbSummary, + optionalFileRecord, + readStageManifest, + stageManifestPath, + writeStageManifest, +};