From 7ba85946dc9254799de22c17a06044bd7b040190 Mon Sep 17 00:00:00 2001 From: que01 Date: Mon, 3 Aug 2026 17:50:21 +0800 Subject: [PATCH] Add optional GLB compression stage --- .trellis/spec/config/index.md | 23 +- .trellis/spec/pipeline/cli-and-stages.md | 113 ++++++- .../check.jsonl | 1 + .../implement.jsonl | 1 + .../08-03-experiment-glb-compression/prd.md | 55 ++++ .../report.md | 99 +++++++ .../task.json | 26 ++ README.md | 45 ++- config/examples/template.json | 6 + docs/changelog.md | 6 + package.json | 3 +- scripts/build-area.js | 97 +++++- scripts/compress-glb.js | 278 ++++++++++++++++++ 13 files changed, 738 insertions(+), 15 deletions(-) create mode 100644 .trellis/tasks/08-03-experiment-glb-compression/check.jsonl create mode 100644 .trellis/tasks/08-03-experiment-glb-compression/implement.jsonl create mode 100644 .trellis/tasks/08-03-experiment-glb-compression/prd.md create mode 100644 .trellis/tasks/08-03-experiment-glb-compression/report.md create mode 100644 .trellis/tasks/08-03-experiment-glb-compression/task.json create mode 100644 scripts/compress-glb.js diff --git a/.trellis/spec/config/index.md b/.trellis/spec/config/index.md index 9d9b0d9..74b42ae 100644 --- a/.trellis/spec/config/index.md +++ b/.trellis/spec/config/index.md @@ -12,7 +12,7 @@ ``` config/areas/.json ← 你写的 - │ build-area.js: normalizeAreaConfig() 补默认值 + 推导 14 个输出路径 + │ build-area.js: normalizeAreaConfig() 补默认值 + 推导输出路径 ▼ /_pipeline/osm2streets-qgis.config.json ← 生成的，不要手改 │ @@ -52,6 +52,7 @@ cp config/examples/template.json config/areas/my-area.json | `qgis` | | 见下 | QGIS/osm2streets 旋钮 | | `osm2streets` | | 见下 | 透传给 osm2streets 的选项 | | `blender` | | 见下 | Blender 侧选项 | +| `compress` | | 见下 | 显式 `compress` 阶段的 GLB 压缩选项 | | `outputs` | | 从 `id` 推导 | 输出路径覆盖，逃生舱 | **路径一律绝对**。`normalizeAreaConfig` 对每一项都做 `path.resolve`，相对路径会 @@ -65,10 +66,11 @@ cp config/examples/template.json config/areas/my-area.json | `blender` | `true` | | | `cesium` | `true` | | -`reimport` 和 `preview` **在这里配也没用**——`normalizeAreaConfig:117-118` 把它们 +`reimport`、`preview` 和 `compress` **在这里配也没用**——`normalizeAreaConfig` 把它们 硬编码为 `false`，只能靠 `--stages` 显式请求。 -> 恢复动作（reimport）和补丁动作（preview）不该被一份配置文件变成默认行为。 +> 恢复动作（reimport）、补丁动作（preview）和替代产物动作（compress）不该被一份 +> 配置文件变成默认行为。 `--stages` 会整体覆盖这里的默认值。 @@ -117,6 +119,18 @@ cp config/examples/template.json config/areas/my-area.json | `treeStyle` | `"natural"` | 合法值见 `generate_scene.py` 的 `TREE_STYLES`（`natural`、`procedural`、`shapespark`） | | `officeOverrides` | `""` | 旧名 `office_overrides` 仍被接受 | +### `compress` + +只影响显式 `--stages compress`。默认压缩链是 texture resize + WebP transcode， +不覆盖默认 `.glb`。 + +| 字段 | 默认 | 说明 | +|---|---|---| +| `textureSize` | `768` | 最大纹理宽高，范围 `64..4096` | +| `quality` | `82` | WebP 质量，范围 `1..100` | +| `effort` | `80` | WebP 编码 effort，范围 `0..100` | +| `meshopt` | `false` | 是否追加 `EXT_meshopt_compression`。开启前要单独验证 Cesium 兼容性 | + ### `outputs`（逃生舱） 默认全部从 `id` 推导为 `//.`。需要定制时逐项覆盖： @@ -134,6 +148,7 @@ cp config/examples/template.json config/areas/my-area.json 可覆盖的键（`build-area.js:87-102`）：`areaDir`、`fileStem`、`geojsonDir`、`gpkg`、 `qgisProject`、`qgisPreview`、`blend`、`render`、`glb`、`metadata`、`cesiumPreview`、 +`compressedFileStem`、`compressedGlb`、`compressedMetadata`、`compressedCesiumPreview`、 `vehicleRoute`、`vehicleModel`、`pipelineDir`。 **优先改 `fileStem` 或 `areaDir`**——它们能一次性影响全部派生路径。逐个覆盖容易漏。 @@ -175,7 +190,7 @@ cp config/examples/template.json config/areas/my-area.json | 布尔字段用 `\|\|` 兜底 | `false` 被翻转 | | 给新字段造顶层平铺别名 | 扩大历史包袱 | | 逐个覆盖 `outputs` 而不用 `fileStem` | 漏掉某个产物路径 | -| 在 `stages` 里配 `reimport` / `preview` | 无效，被硬编码为 false | +| 在 `stages` 里配 `reimport` / `preview` / `compress` | 无效，被硬编码为 false | | 加数值字段不做范围校验 | 错配置在中途才崩，输出已被破坏 | --- diff --git a/.trellis/spec/pipeline/cli-and-stages.md b/.trellis/spec/pipeline/cli-and-stages.md index 1278d7e..00e0ac2 100644 --- a/.trellis/spec/pipeline/cli-and-stages.md +++ b/.trellis/spec/pipeline/cli-and-stages.md @@ -18,6 +18,105 @@ 全部是 CommonJS（`package.json` 的 `"type": "commonjs"`），无构建步骤、无 TypeScript、 零运行时依赖（唯一依赖 `osm2streets-js-node` 只被 `build-osm2streets-qgis.js` 用）。 +## 可选 GLB 压缩阶段 + +### 1. Scope / Trigger + +`compress` 是显式请求的替代产物阶段，不属于 `all`，也不覆盖默认 `.glb`。 +它用于在已有 Cesium GLB 上生成并列压缩 GLB、metadata 和预览页。当前主路径是 +texture resize + WebP transcode。 + +`scripts/compress-glb.js` 是该阶段调用的低层脚本，也可单独运行做实验。 + +### 2. Signatures + +阶段入口： + +```bash +npm run build:area -- --config config/areas/.json --stages compress +npm run build:area -- --config config/areas/.json --stages cesium,compress +``` + +低层脚本入口： + +```bash +npm run compress:glb -- --input in.glb --output out.glb [options] +``` + +可选参数： + +| 参数 | 默认 | 语义 | +|---|---:|---| +| `--texture-size` | `1024` | 最大纹理宽高，范围 `64..4096` | +| `--quality` | `82` | WebP 质量，范围 `1..100` | +| `--effort` | `80` | WebP 编码 effort，范围 `0..100` | +| `--meshopt` | off | 追加 `EXT_meshopt_compression` | +| `--metadata source.json` | none | 复制伴生 metadata 并指向压缩 GLB | +| `--metadata-output out.json` | `.json` | 覆盖 metadata 输出路径 | +| `--preview source.html` | none | 复制 Cesium preview 并指向压缩 metadata | +| `--preview-output out.html` | `-cesium-preview.html` | 覆盖 preview 输出路径 | + +### 3. Contracts + +- 输入必须是现有 `.glb` 文件；`--output` 必须不同于 `--input`。 +- 输出是并列压缩 GLB；默认构建产物不被替换。 +- `compress` 阶段依赖默认 `glb`、`metadata` 和 `cesiumPreview` 已存在。 +- 默认链固定为 `gltf-transform resize -> gltf-transform webp`。 +- `compress` 阶段默认 `textureSize=768`；低层脚本单独运行时默认 `--texture-size 1024`。 +- `build-area.js` 默认写： + - `-compressed-webp768.glb` + - `-compressed-webp768.json` + - `-compressed-webp768-cesium-preview.html` +- 伴生 metadata 的 `asset` 和 `assets[0].url` 改为压缩 GLB 文件名。 +- preview HTML 只替换 `window.OSM_ASSET_PREVIEW_CONFIG` 的 `glbName` / + `metadataName` 和 loading 文案，不改 preview runtime。 +- 成功时 stdout 打印 `GLB_COMPRESS_DONE `，包含压缩前后大小、image / + non-image bytes、结构计数、扩展、metadata / preview 输出路径。 + +### 4. Validation & Error Matrix + +| 条件 | 结果 | +|---|---| +| 缺 `--input` / `--output` | 打印 usage 并退出非零 | +| 输入文件不存在 | `Input GLB not found: ` | +| `--output` 等于 `--input` | 抛错，避免覆盖源 GLB | +| 数值参数超范围 | 抛错并指出合法范围 | +| `gltf-transform` 退出非零 | 抛错并带上 status / signal | +| `--preview` 没有 `--metadata` | 抛错，因为 preview 必须指向存在的 metadata | +| preview HTML 找不到配置块 | 抛错，不做猜测替换 | +| `--stages compress` 但默认 preview 不存在 | `Cesium preview not found: ` | + +### 5. Good/Base/Bad Cases + +- Good: `--stages cesium,compress` 先重导默认 GLB，再生成并列压缩产物。 +- Good: `--texture-size 768 --metadata --preview` 生成压缩 GLB、metadata、HTML， + 源 GLB 保持不变。 +- Base: 只传 `--input --output` 生成压缩 GLB，不生成伴生文件。 +- Bad: 使用 `--meshopt` 后没有做 Cesium 兼容性验证就当默认产物发布。 + +### 6. Tests Required + +- `node --check scripts/compress-glb.js` +- `node --check scripts/build-area.js` +- 对目标区域跑一次 `npm run compress:glb -- ... --metadata --preview` +- 对目标区域跑一次 `npm run build:area -- --stages compress` +- `node scripts/glb-digest.js ` 确认可解析结构和扩展 +- 浏览器/Cesium 预览压缩 HTML，确认 `EXT_texture_webp` 在目标环境可加载 + +### 7. Wrong vs Correct + +Wrong: + +```bash +npm run compress:glb -- --input outputs/a/a.glb --output outputs/a/a.glb +``` + +Correct: + +```bash +npm run compress:glb -- --input outputs/a/a.glb --output outputs/a/a-compressed-webp768.glb +``` + --- ## CLI 参数解析 @@ -94,7 +193,7 @@ gpkg: path.resolve(outputOverrides.gpkg || path.join(areaDir, `${fileStem}.gpkg` --- -## 五个阶段 +## 阶段 | 阶段 | 做什么 | 读 | 写 | |---|---|---|---| @@ -103,8 +202,9 @@ gpkg: path.resolve(outputOverrides.gpkg || path.join(areaDir, `${fileStem}.gpkg` | `blender` | OSM + GeoJSON → 场景 | `.osm`、`osm2streets_web_out/` | `.blend`、`.png` | | `cesium` | 场景 → GLB + 元数据 + 预览页 | `.blend` | `.glb`、`.json`、预览 HTML 及其静态资源 | | `preview` | 只补生成预览页 | `.glb`、`.json` | 预览 HTML 及其静态资源 | +| `compress` | 生成并列压缩 Cesium 产物 | `.glb`、`.json`、默认预览 HTML | `-compressed-webp*.glb/json/html` | -调度是顶层的五个 `if`（`build-area.js:32-46`），顺序固定，**阶段之间不传内存状态， +调度是顶层的阶段 `if`（`build-area.js` 开头），顺序固定，**阶段之间不传内存状态， 只通过磁盘产物耦合**。这就是单跑某个阶段能work 的原因。 `cesium` 阶段结束时会直接调 `writeCesiumPreview(area)`（`build-area.js:285`），所以 @@ -126,7 +226,8 @@ gpkg: path.resolve(outputOverrides.gpkg || path.join(areaDir, `${fileStem}.gpkg` all: ["intermediates", "blender", "cesium"], ``` -`preview` 同样不在 `all` 里——`cesium` 已经包含它。 +`preview` 同样不在 `all` 里——`cesium` 已经包含它。`compress` 也不在 `all` 里—— +它生成的是替代压缩产物，不是 baseline GLB。 ### `intermediates` 与 `reimport` 互斥 @@ -136,9 +237,9 @@ all: ["intermediates", "blender", "cesium"], 这是**显式拒绝而不是警告**——两者同时开，无论谁先跑，另一个的工作都白做。 -`normalizeAreaConfig` 里 `stages.reimport` 和 `stages.preview` 硬编码为 `false` -（`build-area.js:117-118`），**不能从配置文件打开**，只能靠 `--stages` 显式请求。 -恢复动作和补丁动作都不该被一份配置文件变成默认行为。 +`normalizeAreaConfig` 里 `stages.reimport`、`stages.preview` 和 `stages.compress` 硬编码为 `false`， +**不能从配置文件打开**，只能靠 `--stages` 显式请求。 +恢复动作、补丁动作和替代产物动作都不该被一份配置文件变成默认行为。 --- diff --git a/.trellis/tasks/08-03-experiment-glb-compression/check.jsonl b/.trellis/tasks/08-03-experiment-glb-compression/check.jsonl new file mode 100644 index 0000000..9dd3234 --- /dev/null +++ b/.trellis/tasks/08-03-experiment-glb-compression/check.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/08-03-experiment-glb-compression/implement.jsonl b/.trellis/tasks/08-03-experiment-glb-compression/implement.jsonl new file mode 100644 index 0000000..9dd3234 --- /dev/null +++ b/.trellis/tasks/08-03-experiment-glb-compression/implement.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/08-03-experiment-glb-compression/prd.md b/.trellis/tasks/08-03-experiment-glb-compression/prd.md new file mode 100644 index 0000000..de112ea --- /dev/null +++ b/.trellis/tasks/08-03-experiment-glb-compression/prd.md @@ -0,0 +1,55 @@ +# Experiment GLB compression + +## Goal + +Create and validate a repeatable GLB compression experiment for +`nantaizi-lake-innovation-valley`, focused on reducing file size without making +the current uncompressed GLB output the wrong baseline. + +Current measurement: + +- `nantaizi-lake-innovation-valley.glb`: `20.50 MB` +- embedded images: `17.72 MB` +- non-image binary buffer: `2.45 MB` +- conclusion: texture compression / resize should be tested before geometry + compression. + +## Requirements + +1. Add a standalone compression tool or script that consumes an existing GLB and + writes a separate compressed GLB output. +2. Do not replace the default `.glb` in this first experiment. +3. Prefer texture-first compression; geometry compression may be evaluated but + should not be treated as the primary win. +4. Keep the experiment repeatable from the command line. +5. Validate only `nantaizi-lake-innovation-valley`. +6. After visual review accepts the WebP result, integrate it as an explicit + optional `compress` stage that does not run in `all` and does not replace the + default GLB. +7. Report: + - source GLB size + - compressed GLB size + - image vs non-image size before compression + - GLB extensions introduced + - Cesium preview compatibility status or blocker + +## Acceptance Criteria + +- [x] Compression command/script runs from this repo. +- [x] It produces a separate compressed GLB for nantaizi. +- [x] Source GLB is left intact. +- [x] `scripts/glb-digest.js` can inspect the compressed GLB or the blocker is + documented if the digest needs extension support. +- [x] Visual/Cesium compatibility next step is clear. +- [x] Optional `compress` stage can generate compressed nantaizi artifacts from + the existing Cesium outputs. +- [x] `compress` is not included in `all` and cannot be enabled from area + `stages`. +- [x] No `outputs/` files are committed. + +## Out Of Scope + +- Making compressed GLB replace the default pipeline artifact. +- hanyang validation. +- CDN/server gzip/brotli configuration. +- Large frontend UI changes. diff --git a/.trellis/tasks/08-03-experiment-glb-compression/report.md b/.trellis/tasks/08-03-experiment-glb-compression/report.md new file mode 100644 index 0000000..b849c88 --- /dev/null +++ b/.trellis/tasks/08-03-experiment-glb-compression/report.md @@ -0,0 +1,99 @@ +# GLB compression experiment report + +## Baseline + +Target: `nantaizi-lake-innovation-valley` + +- source GLB: `20.50 MB` +- embedded images: `17.72 MB` +- non-image buffers: `2.45 MB` +- counts: `830` nodes, `113` meshes, `32` materials, `18` images, `675` accessors + +The file is texture-heavy, so texture resize / transcode is the primary lever. + +## Implemented + +Added `scripts/compress-glb.js` and `npm run compress:glb`. + +Default chain: + +```text +gltf-transform resize -> gltf-transform webp +``` + +The script: + +- leaves source `.glb` intact +- writes a separate compressed `.glb` +- optionally copies metadata to point at the compressed GLB +- optionally copies the existing Cesium preview HTML to point at compressed + metadata +- prints `GLB_COMPRESS_DONE` with before/after size, image/non-image size, + counts, and extensions + +After visual review accepted the WebP result, added an explicit `compress` +stage to `scripts/build-area.js`: + +```bash +npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress +``` + +The stage calls the same script, writes `-compressed-webp768.*` by +default, and remains outside `all`. + +## Results + +Commands: + +```bash +npm run compress:glb -- --input outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley.glb --output outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-compressed-webp1024.glb --metadata outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley.json --preview outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-cesium-preview.html +npm run compress:glb -- --input outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley.glb --output outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-compressed-webp768.glb --texture-size 768 --metadata outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley.json --preview outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-cesium-preview.html +``` + +Results: + +| Variant | Size | Image bytes | Non-image bytes | Structure | +|---|---:|---:|---:|---| +| source | `20.50 MB` | `17.72 MB` | `2.45 MB` | baseline | +| WebP 1024 | `5.48 MB` | `2.73 MB` | `2.45 MB` | unchanged counts | +| WebP 768 | `4.80 MB` | `2.04 MB` | `2.45 MB` | unchanged counts | + +Both WebP outputs keep `830` nodes, `113` meshes, `32` materials, `18` images, +and `675` accessors. They introduce `EXT_texture_webp` and +`KHR_texture_transform`. + +`scripts/glb-digest.js` can inspect both compressed GLBs and reports the same +node/mesh/material/image/accessor counts as the source. + +Visual review in Cesium on 2026-08-03: the user compared the generated WebP +preview and reported no obvious visible difference from the source preview. + +Extra experiments: + +- `optimize --texture-compress webp --texture-size 1024`: `4.29 MB`, but mesh + and material counts changed because `optimize` runs broader structural passes. +- `meshopt + WebP 1024`: `3.66 MB` with direct `meshopt`, introducing + `EXT_meshopt_compression` and `KHR_mesh_quantization`. +- KTX2 was blocked locally because glTF Transform requires external + KTX-Software `ktx`, which is not installed. + +## Recommendation + +Use the explicit `compress` stage for optional compressed Cesium review / +delivery candidates. Keep the uncompressed GLB as the baseline artifact. Do not +make Meshopt or KTX2 default until Cesium compatibility is verified visually. + +Generated preview candidates: + +- `outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-compressed-webp1024-cesium-preview.html` +- `outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-compressed-webp768-cesium-preview.html` + +Validation: + +- `node --check scripts/compress-glb.js` +- `node --check scripts/build-area.js` +- `npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress` +- `node scripts/glb-digest.js outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-compressed-webp768.glb` +- `node scripts/glb-digest.js outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-compressed-webp1024.glb` +- `python3 ./.trellis/scripts/task.py validate .trellis/tasks/08-03-experiment-glb-compression` +- `git diff --check` diff --git a/.trellis/tasks/08-03-experiment-glb-compression/task.json b/.trellis/tasks/08-03-experiment-glb-compression/task.json new file mode 100644 index 0000000..ac3e620 --- /dev/null +++ b/.trellis/tasks/08-03-experiment-glb-compression/task.json @@ -0,0 +1,26 @@ +{ + "id": "experiment-glb-compression", + "name": "experiment-glb-compression", + "title": "Experiment GLB compression", + "description": "", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "dingkang", + "assignee": "dingkang", + "createdAt": "2026-08-03", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/README.md b/README.md index 7d6f8a5..5def2af 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ - `.glb`：Cesium 可加载的 3D 模型 - `.json`：Cesium 放置元数据和示例代码 - `-cesium-preview.html`：Cesium 本地预览页 +- `-compressed-webp768.glb/json/html`：显式 `compress` 阶段生成的可选压缩预览产物 - `osm2streets_web_out/`：osm2streets GeoJSON 中间层 - `.gpkg` / `.qgz` / `-preview.png`：QGIS 调试资产 @@ -50,10 +51,17 @@ npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages blender npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages cesium npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages preview +npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages compress npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages reimport ``` -`intermediates` 会生成 osm2streets GeoJSON、GeoPackage、QGIS 工程和 QGIS 预览图。`blender` 使用 OSM 和 osm2streets GeoJSON 生成 `.blend`/`.png`。`cesium` 从 `.blend` 导出 `.glb`/`.json`，并生成 Cesium 预览 HTML。`preview` 只在已有 `.glb/.json` 时补生成 HTML。`reimport` 把手工编辑过的 GeoPackage 回导为 GeoJSON，不含在 `all` 里，详见 [QGIS 手工修正工作流](#qgis-手工修正工作流)。 +`intermediates` 会生成 osm2streets GeoJSON、GeoPackage、QGIS 工程和 QGIS 预览图。`blender` 使用 OSM 和 osm2streets GeoJSON 生成 `.blend`/`.png`。`cesium` 从 `.blend` 导出 `.glb`/`.json`，并生成 Cesium 预览 HTML。`preview` 只在已有 `.glb/.json` 时补生成 HTML。`compress` 从已有 `.glb/.json/html` 生成并列压缩产物，不覆盖默认 GLB。`reimport` 把手工编辑过的 GeoPackage 回导为 GeoJSON，不含在 `all` 里，详见 [QGIS 手工修正工作流](#qgis-手工修正工作流)。 + +`compress` 不含在 `all` 里，也不能从配置文件默认开启。需要重导出 Cesium 后立刻生成压缩产物时，显式跑： + +```bash +npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages cesium,compress +``` ## 区域配置 @@ -92,6 +100,12 @@ cp config/examples/template.json config/areas/my-area.json "blender": { "treeStyle": "natural", "officeOverrides": "" + }, + "compress": { + "textureSize": 768, + "quality": 82, + "effort": 80, + "meshopt": false } } ``` @@ -111,7 +125,8 @@ QGIS road-layer knobs: "areaDir": "/absolute/path/to/custom-area", "blend": "/absolute/path/to/custom.blend", "glb": "/absolute/path/to/custom.glb", - "cesiumPreview": "/absolute/path/to/custom-preview.html" + "cesiumPreview": "/absolute/path/to/custom-preview.html", + "compressedGlb": "/absolute/path/to/custom-compressed.glb" } } ``` @@ -174,6 +189,32 @@ Cesium GLB 低层导出： 详见 [blender/README.md](blender/README.md)。 +## 压缩 GLB + +推荐使用显式 `compress` 阶段。它不会覆盖默认 `.glb`，只生成并列的压缩 GLB、 +metadata 和预览页： + +```bash +npm run build:area -- \ + --config config/areas/nantaizi-lake-innovation-valley.json \ + --stages compress +``` + +底层脚本也可以单独调用： + +```bash +npm run compress:glb -- \ + --input outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley.glb \ + --output outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-compressed-webp768.glb \ + --texture-size 768 \ + --metadata outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley.json \ + --preview outputs/nantaizi-lake-innovation-valley/nantaizi-lake-innovation-valley-cesium-preview.html +``` + +该脚本通过 `npx @gltf-transform/cli@4.1.4` 执行 `resize -> webp`。如需进一步压几何， +可追加 `--meshopt`，但会引入 `EXT_meshopt_compression` / `KHR_mesh_quantization`， +应先确认目标 Cesium 版本加载正常。 + ## QGIS 手工修正工作流 如果已经在现成的 `.qgz` 项目里直接编辑了 `gpkg` 图层，不要再重跑 `intermediates`，否则会把手工修改覆盖掉。推荐流程是： diff --git a/config/examples/template.json b/config/examples/template.json index 325b00e..271b22c 100644 --- a/config/examples/template.json +++ b/config/examples/template.json @@ -31,5 +31,11 @@ "blender": { "treeStyle": "natural", "officeOverrides": "" + }, + "compress": { + "textureSize": 768, + "quality": 82, + "effort": 80, + "meshopt": false } } diff --git a/docs/changelog.md b/docs/changelog.md index 645af8e..d9758bf 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,12 @@ ## 2026-08-03 +- 新增显式 `compress` 阶段与 `scripts/compress-glb.js`，用于从已有 Cesium GLB 生成并列 + WebP 压缩产物，不进入 `all`，不覆盖默认 `.glb`。nantaizi 试验结果： + `20.50 MB -> 4.80 MB`，embedded images `17.72 MB -> 2.04 MB`，nodes / meshes / + materials / images / accessors 均保持 `830 / 113 / 32 / 18 / 675`，引入 + `EXT_texture_webp` 和 `KHR_texture_transform`。用户在 Cesium 预览里目测压缩版与 + 默认版无明显区别；Meshopt / KTX2 仍不作为默认路径。 - 固化 Shapespark low-poly plants kit 作为当前植被资产来源：`nantaizi-lake-innovation-valley` 使用 `treeStyle: shapespark`、12 个确定性树变体、`bush-03` scrub 边缘灌木、 `grass-01/02/03` 草坪草簇；Cesium 对 Shapespark cut-out 材质走单独温和 profile diff --git a/package.json b/package.json index 64604b7..5c16077 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "build": "node scripts/build-area.js", "build:area": "node scripts/build-area.js", - "build:qgis": "node scripts/build-osm2streets-qgis.js" + "build:qgis": "node scripts/build-osm2streets-qgis.js", + "compress:glb": "node scripts/compress-glb.js" }, "dependencies": { "osm2streets-js-node": "0.1.4" diff --git a/scripts/build-area.js b/scripts/build-area.js index cbab702..86fb9f6 100755 --- a/scripts/build-area.js +++ b/scripts/build-area.js @@ -44,6 +44,9 @@ if (stages.cesium) { if (stages.preview) { writeCesiumPreview(area); } +if (stages.compress) { + compressCesiumGlb(area); +} console.log("Done."); @@ -82,6 +85,9 @@ function normalizeAreaConfig(raw) { const outputOverrides = raw.outputs || {}; const areaDir = path.resolve(outputOverrides.areaDir || path.join(outputRoot, id)); const fileStem = outputOverrides.fileStem || id; + const compress = normalizeCompressConfig(raw.compress); + const compressedFileStem = outputOverrides.compressedFileStem || + `${fileStem}-compressed-webp${compress.textureSize}${compress.meshopt ? "-meshopt" : ""}`; const outputs = { areaDir, geojsonDir: path.resolve(outputOverrides.geojsonDir || path.join(areaDir, "osm2streets_web_out")), @@ -95,6 +101,15 @@ function normalizeAreaConfig(raw) { cesiumPreview: path.resolve( outputOverrides.cesiumPreview || path.join(areaDir, `${fileStem}-cesium-preview.html`), ), + compressedGlb: path.resolve( + outputOverrides.compressedGlb || path.join(areaDir, `${compressedFileStem}.glb`), + ), + compressedMetadata: path.resolve( + outputOverrides.compressedMetadata || path.join(areaDir, `${compressedFileStem}.json`), + ), + compressedCesiumPreview: path.resolve( + outputOverrides.compressedCesiumPreview || path.join(areaDir, `${compressedFileStem}-cesium-preview.html`), + ), vehicleRoute: path.resolve(outputOverrides.vehicleRoute || path.join(areaDir, `${fileStem}-vehicle-route.json`)), vehicleModel: path.resolve(outputOverrides.vehicleModel || path.join(areaDir, `${fileStem}-vehicle-car.gltf`)), pipelineDir: path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline")), @@ -112,6 +127,7 @@ function normalizeAreaConfig(raw) { cesium: raw.stages?.cesium ?? true, reimport: false, preview: false, + compress: false, }, qgis: { arrowScale: raw.qgis?.arrowScale ?? raw.arrowScale ?? 0.8, @@ -136,6 +152,7 @@ function normalizeAreaConfig(raw) { treeStyle: raw.blender?.treeStyle || "natural", officeOverrides: raw.blender?.officeOverrides || raw.blender?.office_overrides || "", }, + compress, outputs, }; } @@ -147,6 +164,32 @@ function requireText(value, key) { return value; } +function normalizeCompressConfig(raw) { + const value = raw || {}; + return { + textureSize: numberOption(value.textureSize, 768, "compress.textureSize", 64, 4096), + quality: numberOption(value.quality, 82, "compress.quality", 1, 100), + effort: numberOption(value.effort, 80, "compress.effort", 0, 100), + meshopt: booleanOption(value.meshopt, false, "compress.meshopt"), + }; +} + +function numberOption(value, fallback, label, min, max) { + const number = value === undefined ? fallback : Number(value); + if (!Number.isFinite(number) || number < min || number > max) { + throw new Error(`${label} must be a finite number in [${min}, ${max}]`); + } + return number; +} + +function booleanOption(value, fallback, label) { + if (value === undefined) return fallback; + if (typeof value === "boolean") return value; + if (value === "true") return true; + if (value === "false") return false; + throw new Error(`${label} must be boolean`); +} + function splitList(value) { return String(value) .split(",") @@ -158,6 +201,8 @@ function resolveStages(defaults, requested) { if (!requested) return defaults; // 'reimport' is deliberately absent from 'all': it is a recovery step for // hand-edited GeoPackages, never part of a full build. + // 'compress' is also absent from 'all': it creates an alternate Cesium GLB, + // not the baseline asset. const aliases = { all: ["intermediates", "blender", "cesium"], qgis: ["intermediates"], @@ -174,12 +219,22 @@ function resolveStages(defaults, requested) { preview: ["preview"], html: ["preview"], cesiumPreview: ["preview"], + compress: ["compress"], + compression: ["compress"], + compressedCesium: ["compress"], + }; + const out = { + intermediates: false, + reimport: false, + blender: false, + cesium: false, + preview: false, + compress: false, }; - const out = { intermediates: false, reimport: false, blender: false, cesium: false, preview: false }; for (const stage of requested) { const mapped = aliases[stage]; if (!mapped) { - throw new Error(`Unknown stage '${stage}'. Use intermediates, reimport, blender, cesium, preview, or all.`); + throw new Error(`Unknown stage '${stage}'. Use intermediates, reimport, blender, cesium, preview, compress, or all.`); } for (const key of mapped) out[key] = true; } @@ -288,6 +343,44 @@ function exportCesium(area) { writeCesiumPreview(area); } +function compressCesiumGlb(area) { + ensureFile(area.outputs.glb, "Cesium GLB"); + ensureFile(area.outputs.metadata, "Cesium metadata"); + ensureFile(area.outputs.cesiumPreview, "Cesium preview"); + ensureFile(path.join(repoRoot, "scripts", "compress-glb.js"), "GLB compressor"); + fs.mkdirSync(path.dirname(area.outputs.compressedGlb), { recursive: true }); + fs.mkdirSync(path.dirname(area.outputs.compressedMetadata), { recursive: true }); + fs.mkdirSync(path.dirname(area.outputs.compressedCesiumPreview), { recursive: true }); + + const compressArgs = [ + path.join(repoRoot, "scripts", "compress-glb.js"), + "--input", + area.outputs.glb, + "--output", + area.outputs.compressedGlb, + "--texture-size", + String(area.compress.textureSize), + "--quality", + String(area.compress.quality), + "--effort", + String(area.compress.effort), + "--metadata", + area.outputs.metadata, + "--metadata-output", + area.outputs.compressedMetadata, + "--preview", + area.outputs.cesiumPreview, + "--preview-output", + area.outputs.compressedCesiumPreview, + ]; + if (area.compress.meshopt) { + compressArgs.push("--meshopt"); + } + + console.log("Stage: compress (Cesium GLB texture resize + WebP)"); + runCommand(process.execPath, compressArgs, "compress"); +} + function blenderExecutable(area) { return path.join(area.blenderApp, "Contents", "MacOS", "Blender"); } diff --git a/scripts/compress-glb.js b/scripts/compress-glb.js new file mode 100644 index 0000000..37e66ab --- /dev/null +++ b/scripts/compress-glb.js @@ -0,0 +1,278 @@ +#!/usr/bin/env node +"use strict"; + +// Experimental GLB compression wrapper. Keeps the source GLB intact and writes +// a separate compressed artifact for visual comparison in Cesium. + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { spawnSync } = require("child_process"); + +const GLTF_TRANSFORM_VERSION = "4.1.4"; +const PREVIEW_CONFIG_PATTERN = + /window\.OSM_ASSET_PREVIEW_CONFIG\s*=\s*(\{[\s\S]*?\});\s*<\/script>/; + +function parseArgs(argv) { + const values = {}; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (!arg.startsWith("--")) continue; + const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase()); + const next = argv[i + 1]; + if (!next || next.startsWith("--")) { + values[key] = "true"; + } else { + values[key] = next; + i += 1; + } + } + return values; +} + +function usage() { + return [ + "usage: node scripts/compress-glb.js --input in.glb --output out.glb [options]", + "", + "Options:", + " --texture-size 1024 Max texture width/height. Default: 1024", + " --quality 82 WebP quality. Default: 82", + " --effort 80 WebP encoder effort. Default: 80", + " --meshopt Also apply EXT_meshopt_compression", + " --metadata scene.json Write sidecar metadata for the compressed GLB", + " --metadata-output out.json Override compressed metadata path", + " --preview preview.html Write a copied preview pointing to compressed metadata", + " --preview-output out.html Override compressed preview path", + ].join("\n"); +} + +function requireText(value, label) { + if (!value || !String(value).trim()) { + throw new Error(`${label} is required\n\n${usage()}`); + } + return String(value); +} + +function numberOption(value, fallback, label, min, max) { + const number = value === undefined ? fallback : Number(value); + if (!Number.isFinite(number) || number < min || number > max) { + throw new Error(`${label} must be a finite number in [${min}, ${max}]`); + } + return number; +} + +function ensureFile(file, label) { + if (!fs.existsSync(file) || !fs.statSync(file).isFile()) { + throw new Error(`${label} not found: ${file}`); + } +} + +function runGltfTransform(args) { + const command = process.env.NPX_BINARY || "npx"; + const result = spawnSync( + command, + ["--yes", `@gltf-transform/cli@${GLTF_TRANSFORM_VERSION}`, ...args], + { stdio: "inherit" }, + ); + if (result.error) throw result.error; + if (result.status !== 0) { + const signal = result.signal ? ` signal=${result.signal}` : ""; + throw new Error(`gltf-transform ${args[0]} failed with status=${result.status}${signal}`); + } +} + +function readGlbJson(file) { + const buffer = fs.readFileSync(file); + if (buffer.length < 12 || buffer.readUInt32LE(0) !== 0x46546c67) { + throw new Error(`Not a GLB (bad magic): ${file}`); + } + let offset = 12; + while (offset + 8 <= buffer.length) { + const chunkLength = buffer.readUInt32LE(offset); + const chunkType = buffer.readUInt32LE(offset + 4); + const start = offset + 8; + if (chunkType === 0x4e4f534a) { + return JSON.parse(buffer.slice(start, start + chunkLength).toString("utf8")); + } + offset = start + chunkLength; + } + throw new Error(`No JSON chunk found in ${file}`); +} + +function glbStats(file) { + const gltf = readGlbJson(file); + const bufferViews = gltf.bufferViews || []; + const imageViews = new Set( + (gltf.images || []).map((image) => image.bufferView).filter((value) => value !== undefined), + ); + let imageBytes = 0; + let nonImageBytes = 0; + bufferViews.forEach((view, index) => { + if (imageViews.has(index)) imageBytes += view.byteLength || 0; + else nonImageBytes += view.byteLength || 0; + }); + return { + fileBytes: fs.statSync(file).size, + imageBytes, + nonImageBytes, + counts: { + nodes: (gltf.nodes || []).length, + meshes: (gltf.meshes || []).length, + materials: (gltf.materials || []).length, + images: (gltf.images || []).length, + accessors: (gltf.accessors || []).length, + }, + extensionsUsed: (gltf.extensionsUsed || []).slice().sort(), + }; +} + +function mb(bytes) { + return Number((bytes / 1024 / 1024).toFixed(2)); +} + +function metadataOutputPath(output, explicitPath) { + if (explicitPath) return path.resolve(explicitPath); + return output.replace(/\.glb$/i, ".json"); +} + +function previewOutputPath(output, explicitPath) { + if (explicitPath) return path.resolve(explicitPath); + return output.replace(/\.glb$/i, "-cesium-preview.html"); +} + +function writeCompressedMetadata(sourceMetadata, outputGlb, outputMetadata) { + const metadata = JSON.parse(fs.readFileSync(sourceMetadata, "utf8")); + const glbName = path.basename(outputGlb); + metadata.asset = glbName; + metadata.assets = [{ + id: "main", + label: "Compressed Scene", + type: "model", + url: glbName, + enabled: true, + }]; + if (typeof metadata.cesium_js === "string") { + metadata.cesium_js = metadata.cesium_js.replace( + /url: '[^']+\.glb'/, + `url: '${glbName}'`, + ); + } + fs.mkdirSync(path.dirname(outputMetadata), { recursive: true }); + fs.writeFileSync(outputMetadata, `${JSON.stringify(metadata, null, 2)}\n`); +} + +function readPreviewConfig(html) { + const match = html.match(PREVIEW_CONFIG_PATTERN); + if (!match) return null; + return JSON.parse(match[1]); +} + +function escapeScriptJson(value) { + return String(value) + .replaceAll("<", "\\u003c") + .replaceAll(">", "\\u003e") + .replaceAll("&", "\\u0026") + .replaceAll("\u2028", "\\u2028") + .replaceAll("\u2029", "\\u2029"); +} + +function writeCompressedPreview(sourcePreview, outputPreview, outputGlb, outputMetadata) { + const html = fs.readFileSync(sourcePreview, "utf8"); + const config = readPreviewConfig(html); + if (!config) { + throw new Error(`Preview config not found in ${sourcePreview}`); + } + config.glbName = path.basename(outputGlb); + config.metadataName = path.basename(outputMetadata); + const nextHtml = html + .replace( + PREVIEW_CONFIG_PATTERN, + `window.OSM_ASSET_PREVIEW_CONFIG = ${escapeScriptJson(JSON.stringify(config))};`, + ) + .replace(/Loading [^<]+\.glb\.\.\./, `Loading ${path.basename(outputGlb)}...`); + fs.mkdirSync(path.dirname(outputPreview), { recursive: true }); + fs.writeFileSync(outputPreview, nextHtml); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const input = path.resolve(requireText(args.input, "--input")); + const output = path.resolve(requireText(args.output, "--output")); + const textureSize = numberOption(args.textureSize, 1024, "--texture-size", 64, 4096); + const quality = numberOption(args.quality, 82, "--quality", 1, 100); + const effort = numberOption(args.effort, 80, "--effort", 0, 100); + const meshopt = args.meshopt === "true"; + + ensureFile(input, "Input GLB"); + if (input === output) { + throw new Error("--output must differ from --input"); + } + fs.mkdirSync(path.dirname(output), { recursive: true }); + + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "osm-glb-compress-")); + const resized = path.join(tempDir, "resized.glb"); + const webp = meshopt ? path.join(tempDir, "webp.glb") : output; + try { + const before = glbStats(input); + runGltfTransform([ + "resize", input, resized, + "--width", String(textureSize), + "--height", String(textureSize), + ]); + runGltfTransform([ + "webp", resized, webp, + "--quality", String(quality), + "--effort", String(effort), + ]); + if (meshopt) { + runGltfTransform(["meshopt", webp, output, "--level", "high"]); + } + + let metadataOutput = null; + let previewOutput = null; + if (args.metadata) { + const metadata = path.resolve(args.metadata); + ensureFile(metadata, "Source metadata"); + metadataOutput = metadataOutputPath(output, args.metadataOutput); + writeCompressedMetadata(metadata, output, metadataOutput); + } + if (args.preview) { + if (!metadataOutput) { + throw new Error("--preview requires --metadata so the copied preview has a metadata file"); + } + const preview = path.resolve(args.preview); + ensureFile(preview, "Source preview"); + previewOutput = previewOutputPath(output, args.previewOutput); + writeCompressedPreview(preview, previewOutput, output, metadataOutput); + } + + const after = glbStats(output); + console.log("GLB_COMPRESS_DONE " + JSON.stringify({ + input, + output, + texture_size: textureSize, + quality, + effort, + meshopt, + input_mb: mb(before.fileBytes), + output_mb: mb(after.fileBytes), + image_mb_before: mb(before.imageBytes), + image_mb_after: mb(after.imageBytes), + non_image_mb_before: mb(before.nonImageBytes), + non_image_mb_after: mb(after.nonImageBytes), + counts: after.counts, + extensions_used: after.extensionsUsed, + metadata: metadataOutput, + preview: previewOutput, + })); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +try { + main(); +} catch (error) { + console.error(error.message); + process.exit(1); +}