perf: batch Cesium export mesh preparation
This commit is contained in:
1 parent
a4dfbf884a
commit
c6cce44f60
12 files changed
+632
-11
No files matched your search
@@ -0,0 +1,74 @@
|
||||
// Diagnostic: run from repository root with two export directory arguments.
|
||||
// Strict 1e-6 comparison; known control noise is reported, never suppressed.
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const assert = require('assert/strict');
|
||||
const crypto = require('crypto');
|
||||
const {digest} = require(path.join(process.cwd(), 'scripts/glb-digest'));
|
||||
function glb(file) {
|
||||
const data = fs.readFileSync(file);
|
||||
let json, bin;
|
||||
for (let i=12; i<data.length;) {
|
||||
const len=data.readUInt32LE(i), type=data.readUInt32LE(i+4);
|
||||
if(type===0x4e4f534a) json=JSON.parse(data.subarray(i+8,i+8+len));
|
||||
if(type===0x004e4942) bin=data.subarray(i+8,i+8+len);
|
||||
i+=8+len;
|
||||
}
|
||||
function accessor(id) {
|
||||
const a=json.accessors[id];
|
||||
assert(!a.sparse, 'sparse accessor unsupported');
|
||||
const v=json.bufferViews[a.bufferView];
|
||||
const n={SCALAR:1,VEC2:2,VEC3:3,VEC4:4,MAT4:16}[a.type];
|
||||
const [bytes,fn]={5120:[1,'readInt8'],5121:[1,'readUInt8'],5122:[2,'readInt16LE'],5123:[2,'readUInt16LE'],5125:[4,'readUInt32LE'],5126:[4,'readFloatLE']}[a.componentType];
|
||||
const out=[];
|
||||
for(let i=0;i<a.count;i++) for(let j=0;j<n;j++) {
|
||||
out.push(bin[fn]((v.byteOffset||0)+(a.byteOffset||0)+i*(v.byteStride||n*bytes)+j*bytes));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
function imageHash(image) {
|
||||
const v=json.bufferViews[image.bufferView];
|
||||
return crypto.createHash('sha256').update(bin.subarray(v.byteOffset||0,(v.byteOffset||0)+v.byteLength)).digest('hex');
|
||||
}
|
||||
return {json, accessor, imageHash};
|
||||
}
|
||||
const [before,after]=process.argv.slice(2);
|
||||
let failures=0;
|
||||
for(const file of fs.readdirSync(before).filter(f=>f.endsWith('.glb')).sort()) {
|
||||
const issues=[];
|
||||
try {
|
||||
const da=digest(path.join(before,file)), db=digest(path.join(after,file));
|
||||
for(const d of [da,db]) {delete d.fileBytes;delete d.buffers;delete d.counts.accessors;}
|
||||
try {assert.deepEqual(da,db);} catch {issues.push('structural digest differs');}
|
||||
const a=glb(path.join(before,file)),b=glb(path.join(after,file));
|
||||
for(const key of ['nodes','scenes','scene','materials','textures','samplers','animations','skins','extensions','extensionsUsed','extensionsRequired']) {
|
||||
assert.deepEqual(a.json[key],b.json[key],key);
|
||||
}
|
||||
assert.equal(a.json.meshes.length,b.json.meshes.length);
|
||||
let maxError=0;
|
||||
for(let m=0;m<a.json.meshes.length;m++) {
|
||||
const am=a.json.meshes[m],bm=b.json.meshes[m];
|
||||
assert.equal(am.name,bm.name);
|
||||
assert.equal(am.primitives.length,bm.primitives.length);
|
||||
for(let p=0;p<am.primitives.length;p++) {
|
||||
const ap=am.primitives[p],bp=bm.primitives[p];
|
||||
assert.equal(ap.material,bp.material);
|
||||
assert.deepEqual(Object.keys(ap.attributes),Object.keys(bp.attributes));
|
||||
for(const key of ['indices',...Object.keys(ap.attributes)]) {
|
||||
const av=a.accessor(key==='indices'?ap.indices:ap.attributes[key]);
|
||||
const bv=b.accessor(key==='indices'?bp.indices:bp.attributes[key]);
|
||||
if(av.length!==bv.length) {issues.push(`${am.name}/${p}/${key}: count ${av.length} -> ${bv.length}`);continue;}
|
||||
let error=0;
|
||||
for(let i=0;i<av.length;i++) error=Math.max(error,Math.abs(av[i]-bv[i]));
|
||||
maxError=Math.max(maxError,error);
|
||||
if(error>(key==='indices'?0:1e-6)) issues.push(`${am.name}/${p}/${key}: max delta ${error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.equal(a.json.images?.length||0,b.json.images?.length||0);
|
||||
for(let i=0;i<(a.json.images||[]).length;i++) assert.equal(a.imageHash(a.json.images[i]),b.imageHash(b.json.images[i]));
|
||||
console.log(file, issues.length?'FAIL':'PASS', 'max attribute delta',maxError,issues.slice(0,12));
|
||||
} catch(e) {issues.push(e.message);console.log(file,'ERROR',e.message.slice(0,500));}
|
||||
if(issues.length) failures++;
|
||||
}
|
||||
process.exitCode=failures?1:0;
|
||||
@@ -0,0 +1,48 @@
|
||||
# Performance and parity evidence (2026-09-17)
|
||||
|
||||
Environment: local Blender 4.5.12, same existing `.blend` per region, serial runs, all outputs in `/private/tmp/cesium-profile.9ODFNa`. Original exporter preserved there as `original_export_cesium.py`. No source scene or existing output overwritten.
|
||||
|
||||
## Measurements
|
||||
|
||||
| Input / implementation | Export function wall time |
|
||||
| --- | ---: |
|
||||
| Nantaizi original cProfile | 179.596 s |
|
||||
| Nantaizi original control without profiler | 173.227 s |
|
||||
| Nantaizi initial production candidate | 33.389 s |
|
||||
| Nantaizi candidate with shared-mesh guards | 15.337 s |
|
||||
| Nantaizi final fake-user guard correction | 15.905 s |
|
||||
| Fengshu original | 41.396 s |
|
||||
| Fengshu original second control | 36.891 s |
|
||||
| Fengshu optimized | 5.527 s |
|
||||
| Fengshu final fake-user guard correction | 5.638 s |
|
||||
|
||||
Time excludes Blender process startup; `measure.py` uses perf_counter around export() and per-function timing wrappers, no cProfile. Inputs/outputs and eight/seven export calls unchanged. Final Nantaizi speedup 10.89x; Fengshu 6.54–7.34x. Final Nantaizi eight GLBs again pass with zero decoded attribute delta. Both regions' metadata matches after normalizing output directory prefixes.
|
||||
|
||||
## Nantaizi parity
|
||||
|
||||
Unchanged control compared with original profiler output: all eight GLBs have identical project structural digests (existing unstable fields excluded), decoded indices/positions/UVs/normals/tangents/other attributes, full nodes/scenes/materials/textures/samplers and embedded image hashes. Maximum numeric delta 0. Initial and shared-guard production candidates both pass these exact comparisons.
|
||||
|
||||
## Fengshu control noise
|
||||
|
||||
All seven project structural digests match. Six GLBs' decoded attributes also match exactly. `vegetation.glb`, mesh `Tree_Natural_Crowns_DarkMesh`, TANGENT values vary by at most 0.00010001659393310547 between two **unmodified** runs. Optimized versus first baseline has the same maximum; versus second baseline 0.00010000169277191162. Positions, UVs, normals, indices, other meshes and embedded images match. This is not a broad tolerance waiver.
|
||||
|
||||
Installed `io_scene_gltf2/blender/exp/primitive_extract.py:1507` rounds tangents to `ROUNDING_DIGIT`, defined as 4 in `io/com/constants.py:159`. A 0.0001 difference is one exporter quantization step. Existing project digest ignores accessor/buffer layout noise, but no project ignore list was expanded for this work.
|
||||
|
||||
## Edge fixtures
|
||||
|
||||
`blender/tools/check_export_preparation.py` passed in real Blender using original exporter as reference and an unchanged control. Includes asymmetric quad/ngon triangle counts, authored UVs, shared meshes, custom normals, shape keys, Bevel, dependent Boolean, and fake retention user. Compares topology/vertex order/UV/normals/material slots/mesh names/custom properties/sharing/remaining modifiers. Uses 1e-6 numerical tolerance.
|
||||
|
||||
Independent review moved edit-mode selection into the cleanup try/finally and added an injected UV failure fixture asserting restoration of original scene/view layer/collection links and no scene leak. Expanded fixture passed after review. All 64 existing pure Python tests, Python compilation and `git diff --check` pass. No project Python lint/type-check configuration exists. No UI screenshot claim: visual semantics checked through decoded render attributes and identical image bytes; no browser rendering code changed.
|
||||
|
||||
Reproducible strict comparison: from repo root run `node .trellis/tasks/09-17-cesium-export-performance/research/compare-glb.cjs <baseline-directory> <candidate-directory>`. It reports the Fengshu control tangent noise as a failure at 1e-6, deliberately not masking it. Classify that one known result using the unchanged control evidence above; all project structural digest checks pass.
|
||||
|
||||
## Source-backed choices
|
||||
|
||||
- Public selection data API avoids operator entry/exit graph updates.
|
||||
- Empty modifier stack returns before any selection/update.
|
||||
- Same Triangulate modifier is evaluated in a batch; evaluated meshes copied before mutating original data; preserve all data layers. No BMesh approximation.
|
||||
- Same UV operator runs one object at a time in a small temporary scene, switched through actual window.scene; normal graph updates stay enabled.
|
||||
- Shared triangulation skipped using `ID_REAL_USERS = users - fake_user`, matching Blender's failure precondition.
|
||||
- Unknown modifier stacks retain original ordering; no global monkey patch.
|
||||
|
||||
Failed temp_override-only UV investigation was stopped because Blender edit-mode poll requires consistent window scene context; production uses a real temporary scene/window switch instead. Diagnostic global update suppression and mismatched direct-BMesh experiment are not in production.
|
||||
@@ -0,0 +1,82 @@
|
||||
# Research: Triangulation semantics and output-preserving optimization
|
||||
|
||||
- Query: Explain the temporary direct-BMesh experiment's different GLB vertex counts and identify source-backed faster alternatives.
|
||||
- Scope: mixed (project source, installed exporter, official Blender 4.5.0 source)
|
||||
- Date: 2026-09-17
|
||||
|
||||
## Findings
|
||||
|
||||
### Confirmed algorithm mismatch in the experiment
|
||||
|
||||
`blender/export_cesium.py:531` creates a Triangulate modifier and only overrides `min_vertices = 4`. The temporary `/private/tmp/cesium-profile.9ODFNa/profile_export.py:42` instead passes `quad_method='BEAUTY', ngon_method='BEAUTY'` to BMesh.
|
||||
|
||||
Blender **4.5.0** `source/blender/makesdna/DNA_modifier_defaults.h:643-649` defines the modifier defaults as:
|
||||
|
||||
```
|
||||
flag = 0
|
||||
quad_method = MOD_TRIANGULATE_QUAD_SHORTEDGE
|
||||
ngon_method = MOD_TRIANGULATE_NGON_BEAUTY
|
||||
min_vertices = 4
|
||||
```
|
||||
|
||||
Thus the experiment is not algorithm-equivalent: quads use shortest diagonal in the baseline and beauty in the experiment. This is a concrete reason to correct the experiment, not speculative parameter tuning. Do not rely on older Blender documentation's defaults; read a temporary modifier's RNA properties on the actual running version if supporting multiple Blender versions.
|
||||
|
||||
Both routes ultimately call `BM_mesh_triangulate` (official `MOD_triangulate.cc`, `bmo_triangulate.cc:29-44`). The modifier passes `tag_only=false`; the BMesh operator tags its supplied faces and passes `tag_only=true`. Supplying all eligible faces and matching quad/ngon methods aligns the core algorithm.
|
||||
|
||||
### Why GLB vertex counts can differ with equal triangles
|
||||
|
||||
Installed `/Applications/Blender.app/Contents/Resources/4.5/scripts/addons_core/io_scene_gltf2/blender/exp/primitive_extract.py:336-344` explicitly explains that Blender corner attributes are packed into per-vertex records for glTF. Deduplication at lines 825 and 900-901 uses the attribute record, not position alone. Normals, tangents and UV discontinuities therefore split vertices without changing triangle count. Changing the quad diagonal can change subsequent normals and UV unwrap even when all positions and triangle totals remain unchanged.
|
||||
|
||||
Project preparation order (`blender/export_cesium.py:670-676`) is triangulate then unwrap. Hence changing topology before `smart_project` can also change UV islands. Counts alone do not establish visual parity or identify which attribute actually changed in the existing artifacts; inspect decoded attribute arrays per mesh to attribute the observed count difference definitively.
|
||||
|
||||
### Remaining BMesh conversion differences
|
||||
|
||||
Official `MOD_triangulate.cc` sets `calc_face_normal=true`, `calc_vert_normal=false` and carries original-index custom-data masks through conversion. Python `BMesh.from_mesh` (`bmesh_py_types.cc:1376-1414`) defaults both normal flags to true; pass `face_normals=True, vertex_normals=False` to match the modifier's inputs more closely.
|
||||
|
||||
Modifier `flag=0` means **keep_custom_normals is false by default in 4.5.0**. Therefore do not claim custom-normal preservation is the established reason for this scene's discrepancy. If enabled, the modifier explicitly copies evaluated corner normals into a temporary CD_NORMAL layer and restores normalized custom normals after triangulation. Direct BMesh does not reproduce that explicit procedure automatically.
|
||||
|
||||
Python `BMesh.to_mesh` (`bmesh_py_types.cc:1201-1240`) uses `BM_mesh_bm_to_me`, updates shape-key indices/object remaps and tags geometry dirty; modifier output uses `BKE_mesh_from_bmesh_for_eval_nomain`. These are distinct conversion paths. Matching triangulation settings does not constitute a proof of byte-identical mesh output, ordering or custom data. Preserve the baseline fallback for multi-user meshes (`modifier_apply` refuses them); do not change shared topology as a performance side effect.
|
||||
|
||||
### Strongest semantic route: keep the actual modifier, batch evaluation
|
||||
|
||||
For simple single-user meshes without pre-existing modifiers or shape keys, add the same Triangulate modifier through the data API for all eligible objects, evaluate the dependency graph once, then obtain persistent meshes with `bpy.data.meshes.new_from_object(obj.evaluated_get(depsgraph), preserve_all_data_layers=True, depsgraph=depsgraph)`. Stage every result before replacing source data/removing modifiers, because mutation can invalidate evaluated state. This executes the actual modifier implementation instead of duplicating it in Python.
|
||||
|
||||
Official `rna_main_api.cc:1103-1123` says an original object produces undeformed geometry while an evaluated object produces final geometry with all modifiers; preserving all data layers requires a dependency graph. Official `rna_ID.cc:2447-2452` says `evaluated_get` only retrieves the **last** evaluation and does **not** ensure the graph is up to date. An explicit graph evaluation barrier after adding modifiers is mandatory; repeated evaluation per object defeats the optimization.
|
||||
|
||||
Limit this fast path to proven equivalent cases. Existing modifier stacks, hidden modifiers, failing modifiers, shape keys, shared meshes and inter-object dependencies are not generally equivalent to the current sequential `modifier_apply` loop. Keep the existing operator fallback for those cases until separately justified. New datablocks must preserve material slots, required custom properties and semantic names; parity must detect unintended instance splitting or GLB naming differences.
|
||||
|
||||
### Recommended decision
|
||||
|
||||
1. Correct the demonstrated quad-method mismatch before drawing conclusions from direct-BMesh measurements.
|
||||
2. Prefer batched evaluation of the original modifier if exact parity remains the requirement; its geometric semantics have a stronger equivalence basis.
|
||||
3. Keep graph updates at real dependency boundaries; never ship the temporary global `_view_layer_update` monkey patch.
|
||||
4. Require decoded triangle attributes and existing project structure parity, including UVs/normals/tangents and dynamic/shared assets. Do not accept matching counts as sufficient evidence.
|
||||
|
||||
## Files found
|
||||
|
||||
- `blender/export_cesium.py` — preparation order and baseline modifier/UV operations.
|
||||
- `/private/tmp/cesium-profile.9ODFNa/profile_export.py` — temporary monkey-patch and direct-BMesh experiment.
|
||||
- `/Applications/Blender.app/Contents/Resources/4.5/scripts/addons_core/io_scene_gltf2/blender/exp/primitive_extract.py` — glTF corner-attribute splitting and deduplication.
|
||||
|
||||
## External references
|
||||
|
||||
All source references pinned to Blender `v4.5.0`:
|
||||
|
||||
- https://github.com/blender/blender/blob/v4.5.0/source/blender/makesdna/DNA_modifier_defaults.h#L643
|
||||
- https://github.com/blender/blender/blob/v4.5.0/source/blender/modifiers/intern/MOD_triangulate.cc
|
||||
- https://github.com/blender/blender/blob/v4.5.0/source/blender/bmesh/operators/bmo_triangulate.cc
|
||||
- https://github.com/blender/blender/blob/v4.5.0/source/blender/python/bmesh/bmesh_py_types.cc#L1376
|
||||
- https://github.com/blender/blender/blob/v4.5.0/source/blender/makesrna/intern/rna_main_api.cc#L1103
|
||||
- https://github.com/blender/blender/blob/v4.5.0/source/blender/makesrna/intern/rna_ID.cc#L2447
|
||||
|
||||
## Related specs
|
||||
|
||||
- `.trellis/spec/blender/index.md` — Blender/standard-Python boundary and parity requirements.
|
||||
- `.trellis/spec/blender/asset-generation.md` — deterministic geometry, shared meshes and dynamic export assets.
|
||||
- `.trellis/spec/blender/testing.md` — mathematical expectations and real Blender parity.
|
||||
|
||||
## Caveats / Not Found
|
||||
|
||||
- Research only: no Blender runs or changes to product code were performed.
|
||||
- The algorithm mismatch is proven from source; the exact per-attribute cause of every previously observed vertex-count change is not measured here.
|
||||
- Batch evaluated-mesh conversion is a source-backed candidate, not a claim of verified output parity or measured speedup.
|
||||
Reference in new issue
Block a user