refactor: 抽 osmassets 包，要素注册表，材质单一定义源 (P0-P3)

generate_scene.py 从 1271 → 816 行 (-455) P0: 纯函数搬家 - osmassets/osm.py: parse_osm / Projector / parse_height - osmassets/geom.py: clip_polygon / point_in_polygon / distance_to_ring / sample_tree_row 等 - blender/tests/test_pure.py: 42 个 unittest (脱离 bpy 运行) P1: 单一定义源 - osmassets/catalog.py: ROAD_LAYERS + MATERIALS (含 cesium 导出参数) - 对接 osm2streets_scene_style.json 做图层一致性 warning - 干掉 road_mats / layer_z / 材质参数三份副本 P2: 要素注册表 - osmassets/{water,grass,scrub}.py: 每个要素一个 assemble() 函数 - build() 中的 if/elif 链收缩为注册表调用 - 计数器集中到 counts 字典 P3: 材质契约化 - catalog.py 扩展 CESIUM_EXPORT 段 (tint/metallic/emission) - 标记已发现的死条目 Office White Metal Facade (四表各一组) 校验: - parity.js + scene_digest.py + glb-digest.js 三位一体 - control-1 vs p0/p1/p2a/p2b/p3-counts: 两区域全 PARITY OK - 42 个纯 Python 测试全部通过 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-29 17:59:32 +08:00
parent 23ae63bc2a
commit 24e02e2041
16 changed files with 1924 additions and 598 deletions
--- a/scripts/glb-digest.js
+++ b/scripts/glb-digest.js
@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+"use strict";
+
+// Structural digest of a GLB, for the osmassets refactor parity check.
+//
+// Byte-comparing the GLB is too strict: Blender packs images in hash-map order
+// and the buffer padding shifts with it, so two runs of identical code can
+// differ.  This reads the glTF JSON chunk instead and reports the parts that
+// carry meaning downstream in Cesium — node/mesh/material identity and PBR
+// values — plus buffer lengths as a coarse size check.
+//
+//   node scripts/glb-digest.js <file.glb> [--out digest.json]
+
+const fs = require("fs");
+const path = require("path");
+
+function readGlbJson(file) {
+  const buffer = fs.readFileSync(file);
+  if (buffer.length < 12 || buffer.readUInt32LE(0) !== 0x46546c67) {
+    throw new Error(`Not a GLB (bad magic): ${file}`);
+  }
+  const total = buffer.readUInt32LE(8);
+  let offset = 12;
+  while (offset + 8 <= Math.min(total, buffer.length)) {
+    const chunkLength = buffer.readUInt32LE(offset);
+    const chunkType = buffer.readUInt32LE(offset + 4);
+    const start = offset + 8;
+    if (chunkType === 0x4e4f534a) {
+      return JSON.parse(buffer.slice(start, start + chunkLength).toString("utf8"));
+    }
+    offset = start + chunkLength;
+  }
+  throw new Error(`No JSON chunk found in ${file}`);
+}
+
+function round(value) {
+  if (typeof value === "number") return Number(value.toFixed(6));
+  if (Array.isArray(value)) return value.map(round);
+  return value;
+}
+
+function materialDigest(material) {
+  const pbr = material.pbrMetallicRoughness || {};
+  return {
+    name: material.name || null,
+    baseColorFactor: round(pbr.baseColorFactor || null),
+    metallicFactor: round(pbr.metallicFactor ?? null),
+    roughnessFactor: round(pbr.roughnessFactor ?? null),
+    hasBaseColorTexture: Boolean(pbr.baseColorTexture),
+    hasNormalTexture: Boolean(material.normalTexture),
+    emissiveFactor: round(material.emissiveFactor || null),
+    emissiveStrength: round(
+      material.extensions?.KHR_materials_emissive_strength?.emissiveStrength ?? null,
+    ),
+    alphaMode: material.alphaMode || null,
+    doubleSided: material.doubleSided ?? null,
+  };
+}
+
+function digest(file) {
+  const gltf = readGlbJson(file);
+  const meshes = (gltf.meshes || []).map((mesh) => ({
+    name: mesh.name || null,
+    primitives: (mesh.primitives || []).map((primitive) => ({
+      material: primitive.material ?? null,
+      attributes: Object.keys(primitive.attributes || {}).sort(),
+      // Vertex/index counts live on the accessors; they are the real geometry
+      // fingerprint and stay stable regardless of buffer layout.
+      count: gltf.accessors?.[primitive.attributes?.POSITION]?.count ?? null,
+      indices: gltf.accessors?.[primitive.indices]?.count ?? null,
+    })),
+  }));
+  return {
+    file: path.basename(file),
+    fileBytes: fs.statSync(file).size,
+    counts: {
+      nodes: (gltf.nodes || []).length,
+      meshes: meshes.length,
+      materials: (gltf.materials || []).length,
+      images: (gltf.images || []).length,
+      accessors: (gltf.accessors || []).length,
+    },
+    extensionsUsed: (gltf.extensionsUsed || []).slice().sort(),
+    buffers: (gltf.buffers || []).map((buffer) => buffer.byteLength),
+    nodes: (gltf.nodes || [])
+      .map((node) => ({
+        name: node.name || null,
+        mesh: node.mesh ?? null,
+        translation: round(node.translation || null),
+        rotation: round(node.rotation || null),
+        scale: round(node.scale || null),
+        extras: node.extras ?? null,
+      }))
+      .sort((a, b) => String(a.name).localeCompare(String(b.name))),
+    meshes: meshes.sort((a, b) => String(a.name).localeCompare(String(b.name))),
+    materials: (gltf.materials || [])
+      .map(materialDigest)
+      .sort((a, b) => String(a.name).localeCompare(String(b.name))),
+    images: (gltf.images || [])
+      .map((image) => ({ name: image.name || null, mimeType: image.mimeType || null }))
+      .sort((a, b) => String(a.name).localeCompare(String(b.name))),
+  };
+}
+
+const argv = process.argv.slice(2);
+const file = argv.find((arg) => !arg.startsWith("--"));
+if (!file) {
+  console.error("usage: node scripts/glb-digest.js <file.glb> [--out digest.json]");
+  process.exit(1);
+}
+const outIndex = argv.indexOf("--out");
+const result = digest(path.resolve(file));
+const text = `${JSON.stringify(result, null, 2)}\n`;
+if (outIndex >= 0 && argv[outIndex + 1]) {
+  const out = path.resolve(argv[outIndex + 1]);
+  fs.mkdirSync(path.dirname(out), { recursive: true });
+  fs.writeFileSync(out, text);
+  console.log(`GLB digest: ${out}`);
+} else {
+  process.stdout.write(text);
+}
--- a/scripts/parity.js
+++ b/scripts/parity.js
@@ -0,0 +1,270 @@
+#!/usr/bin/env node
+"use strict";
+
+// Parity harness for the osmassets refactor (docs/refactor-plan.md).
+//
+//   node scripts/parity.js capture <label> [--areas a,b] [--stages blender,cesium]
+//   node scripts/parity.js compare <labelA> <labelB>
+//
+// capture runs the pipeline and snapshots everything that must not change:
+// the stage stdout markers, a structural digest of the .blend, a structural
+// digest of the .glb, and a hash of the render PNG.  compare diffs two
+// snapshots field by field.
+//
+// Snapshots live under outputs/_refactor-baseline/<label>/, which is inside
+// the gitignored outputs/ tree — baselines are local scratch, not artifacts.
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const { spawnSync } = require("child_process");
+
+const repoRoot = path.resolve(__dirname, "..");
+const baselineRoot = path.join(repoRoot, "outputs", "_refactor-baseline");
+const DEFAULT_AREAS = ["nantaizi-lake-innovation-valley", "hanyang-block"];
+
+// Fields a control run (identical code, run twice) proved unstable.  They are
+// still recorded — a human reading a snapshot wants them — but comparing them
+// would bury real regressions under noise.
+//
+//   files.*.sha256 / .bytes for blend, glb, render
+//     .blend embeds absolute paths and packs images in hash-map order, so the
+//     file hash moves while the structural digest stays put.  The EEVEE render
+//     is likewise not bit-reproducible.
+//   glbDigest.fileBytes / .buffers / .counts.accessors
+//     The glTF exporter deduplicates identical accessors.  smart_project UVs
+//     carry float noise, so two runs can differ by one shared UV accessor
+//     (observed: 399 vs 398 accessors, 720 bytes) with identical nodes,
+//     meshes, primitives, materials and images.
+//
+// What remains compared is the real contract: the SCENE_DONE / CESIUM markers,
+// the full .blend structural digest (objects, meshes, materials, custom
+// properties), and the GLB node/mesh/material/image structure.
+const IGNORED_PATHS = new Set([
+  "capturedAt",
+  "durationMs",
+  "label",
+  "files.blend.sha256",
+  "files.glb.sha256",
+  "files.glb.bytes",
+  "files.render.sha256",
+  "files.render.bytes",
+  "glbDigest.fileBytes",
+  "glbDigest.buffers",
+  "glbDigest.counts.accessors",
+]);
+
+function main() {
+  const [command, ...rest] = process.argv.slice(2);
+  if (command === "capture") return capture(rest);
+  if (command === "compare") return compare(rest);
+  console.error("usage: parity.js capture <label> [--areas a,b] [--stages s]");
+  console.error("       parity.js compare <labelA> <labelB>");
+  process.exit(1);
+}
+
+function parseFlags(argv) {
+  const flags = {};
+  const positional = [];
+  for (let i = 0; i < argv.length; i += 1) {
+    if (argv[i].startsWith("--") && i + 1 < argv.length) {
+      flags[argv[i].slice(2)] = argv[i + 1];
+      i += 1;
+    } else {
+      positional.push(argv[i]);
+    }
+  }
+  return { flags, positional };
+}
+
+function capture(argv) {
+  const { flags, positional } = parseFlags(argv);
+  const label = positional[0];
+  if (!label) throw new Error("capture needs a label");
+  const areas = (flags.areas ? flags.areas.split(",") : DEFAULT_AREAS)
+    .map((a) => a.trim())
+    .filter(Boolean);
+  const stages = flags.stages || "blender,cesium";
+
+  for (const area of areas) {
+    const configPath = path.join(repoRoot, "config", "areas", `${area}.json`);
+    if (!fs.existsSync(configPath)) throw new Error(`No config for area: ${area}`);
+    const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
+    const areaDir = path.join(repoRoot, "outputs", area);
+    const stem = area;
+    const outDir = path.join(baselineRoot, label, area);
+    fs.mkdirSync(outDir, { recursive: true });
+
+    console.log(`\n=== parity capture [${label}] ${area} (stages: ${stages}) ===`);
+    const started = Date.now();
+    const run = spawnSync(process.execPath, [
+      path.join(repoRoot, "scripts", "build-area.js"),
+      "--config", configPath,
+      "--stages", stages,
+    ], { cwd: repoRoot, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+    const stdout = `${run.stdout || ""}`;
+    const stderr = `${run.stderr || ""}`;
+    process.stdout.write(stdout);
+    if (run.status !== 0) {
+      process.stderr.write(stderr);
+      throw new Error(`build-area failed for ${area} (exit ${run.status})`);
+    }
+
+    const snapshot = {
+      label,
+      area,
+      stages,
+      capturedAt: new Date().toISOString(),
+      durationMs: Date.now() - started,
+      markers: {
+        scene: parseMarker(stdout, "SCENE_DONE"),
+        cesium: parseMarker(stdout, "CESIUM_EXPORT_DONE"),
+      },
+      files: {},
+    };
+
+    const blend = path.join(areaDir, `${stem}.blend`);
+    if (fs.existsSync(blend)) {
+      snapshot.files.blend = fileStat(blend);
+      snapshot.blendDigest = blendDigest(config, blend, path.join(outDir, "blend-digest.json"));
+    }
+    const glb = path.join(areaDir, `${stem}.glb`);
+    if (fs.existsSync(glb)) {
+      snapshot.files.glb = fileStat(glb);
+      snapshot.glbDigest = glbDigest(glb, path.join(outDir, "glb-digest.json"));
+    }
+    for (const [key, file] of [
+      ["render", path.join(areaDir, `${stem}.png`)],
+      ["metadata", path.join(areaDir, `${stem}.json`)],
+    ]) {
+      if (fs.existsSync(file)) snapshot.files[key] = fileStat(file);
+    }
+    if (fs.existsSync(path.join(areaDir, `${stem}.json`))) {
+      snapshot.metadata = JSON.parse(
+        fs.readFileSync(path.join(areaDir, `${stem}.json`), "utf8"),
+      );
+    }
+
+    writeJson(path.join(outDir, "snapshot.json"), snapshot);
+    console.log(`Snapshot: ${path.join(outDir, "snapshot.json")}`);
+  }
+}
+
+function parseMarker(stdout, marker) {
+  const line = stdout.split("\n").find((l) => l.startsWith(`${marker} `));
+  if (!line) return null;
+  try {
+    return JSON.parse(line.slice(marker.length + 1));
+  } catch (error) {
+    return { unparsed: line };
+  }
+}
+
+function fileStat(file) {
+  const buffer = fs.readFileSync(file);
+  return {
+    bytes: buffer.length,
+    sha256: crypto.createHash("sha256").update(buffer).digest("hex"),
+  };
+}
+
+function blendDigest(config, blend, outFile) {
+  const blenderApp = config.blenderApp || "/Applications/Blender.app";
+  const blender = path.join(blenderApp, "Contents", "MacOS", "Blender");
+  const run = spawnSync(blender, [
+    "--background", "--factory-startup",
+    "--python", path.join(repoRoot, "blender", "tools", "scene_digest.py"),
+    "--", "--blend", blend, "--out", outFile,
+  ], { cwd: repoRoot, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+  if (run.status !== 0) {
+    process.stderr.write(`${run.stdout || ""}${run.stderr || ""}`);
+    throw new Error(`scene_digest failed for ${blend}`);
+  }
+  return JSON.parse(fs.readFileSync(outFile, "utf8"));
+}
+
+function glbDigest(glb, outFile) {
+  const run = spawnSync(process.execPath, [
+    path.join(repoRoot, "scripts", "glb-digest.js"), glb, "--out", outFile,
+  ], { cwd: repoRoot, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+  if (run.status !== 0) {
+    process.stderr.write(`${run.stdout || ""}${run.stderr || ""}`);
+    throw new Error(`glb-digest failed for ${glb}`);
+  }
+  return JSON.parse(fs.readFileSync(outFile, "utf8"));
+}
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function compare(argv) {
+  const [a, b] = argv;
+  if (!a || !b) throw new Error("compare needs two labels");
+  const areas = fs.readdirSync(path.join(baselineRoot, a))
+    .filter((entry) => fs.existsSync(path.join(baselineRoot, a, entry, "snapshot.json")));
+
+  let differences = 0;
+  for (const area of areas) {
+    const left = readSnapshot(a, area);
+    const right = readSnapshot(b, area);
+    if (!right) {
+      console.log(`\n[${area}] missing in ${b} — skipped`);
+      continue;
+    }
+    const diffs = [];
+    diffValues("", left, right, diffs);
+    console.log(`\n=== ${area}: ${a} vs ${b} ===`);
+    if (!diffs.length) {
+      console.log("identical");
+    } else {
+      differences += diffs.length;
+      for (const line of diffs.slice(0, 200)) console.log(line);
+      if (diffs.length > 200) console.log(`… ${diffs.length - 200} more`);
+    }
+  }
+  console.log(`\n${differences === 0 ? "PARITY OK" : `PARITY DIFF (${differences})`}`);
+  process.exitCode = differences === 0 ? 0 : 2;
+}
+
+function readSnapshot(label, area) {
+  const file = path.join(baselineRoot, label, area, "snapshot.json");
+  return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : null;
+}
+
+function diffValues(pathKey, left, right, out) {
+  if (IGNORED_PATHS.has(pathKey)) return;
+  if (left === right) return;
+  const bothObjects = left && right && typeof left === "object" && typeof right === "object";
+  if (!bothObjects) {
+    out.push(`  ${pathKey || "<root>"}: ${format(left)} -> ${format(right)}`);
+    return;
+  }
+  if (Array.isArray(left) !== Array.isArray(right)) {
+    out.push(`  ${pathKey}: array/object mismatch`);
+    return;
+  }
+  if (Array.isArray(left)) {
+    if (left.length !== right.length) {
+      out.push(`  ${pathKey}.length: ${left.length} -> ${right.length}`);
+    }
+    const limit = Math.min(left.length, right.length);
+    for (let i = 0; i < limit; i += 1) {
+      diffValues(`${pathKey}[${i}]`, left[i], right[i], out);
+    }
+    return;
+  }
+  const keys = new Set([...Object.keys(left), ...Object.keys(right)]);
+  for (const key of [...keys].sort()) {
+    diffValues(pathKey ? `${pathKey}.${key}` : key, left[key], right[key], out);
+  }
+}
+
+function format(value) {
+  if (value === undefined) return "<missing>";
+  const text = JSON.stringify(value);
+  return text && text.length > 120 ? `${text.slice(0, 117)}…` : text;
+}
+
+main();