que01/osmWorkflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add GLB stage manifests

Browse Source
This commit is contained in:
que01
2026-08-04 09:45:51 +08:00
parent b7b5530430
commit 2832383f6e
17 changed files with 579 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
scripts/build-area.js
View File

@@ -4,6 +4,14 @@ const fs = require("fs");
const path = require("path");
const { spawnSync } = require("child_process");
const { readAreaConfig } = require("./lib/area-config");
const { digest: glbDigest } = require("./glb-digest");
const {
fileRecord,
glbBudgetWarnings,
glbSummary,
optionalFileRecord,
writeStageManifest,
} = require("./lib/stage-manifest");
const repoRoot = path.resolve(__dirname, "..");
const args = parseArgs(process.argv.slice(2));
@@ -206,6 +214,8 @@ function exportCesium(area) {
fs.mkdirSync(path.dirname(area.outputs.metadata), { recursive: true });
console.log("Stage: cesium");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(blenderExecutable(area), [
"--background",
"--python",
@@ -219,6 +229,30 @@ function exportCesium(area) {
area.outputs.metadata,
], "cesium");
writeCesiumPreview(area);
const finished = Date.now();
const digest = glbDigest(area.outputs.glb);
writeStageManifest(area, {
stage: "cesium",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
blend: fileRecord(area.outputs.blend),
},
outputs: {
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
vehicleRoute: optionalFileRecord(area.outputs.vehicleRoute),
vehicleModel: optionalFileRecord(area.outputs.vehicleModel),
},
summary: {
glb: glbSummary(digest),
},
warnings: glbBudgetWarnings("Cesium", digest),
});
}
function compressCesiumGlb(area) {
@@ -256,7 +290,43 @@ function compressCesiumGlb(area) {
}
console.log("Stage: compress (Cesium GLB texture resize + WebP)");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(process.execPath, compressArgs, "compress");
const finished = Date.now();
const sourceDigest = glbDigest(area.outputs.glb);
const compressedDigest = glbDigest(area.outputs.compressedGlb);
writeStageManifest(area, {
stage: "compress",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
},
outputs: {
compressedGlb: fileRecord(area.outputs.compressedGlb),
compressedMetadata: fileRecord(area.outputs.compressedMetadata),
compressedCesiumPreview: fileRecord(area.outputs.compressedCesiumPreview),
},
summary: {
sourceGlb: glbSummary(sourceDigest),
compressedGlb: glbSummary(compressedDigest),
options: {
textureSize: area.compress.textureSize,
quality: area.compress.quality,
effort: area.compress.effort,
meshopt: area.compress.meshopt,
},
compressionRatio: Number((compressedDigest.fileBytes / sourceDigest.fileBytes).toFixed(4)),
savedBytes: sourceDigest.fileBytes - compressedDigest.fileBytes,
},
warnings: glbBudgetWarnings("Compressed", compressedDigest),
});
}
function blenderExecutable(area) {

146
scripts/diagnose-area.js
View File

@@ -5,6 +5,7 @@ const fs = require("fs");
const path = require("path");
const { readAreaConfig } = require("./lib/area-config");
const { digest: glbDigest } = require("./glb-digest");
const { fileRecord, readStageManifest, stageManifestPath } = require("./lib/stage-manifest");
const repoRoot = path.resolve(__dirname, "..");
const DEFAULT_CONFIG = path.join(repoRoot, "config", "areas", "nantaizi-lake-innovation-valley.json");
@@ -293,7 +294,106 @@ function metadataSummary(file, warnings) {
}
}
function collectWarnings(area, osm, artifacts, glb, metadata) {
function stageManifestStatus(area) {
const stages = [
{
stage: "cesium",
expected: fs.existsSync(area.outputs.glb),
inputs: {
blend: area.outputs.blend,
},
outputs: {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
cesiumPreview: area.outputs.cesiumPreview,
},
},
{
stage: "compress",
expected: fs.existsSync(area.outputs.compressedGlb),
inputs: {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
cesiumPreview: area.outputs.cesiumPreview,
},
outputs: {
compressedGlb: area.outputs.compressedGlb,
compressedMetadata: area.outputs.compressedMetadata,
compressedCesiumPreview: area.outputs.compressedCesiumPreview,
},
},
];
return stages.map((entry) => {
const file = stageManifestPath(area, entry.stage);
try {
const manifest = readStageManifest(area, entry.stage);
if (!manifest) {
return {
stage: entry.stage,
path: file,
expected: entry.expected,
exists: false,
valid: false,
fresh: false,
manifestWarnings: [],
issues: entry.expected ? ["manifest missing"] : [],
};
}
const issues = [
...manifestFileIssues(manifest.inputs || {}, entry.inputs, "input"),
...manifestFileIssues(manifest.outputs || {}, entry.outputs, "output"),
];
return {
stage: entry.stage,
path: file,
expected: entry.expected,
exists: true,
valid: true,
fresh: issues.length === 0,
finishedAt: manifest.finishedAt || null,
durationMs: manifest.durationMs ?? null,
summary: manifest.summary || null,
manifestWarnings: Array.isArray(manifest.warnings) ? manifest.warnings : [],
issues,
};
} catch (error) {
return {
stage: entry.stage,
path: file,
expected: entry.expected,
exists: fs.existsSync(file),
valid: false,
fresh: false,
manifestWarnings: [],
issues: [`manifest unreadable: ${error.message}`],
};
}
});
}
function manifestFileIssues(records, expectedFiles, label) {
const issues = [];
for (const [key, file] of Object.entries(expectedFiles)) {
const recorded = records[key];
if (!recorded) {
issues.push(`${label} ${key} not recorded`);
continue;
}
if (!fs.existsSync(file)) {
issues.push(`${label} ${key} file missing`);
continue;
}
const current = fileRecord(file);
if (recorded.bytes !== current.bytes) {
issues.push(`${label} ${key} bytes changed`);
} else if (recorded.sha256 && recorded.sha256 !== current.sha256) {
issues.push(`${label} ${key} sha256 changed`);
}
}
return issues;
}
function collectWarnings(area, osm, artifacts, manifests, glb, metadata) {
const warnings = [];
if (!osm.bounds) warnings.push("OSM has no valid <bounds>; scene extent may be wrong.");
if (osm.ways.missingNodeRefs) {
@@ -312,6 +412,18 @@ function collectWarnings(area, osm, artifacts, glb, metadata) {
warnings.push(`Artifact has wrong type: ${artifact.label} (${artifact.path}).`);
}
}
for (const manifest of manifests) {
if (manifest.expected && !manifest.exists) {
warnings.push(`Expected stage manifest missing: ${manifest.stage} (${manifest.path}).`);
} else if (manifest.exists && !manifest.valid) {
warnings.push(`Stage manifest invalid: ${manifest.stage} (${manifest.issues.join("; ")}).`);
} else if (manifest.exists && !manifest.fresh) {
warnings.push(`Stage manifest stale: ${manifest.stage} (${manifest.issues.join("; ")}).`);
}
for (const warning of manifest.manifestWarnings) {
warnings.push(`Stage manifest warning (${manifest.stage}): ${warning}.`);
}
}
if (glb) {
if (glb.fileBytes > BUDGETS.glbBytes) {
warnings.push(`GLB size ${mb(glb.fileBytes)} MB exceeds budget ${mb(BUDGETS.glbBytes)} MB.`);
@@ -332,7 +444,7 @@ function collectWarnings(area, osm, artifacts, glb, metadata) {
return warnings;
}
function printReport(area, configPath, osm, artifacts, glb, metadata, warnings) {
function printReport(area, configPath, osm, artifacts, manifests, glb, metadata, warnings) {
console.log("Area diagnostics");
console.log(`Area: ${area.id}`);
console.log(`Config: ${configPath}`);
@@ -381,6 +493,31 @@ function printReport(area, configPath, osm, artifacts, glb, metadata, warnings)
if (metadata) {
console.log(` metadata asset: ${metadata.asset || "missing"}, assets: ${metadata.assets}`);
}
console.log("");
console.log("Stage manifests");
for (const manifest of manifests) {
let state = "absent";
if (manifest.exists && !manifest.valid) state = "invalid";
else if (manifest.exists && !manifest.fresh) state = "stale";
else if (manifest.exists) state = "ok";
else if (manifest.expected) state = "missing";
const timing = manifest.finishedAt
? `, finished ${manifest.finishedAt}, ${manifest.durationMs ?? "?"} ms`
: "";
const issues = manifest.issues.length ? `, ${manifest.issues.join("; ")}` : "";
console.log(` ${state.padEnd(7)} ${manifest.stage}: ${manifest.path}${timing}${issues}`);
for (const warning of manifest.manifestWarnings) {
console.log(` warning: ${warning}`);
}
const glbSummary = manifest.summary?.glb || manifest.summary?.compressedGlb;
if (glbSummary?.counts) {
console.log(
` GLB: ${formatBytes(glbSummary.fileBytes)}, ${glbSummary.counts.nodes} nodes, ` +
`${glbSummary.counts.meshes} meshes, ${glbSummary.counts.images} images`,
);
}
}
if (glb) {
console.log("");
console.log("GLB digest");
@@ -425,14 +562,15 @@ function main() {
const area = readAreaConfig(configPath, { repoRoot });
const osm = parseOsm(fs.readFileSync(area.input, "utf8"));
const artifacts = artifactStatus(area);
const manifests = stageManifestStatus(area);
const metadataWarnings = [];
const metadata = metadataSummary(area.outputs.metadata, metadataWarnings);
const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null;
const warnings = [
...metadataWarnings,
...collectWarnings(area, osm, artifacts, glb, metadata),
...collectWarnings(area, osm, artifacts, manifests, glb, metadata),
];
printReport(area, configPath, osm, artifacts, glb, metadata, warnings);
printReport(area, configPath, osm, artifacts, manifests, glb, metadata, warnings);
}
main();

4
scripts/lib/area-config.js
View File

@@ -25,6 +25,7 @@ function normalizeAreaConfig(raw, options = {}) {
const compress = normalizeCompressConfig(raw.compress);
const compressedFileStem = outputOverrides.compressedFileStem ||
`${fileStem}-compressed-webp${compress.textureSize}${compress.meshopt ? "-meshopt" : ""}`;
const pipelineDir = path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline"));
const outputs = {
areaDir,
geojsonDir: path.resolve(outputOverrides.geojsonDir || path.join(areaDir, "osm2streets_web_out")),
@@ -49,7 +50,8 @@ function normalizeAreaConfig(raw, options = {}) {
),
vehicleRoute: path.resolve(outputOverrides.vehicleRoute || path.join(areaDir, `${fileStem}-vehicle-route.json`)),
vehicleModel: path.resolve(outputOverrides.vehicleModel || path.join(areaDir, `${fileStem}-vehicle-car.gltf`)),
pipelineDir: path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline")),
pipelineDir,
stageManifestDir: path.resolve(outputOverrides.stageManifestDir || path.join(pipelineDir, "stages")),
};
return {

100
scripts/lib/stage-manifest.js Normal file
View File

@@ -0,0 +1,100 @@
"use strict";
const crypto = require("crypto");
const fs = require("fs");
const path = require("path");
const MANIFEST_VERSION = 1;
const BUDGETS = {
glbBytes: 25 * 1024 * 1024,
glbNodes: 1000,
glbImages: 24,
};
function stageManifestPath(area, stage) {
return path.join(area.outputs.stageManifestDir, `${stage}.manifest.json`);
}
function readStageManifest(area, stage) {
const file = stageManifestPath(area, stage);
if (!fs.existsSync(file)) return null;
return JSON.parse(fs.readFileSync(file, "utf8"));
}
function writeStageManifest(area, manifest) {
const file = stageManifestPath(area, manifest.stage);
const payload = {
manifestVersion: MANIFEST_VERSION,
area: area.id,
...manifest,
};
fs.mkdirSync(path.dirname(file), { recursive: true });
const temp = `${file}.tmp`;
fs.writeFileSync(temp, `${JSON.stringify(payload, null, 2)}\n`);
fs.renameSync(temp, file);
console.log(`Stage manifest: ${file}`);
return file;
}
function fileRecord(file) {
const stat = fs.statSync(file);
const record = {
path: file,
bytes: stat.size,
modifiedAt: stat.mtime.toISOString(),
};
if (stat.isFile()) {
record.sha256 = sha256(file);
}
return record;
}
function optionalFileRecord(file) {
if (!fs.existsSync(file)) return null;
return fileRecord(file);
}
function glbSummary(digest) {
if (!digest) return null;
return {
fileBytes: digest.fileBytes,
counts: digest.counts,
extensionsUsed: digest.extensionsUsed,
};
}
function glbBudgetWarnings(label, digest, budgets = BUDGETS) {
if (!digest) return [];
const warnings = [];
if (digest.fileBytes > budgets.glbBytes) {
warnings.push(`${label} GLB size ${mb(digest.fileBytes)} MB exceeds budget ${mb(budgets.glbBytes)} MB`);
}
if (digest.counts.nodes > budgets.glbNodes) {
warnings.push(`${label} GLB nodes ${digest.counts.nodes} exceed budget ${budgets.glbNodes}`);
}
if (digest.counts.images > budgets.glbImages) {
warnings.push(`${label} GLB images ${digest.counts.images} exceed budget ${budgets.glbImages}`);
}
return warnings;
}
function sha256(file) {
const hash = crypto.createHash("sha256");
hash.update(fs.readFileSync(file));
return hash.digest("hex");
}
function mb(bytes) {
return Number((bytes / 1024 / 1024).toFixed(2));
}
module.exports = {
BUDGETS,
fileRecord,
glbBudgetWarnings,
glbSummary,
optionalFileRecord,
readStageManifest,
stageManifestPath,
writeStageManifest,
};