que01/osmWorkflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pipeline): make compressed assets the default delivery

Browse Source
This commit is contained in:
que01
2026-08-11 15:30:04 +08:00
parent b68be063ad
commit 0790cbd0a9
11 changed files with 143 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
scripts/build-area.js
View File

@@ -347,74 +347,82 @@ function compressCesiumGlb(area) {
ensureFile(area.outputs.metadata, "Cesium metadata");
ensureFile(area.outputs.cesiumPreview, "Cesium preview");
ensureFile(path.join(repoRoot, "scripts", "compress-glb.js"), "GLB compressor");
fs.mkdirSync(path.dirname(area.outputs.compressedGlb), { recursive: true });
fs.mkdirSync(path.dirname(area.outputs.compressedMetadata), { recursive: true });
fs.mkdirSync(path.dirname(area.outputs.compressedCesiumPreview), { recursive: true });
const compressArgs = [
path.join(repoRoot, "scripts", "compress-glb.js"),
"--input",
area.outputs.glb,
"--output",
area.outputs.compressedGlb,
"--texture-size",
String(area.compress.textureSize),
"--quality",
String(area.compress.quality),
"--effort",
String(area.compress.effort),
"--metadata",
area.outputs.metadata,
"--metadata-output",
area.outputs.compressedMetadata,
"--preview",
area.outputs.cesiumPreview,
"--preview-output",
area.outputs.compressedCesiumPreview,
];
if (area.compress.meshopt) {
compressArgs.push("--meshopt");
}
console.log("Stage: compress (Cesium GLB texture resize + WebP)");
const started = Date.now();
const startedAt = new Date(started).toISOString();
runCommand(process.execPath, compressArgs, "compress");
const finished = Date.now();
const sourceDigest = glbDigest(area.outputs.glb);
const compressedDigest = glbDigest(area.outputs.compressedGlb);
writeStageManifest(area, {
stage: "compress",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
},
outputs: {
compressedGlb: fileRecord(area.outputs.compressedGlb),
compressedMetadata: fileRecord(area.outputs.compressedMetadata),
compressedCesiumPreview: fileRecord(area.outputs.compressedCesiumPreview),
},
summary: {
sourceGlb: glbSummary(sourceDigest),
compressedGlb: glbSummary(compressedDigest),
budget: evaluateGlbBudget(compressedDigest, area.budget),
options: {
textureSize: area.compress.textureSize,
quality: area.compress.quality,
effort: area.compress.effort,
meshopt: area.compress.meshopt,
fs.mkdirSync(area.outputs.pipelineDir, { recursive: true });
const temporaryDir = fs.mkdtempSync(path.join(area.outputs.pipelineDir, "compress-"));
// The compressor rejects an in-place transform. Preserve the exporter output
// in a temporary directory, then atomically promote the compressed delivery.
const sourceDir = path.join(temporaryDir, "source");
const outputDir = path.join(temporaryDir, "delivery");
const stagedSourceGlb = path.join(sourceDir, path.basename(area.outputs.glb));
const stagedSourceMetadata = path.join(sourceDir, path.basename(area.outputs.metadata));
const stagedSourcePreview = path.join(sourceDir, path.basename(area.outputs.cesiumPreview));
const stagedDeliveryGlb = path.join(outputDir, path.basename(area.outputs.glb));
const stagedDeliveryMetadata = path.join(outputDir, path.basename(area.outputs.metadata));
const stagedDeliveryPreview = path.join(outputDir, path.basename(area.outputs.cesiumPreview));
try {
fs.mkdirSync(sourceDir, { recursive: true });
fs.copyFileSync(area.outputs.glb, stagedSourceGlb);
fs.copyFileSync(area.outputs.metadata, stagedSourceMetadata);
fs.copyFileSync(area.outputs.cesiumPreview, stagedSourcePreview);
const sourceDigest = glbDigest(stagedSourceGlb);
const compressArgs = [
path.join(repoRoot, "scripts", "compress-glb.js"),
"--input", stagedSourceGlb,
"--output", stagedDeliveryGlb,
"--texture-size", String(area.compress.textureSize),
"--quality", String(area.compress.quality),
"--effort", String(area.compress.effort),
"--metadata", stagedSourceMetadata,
"--metadata-output", stagedDeliveryMetadata,
"--preview", stagedSourcePreview,
"--preview-output", stagedDeliveryPreview,
];
if (area.compress.meshopt) compressArgs.push("--meshopt");
console.log("Stage: compress (Cesium GLB texture resize + WebP)");
runCommand(process.execPath, compressArgs, "compress");
ensureFile(stagedDeliveryGlb, "Compressed delivery GLB");
ensureFile(stagedDeliveryMetadata, "Compressed delivery metadata");
ensureFile(stagedDeliveryPreview, "Compressed delivery preview");
const compressedDigest = glbDigest(stagedDeliveryGlb);
fs.renameSync(stagedDeliveryGlb, area.outputs.glb);
fs.renameSync(stagedDeliveryMetadata, area.outputs.metadata);
fs.renameSync(stagedDeliveryPreview, area.outputs.cesiumPreview);
const finished = Date.now();
writeStageManifest(area, {
stage: "compress",
status: "ok",
config: configPath,
startedAt,
finishedAt: new Date(finished).toISOString(),
durationMs: finished - started,
inputs: {},
outputs: {
glb: fileRecord(area.outputs.glb),
metadata: fileRecord(area.outputs.metadata),
cesiumPreview: fileRecord(area.outputs.cesiumPreview),
},
compressionRatio: Number((compressedDigest.fileBytes / sourceDigest.fileBytes).toFixed(4)),
savedBytes: sourceDigest.fileBytes - compressedDigest.fileBytes,
},
warnings: glbBudgetWarnings("Compressed", compressedDigest, area.budget),
});
summary: {
sourceGlb: glbSummary(sourceDigest),
glb: glbSummary(compressedDigest),
budget: evaluateGlbBudget(compressedDigest, area.budget),
options: {
textureSize: area.compress.textureSize,
quality: area.compress.quality,
effort: area.compress.effort,
meshopt: area.compress.meshopt,
},
compressionRatio: Number((compressedDigest.fileBytes / sourceDigest.fileBytes).toFixed(4)),
savedBytes: sourceDigest.fileBytes - compressedDigest.fileBytes,
},
warnings: glbBudgetWarnings("Compressed", compressedDigest, area.budget),
});
} finally {
fs.rmSync(temporaryDir, { recursive: true, force: true });
}
}
function blenderExecutable(area) {

4
scripts/compress-glb.js
View File

@@ -1,8 +1,8 @@
#!/usr/bin/env node
"use strict";
// Experimental GLB compression wrapper. Keeps the source GLB intact and writes
// a separate compressed artifact for visual comparison in Cesium.
// GLB compression wrapper. The caller supplies distinct input and output files;
// build-area.js uses staging files before promoting the compressed delivery.
const fs = require("fs");
const os = require("os");

13
scripts/lib/area-config.js
View File

@@ -25,8 +25,6 @@ function normalizeAreaConfig(raw, options = {}) {
const fileStem = outputOverrides.fileStem || id;
const compress = normalizeCompressConfig(raw.compress);
const budget = normalizeBudgetConfig(raw.budget);
const compressedFileStem = outputOverrides.compressedFileStem ||
`${fileStem}-compressed-webp${compress.textureSize}${compress.meshopt ? "-meshopt" : ""}`;
const pipelineDir = path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline"));
const geojsonDir = path.resolve(outputOverrides.geojsonDir || path.join(areaDir, "osm2streets_web_out"));
const outputs = {
@@ -47,15 +45,6 @@ function normalizeAreaConfig(raw, options = {}) {
cesiumPreview: path.resolve(
outputOverrides.cesiumPreview || path.join(areaDir, `${fileStem}-cesium-preview.html`),
),
compressedGlb: path.resolve(
outputOverrides.compressedGlb || path.join(areaDir, `${compressedFileStem}.glb`),
),
compressedMetadata: path.resolve(
outputOverrides.compressedMetadata || path.join(areaDir, `${compressedFileStem}.json`),
),
compressedCesiumPreview: path.resolve(
outputOverrides.compressedCesiumPreview || path.join(areaDir, `${compressedFileStem}-cesium-preview.html`),
),
vehicleRoute: path.resolve(outputOverrides.vehicleRoute || path.join(areaDir, `${fileStem}-vehicle-route.json`)),
vehicleModel: path.resolve(outputOverrides.vehicleModel || path.join(areaDir, `${fileStem}-vehicle-car.gltf`)),
trafficSignalAssemblies: path.resolve(
@@ -80,7 +69,7 @@ function normalizeAreaConfig(raw, options = {}) {
cesium: raw.stages?.cesium ?? true,
reimport: false,
preview: false,
compress: false,
compress: true,
},
qgis: {
arrowScale: raw.qgis?.arrowScale ?? raw.arrowScale ?? 0.8,

27
scripts/lib/area-diagnostics.js
View File

@@ -334,9 +334,6 @@ function artifactStatus(area) {
["Cesium GLB", area.outputs.glb, true, "file"],
["Cesium metadata", area.outputs.metadata, true, "file"],
["Cesium preview", area.outputs.cesiumPreview, true, "file"],
["Compressed GLB", area.outputs.compressedGlb, false, "file"],
["Compressed metadata", area.outputs.compressedMetadata, false, "file"],
["Compressed preview", area.outputs.compressedCesiumPreview, false, "file"],
];
return entries.map(([label, file, expected, type]) => {
const exists = fs.existsSync(file);
@@ -374,6 +371,7 @@ function metadataSummary(file, warnings) {
}
function stageManifestStatus(area, configPath = null) {
const compressionComplete = fs.existsSync(stageManifestPath(area, "compress"));
const reimportManifest = stageManifestPath(area, "reimport");
const hasReimportManifest = fs.existsSync(reimportManifest);
const derivedConfig = path.join(area.outputs.pipelineDir, "osm2streets-qgis.config.json");
@@ -446,7 +444,7 @@ function stageManifestStatus(area, configPath = null) {
inputs: {
blend: area.outputs.blend,
},
outputs: {
outputs: compressionComplete ? {} : {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
},
@@ -457,15 +455,20 @@ function stageManifestStatus(area, configPath = null) {
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
glb: area.outputs.glb,
metadata: area.outputs.metadata,
...(compressionComplete ? {} : {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
}),
lanePolygons: path.join(area.outputs.geojsonDir, "lane_polygons.geojson"),
network: path.join(area.outputs.geojsonDir, "network.json"),
intersectionSurface: path.join(area.outputs.geojsonDir, "intersection_surface.geojson"),
previewCss: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.css"),
previewJs: path.join(path.resolve(__dirname, ".."), "lib", "cesium-preview.js"),
},
outputs: {
outputs: compressionComplete ? {
vehicleRoute: area.outputs.vehicleRoute,
vehicleModel: area.outputs.vehicleModel,
} : {
cesiumPreview: area.outputs.cesiumPreview,
vehicleRoute: area.outputs.vehicleRoute,
vehicleModel: area.outputs.vehicleModel,
@@ -473,17 +476,13 @@ function stageManifestStatus(area, configPath = null) {
},
{
stage: "compress",
expected: fs.existsSync(area.outputs.compressedGlb),
inputs: {
expected: fs.existsSync(area.outputs.glb),
inputs: {},
outputs: {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
cesiumPreview: area.outputs.cesiumPreview,
},
outputs: {
compressedGlb: area.outputs.compressedGlb,
compressedMetadata: area.outputs.compressedMetadata,
compressedCesiumPreview: area.outputs.compressedCesiumPreview,
},
},
];
return stages.map((entry) => {

4
scripts/lib/build-stages.js
View File

@@ -6,11 +6,11 @@ const STAGES = [
{ id: "blender", label: "Blender scene", description: "Generate the editable scene and render" },
{ id: "cesium", label: "Cesium export", description: "Export GLB, metadata, and preview" },
{ id: "preview", label: "Preview refresh", description: "Regenerate preview HTML and vehicles" },
{ id: "compress", label: "Compressed preview", description: "Create optional compressed GLB artifacts" },
{ id: "compress", label: "Compress delivery assets", description: "Replace standard GLB, metadata, and preview with compressed deliverables" },
];
const ALIASES = {
all: ["intermediates", "blender", "cesium"],
all: ["intermediates", "blender", "cesium", "compress"],
qgis: ["intermediates"], osm2streets: ["intermediates"], geojson: ["intermediates"], intermediate: ["intermediates"],
intermediates: ["intermediates"], reimport: ["reimport"], gpkg: ["reimport"], blender: ["blender"], scene: ["blender"],
cesium: ["cesium"], glb: ["cesium"], preview: ["preview"], html: ["preview"], cesiumPreview: ["preview"],

4
scripts/test-asset-budgets.js
View File

@@ -71,6 +71,10 @@ assert.equal(
const configPath = path.join(tempDir, "area.json");
fs.writeFileSync(configPath, `${JSON.stringify(base)}\n`);
const area = normalizeAreaConfig(base);
assert.equal(area.stages.compress, true);
assert.equal("compressedGlb" in area.outputs, false);
assert.equal("compressedMetadata" in area.outputs, false);
assert.equal("compressedCesiumPreview" in area.outputs, false);
fs.mkdirSync(area.outputs.geojsonDir, { recursive: true });
for (const file of [area.outputs.glb, area.outputs.metadata, area.outputs.cesiumPreview, area.outputs.vehicleRoute, area.outputs.vehicleModel]) {
fs.writeFileSync(file, "fixture\n");

3
scripts/test-build-stages.js
View File

@@ -3,6 +3,7 @@
const assert = require("assert");
const { resolveStages, canonicalStages } = require("./lib/build-stages");
assert.deepEqual(canonicalStages(resolveStages({}, ["compress", "blender", "preview"])), ["blender", "preview", "compress"]);
assert.deepEqual(canonicalStages(resolveStages({}, ["all"])), ["intermediates", "blender", "cesium"]);
assert.deepEqual(canonicalStages(resolveStages({}, ["all"])), ["intermediates", "blender", "cesium", "compress"]);
assert.deepEqual(canonicalStages(resolveStages({ intermediates: true, blender: true, cesium: true, compress: true })), ["intermediates", "blender", "cesium", "compress"]);
assert.throws(() => resolveStages({}, ["intermediates", "reimport"]), /mutually exclusive/);
console.log("Build stage tests passed.");