diff --git a/README.md b/README.md index b60387d..2df9127 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,12 @@ ## 目标产物 -每个区域默认输出到 `outputs//`: +每个区域默认输出到 `outputs//`。其中只有 `package/` 是可复制给下游的发布边界: + +- `package/manifest.json`:版本化 `osm-asset-package/v1` manifest,声明 WGS84 anchor、ENU(X east/Y north/Z up,米)、heading correction 和包内模型 URL +- `package/models/.glb`:完整静态场景;`roads.glb`、`buildings.glb`、`vegetation.glb`、`water.glb` 是可选重叠分层,默认不要与完整场景同时加载 + +其余内容仍是本项目工作目录: - `.blend`:Blender 场景,包含道路、建筑、水体、植被等 - `.png`:Blender 预览渲染 @@ -61,7 +66,7 @@ npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json npm run build:area -- --config config/areas/nantaizi-lake-innovation-valley.json --stages reimport ``` -`intermediates` 会生成 osm2streets GeoJSON、GeoPackage、QGIS 工程和 QGIS 预览图。`blender` 使用 OSM 和 osm2streets GeoJSON 生成 `.blend`/`.png`。`cesium` 从 `.blend` 导出 `.glb`/`.json`,并生成 Cesium 预览 HTML。完整构建会接着运行 `compress`,以标准 `.glb/.json/html` 名称交付压缩版本;未压缩导出仅存在于构建时临时目录。`preview` 只在已有 `.glb/.json` 时补生成 HTML。`reimport` 把手工编辑过的 GeoPackage 回导为 GeoJSON,不含在 `all` 里,详见 [QGIS 手工修正工作流](#qgis-手工修正工作流)。 +`intermediates` 会生成 osm2streets GeoJSON、GeoPackage、QGIS 工程和 QGIS 预览图。`blender` 使用 OSM 和 osm2streets GeoJSON 生成 `.blend`/`.png`。`cesium` 从 `.blend` 导出包 staging,`compress` 压缩 staging 主 GLB,`package` 校验并原子发布 `package/`,`preview` 生成仅用于本地验证的 HTML 和 `_preview/` 动态描述符。`reimport` 把手工编辑过的 GeoPackage 回导为 GeoJSON,不含在 `all` 里,详见 [QGIS 手工修正工作流](#qgis-手工修正工作流)。 Cesium 预览默认显示完整场景。点击 `Inspect` 后会按道路、建筑、绿化与设施、水体加载辅助 GLB;该模式用于单独检查生成结果,主 GLB 是压缩后的下游交付资产。 diff --git a/blender/export_cesium.py b/blender/export_cesium.py index 73beb9f..90939ad 100644 --- a/blender/export_cesium.py +++ b/blender/export_cesium.py @@ -579,7 +579,8 @@ def export_glb(filepath, meshes): def semantic_asset_specs(glb_path, meshes): """Derive optional inspection assets from the scene's named collections.""" - stem, extension = os.path.splitext(glb_path) + model_dir, filename = os.path.split(glb_path) + _, extension = os.path.splitext(filename) by_collection = {} for collection in bpy.context.scene.collection.children: by_collection[collection.name] = set(collection.all_objects) @@ -592,7 +593,7 @@ def semantic_asset_specs(glb_path, meshes): subset = [obj for obj in meshes if obj in objects] if not subset: continue - path = stem + "-" + asset_id + extension + path = os.path.join(model_dir, asset_id + extension) specs.append({ "id": asset_id, "label": label, @@ -673,42 +674,19 @@ def export(args): center_lat = (bounds["min_lat"] + bounds["max_lat"]) / 2.0 else: center_lon = center_lat = 0.0 + area_id = os.path.splitext(os.path.basename(args["glb"]))[0] metadata = { - "asset": os.path.basename(args["glb"]), - "assets": [{ - "id": "main", - "label": "Scene", - "type": "model", - "url": os.path.basename(args["glb"]), - "enabled": True, - }, { - "id": "traffic-dynamic", - "label": "Traffic signals dynamic", - "type": "model", - "url": os.path.basename(args["dynamic_glb"]) if args.get("dynamic_glb") and dynamic_meshes else "", - "enabled": True, - "category": "dynamic", - }, { - "id": "traffic-countdown-0", "label": "Traffic countdown group 0", "type": "model", - "url": os.path.basename(args["countdown_0_glb"]), "enabled": True, "category": "countdown", "phaseGroup": 0, - }, { - "id": "traffic-countdown-1", "label": "Traffic countdown group 1", "type": "model", - "url": os.path.basename(args["countdown_1_glb"]), "enabled": True, "category": "countdown", "phaseGroup": 1, - }] + [{ - "id": asset["id"], - "label": asset["label"], - "type": "model", - "url": os.path.basename(asset["path"]), - "enabled": False, - "category": "semantic", + "schema": "osm-asset-package/v1", + "packageVersion": "1.0.0", + "areaId": area_id, + "coordinateSystem": {"axes": "ENU", "units": "meters", "x": "east", "y": "north", "z": "up"}, + "placement": {"longitude": center_lon, "latitude": center_lat, "height": 0.35, "headingCorrectionDegrees": -90.0}, + "bounds": {"minLon": bounds.get("min_lon", center_lon), "minLat": bounds.get("min_lat", center_lat), "maxLon": bounds.get("max_lon", center_lon), "maxLat": bounds.get("max_lat", center_lat)}, + "assets": [{"id": "main", "role": "scene", "category": "scene", "uri": "models/" + os.path.basename(args["glb"]), "defaultLoad": True}] + [{ + "id": asset["id"], "role": "layer", "category": asset["id"], + "uri": "models/" + os.path.basename(asset["path"]), "defaultLoad": False, } for asset in semantic_assets], - "coordinate_system": "local ENU meters (X east, Y north, Z up)", - "heading_correction_degrees": -90.0, - "anchor": {"longitude": center_lon, "latitude": center_lat, "height": 0.35}, - "bounds": bounds, - "source_osm": scene.get("source_osm", ""), - "source_geojson": scene.get("source_geojson", ""), - "scene_stats": { + "sceneStats": { "buildings": scene.get("building_count", 0), "industrial_buildings": scene.get("industrial_building_count", 0), "lake_polygons": scene.get("lake_count", 0), @@ -717,15 +695,6 @@ def export(args): "scrub_polygons": scene.get("scrub_count", 0), "fountains": scene.get("fountain_count", 0), }, - "cesium_js": ( - "const p = Cesium.Cartesian3.fromDegrees(" + - f"{center_lon:.8f}, {center_lat:.8f}, 0.35);\n" + - "const enu = Cesium.Transforms.eastNorthUpToFixedFrame(p);\n" + - "const correction = Cesium.Matrix3.fromRotationZ(Cesium.Math.toRadians(-90.0));\n" + - "const modelMatrix = Cesium.Matrix4.multiplyByMatrix3(enu, correction, new Cesium.Matrix4());\n" + - "Cesium.Model.fromGltfAsync({ url: '" + os.path.basename(args["glb"]) + - "', modelMatrix }).then(model => viewer.scene.primitives.add(model));" - ), } os.makedirs(os.path.dirname(args["metadata"]), exist_ok=True) with open(args["metadata"], "w", encoding="utf-8") as handle: diff --git a/examples/asset-package-resolver.js b/examples/asset-package-resolver.js new file mode 100644 index 0000000..69b5eb8 --- /dev/null +++ b/examples/asset-package-resolver.js @@ -0,0 +1,13 @@ +"use strict"; + +function resolveAsset(manifestUrl, manifest, id = "main") { + const asset = manifest.assets.find((candidate) => candidate.id === id); + if (!asset) throw new Error(`Unknown asset '${id}'`); + return new URL(asset.uri, manifestUrl).href; +} + +function placement(manifest) { + return { ...manifest.placement, coordinateSystem: manifest.coordinateSystem }; +} + +module.exports = { resolveAsset, placement }; diff --git a/examples/cesium-asset-package.js b/examples/cesium-asset-package.js new file mode 100644 index 0000000..9202108 --- /dev/null +++ b/examples/cesium-asset-package.js @@ -0,0 +1,20 @@ +"use strict"; + +const { resolveAsset, placement } = require("./asset-package-resolver"); + +async function loadPackage(viewer, manifestUrl, assetId = "main") { + const manifest = await fetch(manifestUrl).then((response) => { + if (!response.ok) throw new Error(`Could not load manifest: ${response.status}`); + return response.json(); + }); + const p = placement(manifest); + const origin = Cesium.Cartesian3.fromDegrees(p.longitude, p.latitude, p.height); + const enu = Cesium.Transforms.eastNorthUpToFixedFrame(origin); + const rotation = Cesium.Matrix3.fromRotationZ(Cesium.Math.toRadians(p.headingCorrectionDegrees)); + const modelMatrix = Cesium.Matrix4.multiplyByMatrix3(enu, rotation, new Cesium.Matrix4()); + const model = await Cesium.Model.fromGltfAsync({ url: resolveAsset(manifestUrl, manifest, assetId), modelMatrix }); + viewer.scene.primitives.add(model); + return { manifest, model, placement: p }; +} + +module.exports = { loadPackage }; diff --git a/examples/three-asset-package.js b/examples/three-asset-package.js new file mode 100644 index 0000000..565d50e --- /dev/null +++ b/examples/three-asset-package.js @@ -0,0 +1,17 @@ +"use strict"; + +const { resolveAsset, placement } = require("./asset-package-resolver"); + +async function loadPackage(loader, manifestUrl, assetId = "main", handoffEnuPlacement) { + const manifest = await fetch(manifestUrl).then((response) => { + if (!response.ok) throw new Error(`Could not load manifest: ${response.status}`); + return response.json(); + }); + const scene = await loader.loadAsync(resolveAsset(manifestUrl, manifest, assetId)); + // Three.js does not georeference local ENU itself; the embedding host owns it. + const enuPlacement = placement(manifest); + if (handoffEnuPlacement) handoffEnuPlacement(enuPlacement, scene.scene); + return { manifest, scene: scene.scene, placement: enuPlacement }; +} + +module.exports = { loadPackage }; diff --git a/package.json b/package.json index e34ad7a..b273465 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "test:budgets": "node scripts/test-asset-budgets.js", "test:preview-assets": "node scripts/test-preview-assets.js", "test:compress-glb": "node scripts/test-compress-glb.js", + "test:package-contract": "node scripts/test-package-contract.js", + "test:package-examples": "node scripts/test-package-examples.js", "test:turn-lane-arrows": "node scripts/test-turn-lane-arrows.js", "test:traffic-signals": "node scripts/test-traffic-signals.js", "render:turn-lane-arrow-samples": "node scripts/render-turn-lane-arrow-samples.js" diff --git a/scripts/build-area.js b/scripts/build-area.js index 1637178..bae641e 100755 --- a/scripts/build-area.js +++ b/scripts/build-area.js @@ -5,6 +5,7 @@ const path = require("path"); const { spawnSync } = require("child_process"); const { readAreaConfig } = require("./lib/area-config"); const { resolveStages } = require("./lib/build-stages"); +const { validateManifest, addIntegrity } = require("./lib/package-contract"); const { blenderExecutable: resolveBlenderExecutable } = require("./lib/tool-paths"); const { SCENE_LAYERS, @@ -59,12 +60,15 @@ if (stages.blender) { if (stages.cesium) { exportCesium(area); } -if (stages.preview) { - writeCesiumPreview(area); -} if (stages.compress) { compressCesiumGlb(area); } +if (stages.package) { + publishPackage(area); +} +if (stages.preview) { + writeCesiumPreview(area); +} console.log("Done."); @@ -273,6 +277,7 @@ function exportCesium(area) { ensureFile(blenderExecutable(area), "Blender executable"); ensureFile(area.outputs.blend, "Blend scene"); ensureFile(path.join(repoRoot, "blender", "export_cesium.py"), "Cesium exporter"); + fs.rmSync(area.outputs.packageStagingDir, { recursive: true, force: true }); fs.mkdirSync(path.dirname(area.outputs.glb), { recursive: true }); fs.mkdirSync(path.dirname(area.outputs.metadata), { recursive: true }); @@ -302,7 +307,6 @@ function exportCesium(area) { ensureFile(area.outputs.trafficSignalsCountdown0Glb, "Traffic countdown group 0 GLB"); ensureFile(area.outputs.trafficSignalsCountdown1Glb, "Traffic countdown group 1 GLB"); const semanticAssets = semanticAssetRecords(area); - writeCesiumPreview(area); const finished = Date.now(); const digest = glbDigest(area.outputs.glb); writeStageManifest(area, { @@ -334,8 +338,8 @@ function semanticAssetRecords(area) { const assets = Array.isArray(metadata.assets) ? metadata.assets : []; const records = {}; for (const asset of assets) { - if (asset.category !== "semantic") continue; - const file = path.join(area.outputs.areaDir, asset.url); + if (asset.role !== "layer") continue; + const file = path.resolve(area.outputs.packageStagingDir, asset.uri); ensureFile(file, `Cesium semantic asset '${asset.id}'`); records[asset.id] = fileRecord(file); } @@ -345,7 +349,6 @@ function semanticAssetRecords(area) { function compressCesiumGlb(area) { ensureFile(area.outputs.glb, "Cesium GLB"); ensureFile(area.outputs.metadata, "Cesium metadata"); - ensureFile(area.outputs.cesiumPreview, "Cesium preview"); ensureFile(path.join(repoRoot, "scripts", "compress-glb.js"), "GLB compressor"); const started = Date.now(); const startedAt = new Date(started).toISOString(); @@ -358,15 +361,12 @@ function compressCesiumGlb(area) { const outputDir = path.join(temporaryDir, "delivery"); const stagedSourceGlb = path.join(sourceDir, path.basename(area.outputs.glb)); const stagedSourceMetadata = path.join(sourceDir, path.basename(area.outputs.metadata)); - const stagedSourcePreview = path.join(sourceDir, path.basename(area.outputs.cesiumPreview)); const stagedDeliveryGlb = path.join(outputDir, path.basename(area.outputs.glb)); const stagedDeliveryMetadata = path.join(outputDir, path.basename(area.outputs.metadata)); - const stagedDeliveryPreview = path.join(outputDir, path.basename(area.outputs.cesiumPreview)); try { fs.mkdirSync(sourceDir, { recursive: true }); fs.copyFileSync(area.outputs.glb, stagedSourceGlb); fs.copyFileSync(area.outputs.metadata, stagedSourceMetadata); - fs.copyFileSync(area.outputs.cesiumPreview, stagedSourcePreview); const sourceDigest = glbDigest(stagedSourceGlb); const compressArgs = [ path.join(repoRoot, "scripts", "compress-glb.js"), @@ -377,8 +377,6 @@ function compressCesiumGlb(area) { "--effort", String(area.compress.effort), "--metadata", stagedSourceMetadata, "--metadata-output", stagedDeliveryMetadata, - "--preview", stagedSourcePreview, - "--preview-output", stagedDeliveryPreview, ]; if (area.compress.meshopt) compressArgs.push("--meshopt"); @@ -386,11 +384,9 @@ function compressCesiumGlb(area) { runCommand(process.execPath, compressArgs, "compress"); ensureFile(stagedDeliveryGlb, "Compressed delivery GLB"); ensureFile(stagedDeliveryMetadata, "Compressed delivery metadata"); - ensureFile(stagedDeliveryPreview, "Compressed delivery preview"); const compressedDigest = glbDigest(stagedDeliveryGlb); fs.renameSync(stagedDeliveryGlb, area.outputs.glb); fs.renameSync(stagedDeliveryMetadata, area.outputs.metadata); - fs.renameSync(stagedDeliveryPreview, area.outputs.cesiumPreview); const finished = Date.now(); writeStageManifest(area, { stage: "compress", @@ -403,7 +399,6 @@ function compressCesiumGlb(area) { outputs: { glb: fileRecord(area.outputs.glb), metadata: fileRecord(area.outputs.metadata), - cesiumPreview: fileRecord(area.outputs.cesiumPreview), }, summary: { sourceGlb: glbSummary(sourceDigest), @@ -425,6 +420,34 @@ function compressCesiumGlb(area) { } } +function publishPackage(area) { + ensureFile(area.outputs.packageStagingManifest, "Staged package manifest"); + const started = Date.now(); + const manifest = JSON.parse(fs.readFileSync(area.outputs.packageStagingManifest, "utf8")); + validateManifest(manifest, area.outputs.packageStagingDir); + addIntegrity(manifest, area.outputs.packageStagingDir); + fs.writeFileSync(area.outputs.packageStagingManifest, `${JSON.stringify(manifest, null, 2)}\n`); + validateManifest(manifest, area.outputs.packageStagingDir); + const backup = `${area.outputs.packageDir}.previous`; + fs.rmSync(backup, { recursive: true, force: true }); + try { + if (fs.existsSync(area.outputs.packageDir)) fs.renameSync(area.outputs.packageDir, backup); + fs.renameSync(area.outputs.packageStagingDir, area.outputs.packageDir); + fs.rmSync(backup, { recursive: true, force: true }); + } catch (error) { + if (!fs.existsSync(area.outputs.packageDir) && fs.existsSync(backup)) fs.renameSync(backup, area.outputs.packageDir); + throw error; + } + const finished = Date.now(); + writeStageManifest(area, { + stage: "package", status: "ok", config: configPath, + startedAt: new Date(started).toISOString(), finishedAt: new Date(finished).toISOString(), durationMs: finished - started, + inputs: { stagingManifest: fileRecord(path.join(area.outputs.packageDir, "manifest.json")) }, + outputs: { packageDir: fileRecord(area.outputs.packageDir), manifest: fileRecord(area.outputs.packageManifest) }, + summary: { assets: manifest.assets.length, packageDir: area.outputs.packageDir }, warnings: [], + }); +} + function blenderExecutable(area) { return resolveBlenderExecutable(area.blenderApp); } @@ -447,8 +470,7 @@ function runCommand(command, commandArgs, stage) { } function writeCesiumPreview(area) { - ensureFile(area.outputs.glb, "Cesium GLB"); - ensureFile(area.outputs.metadata, "Cesium metadata"); + ensureFile(area.outputs.packageManifest, "Published asset package manifest"); ensureFile(area.outputs.trafficSignals, "Traffic signal anchors"); const lanePolygons = path.join(area.outputs.geojsonDir, "lane_polygons.geojson"); const network = path.join(area.outputs.geojsonDir, "network.json"); @@ -465,11 +487,18 @@ function writeCesiumPreview(area) { writeVehicleRoute(area, vehicleRoute); const vehicleModelNames = writeVehicleModel(area); writeCesiumPreviewSupportFiles(path.dirname(htmlPath)); - const glbName = path.basename(area.outputs.glb); - const metadataName = path.basename(area.outputs.metadata); - const routeName = path.basename(area.outputs.vehicleRoute); - const vehicleModelName = path.basename(area.outputs.vehicleModel); - fs.writeFileSync(htmlPath, cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, area.id, vehicleModelNames, previewRelativePath(area.outputs.areaDir, area.outputs.trafficSignals))); + const glbName = "package/manifest.json"; + const metadataName = "package/manifest.json"; + const routeName = previewRelativePath(area.outputs.areaDir, area.outputs.vehicleRoute); + const vehicleModelName = previewRelativePath(area.outputs.areaDir, area.outputs.vehicleModel); + const descriptor = { routeName: previewRelativePath(area.outputs.areaDir, area.outputs.vehicleRoute), vehicleModelName: previewRelativePath(area.outputs.areaDir, area.outputs.vehicleModel), vehicleModelNames: vehicleModelNames.map((name) => `_preview/${name}`), trafficSignalsName: previewRelativePath(area.outputs.areaDir, area.outputs.trafficSignals), assets: [ + { id: "traffic-dynamic", url: previewRelativePath(area.outputs.areaDir, area.outputs.trafficSignalsDynamicGlb), category: "dynamic", enabled: true }, + { id: "traffic-countdown-0", url: previewRelativePath(area.outputs.areaDir, area.outputs.trafficSignalsCountdown0Glb), category: "countdown", enabled: true, phaseGroup: 0 }, + { id: "traffic-countdown-1", url: previewRelativePath(area.outputs.areaDir, area.outputs.trafficSignalsCountdown1Glb), category: "countdown", enabled: true, phaseGroup: 1 }, + ] }; + fs.mkdirSync(area.outputs.previewDir, { recursive: true }); + fs.writeFileSync(area.outputs.previewDescriptor, `${JSON.stringify(descriptor, null, 2)}\n`); + fs.writeFileSync(htmlPath, cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, area.id, vehicleModelNames, previewRelativePath(area.outputs.areaDir, area.outputs.trafficSignals), "_preview/descriptor.json")); console.log(`Cesium preview: ${htmlPath}`); const finished = Date.now(); writeStageManifest(area, { @@ -482,8 +511,8 @@ function writeCesiumPreview(area) { inputs: { config: fileRecord(configPath), osm: fileRecord(area.input), - glb: fileRecord(area.outputs.glb), - metadata: fileRecord(area.outputs.metadata), + glb: fileRecord(area.outputs.packagePrimaryGlb), + metadata: fileRecord(area.outputs.packageManifest), lanePolygons: fileRecord(lanePolygons), network: fileRecord(network), intersectionSurface: fileRecord(intersectionSurface), diff --git a/scripts/compress-glb.js b/scripts/compress-glb.js index 887230c..73f5130 100644 --- a/scripts/compress-glb.js +++ b/scripts/compress-glb.js @@ -143,30 +143,25 @@ function previewOutputPath(output, explicitPath) { function writeCompressedMetadata(sourceMetadata, outputGlb, outputMetadata) { const metadata = JSON.parse(fs.readFileSync(sourceMetadata, "utf8")); const glbName = path.basename(outputGlb); - metadata.asset = glbName; + if (metadata.asset !== undefined) metadata.asset = glbName; const assets = Array.isArray(metadata.assets) ? metadata.assets : []; const main = assets.find((asset) => asset.id === "main"); metadata.assets = assets.length ? assets.map((asset) => asset.id === "main" ? { ...asset, label: "Compressed Scene", - url: glbName, - enabled: true, + ...(Object.hasOwn(asset, "uri") ? { uri: `models/${glbName}`, defaultLoad: true } : { url: glbName, enabled: true }), } : asset) : [{ id: "main", label: "Compressed Scene", - type: "model", - url: glbName, - enabled: true, + role: "scene", category: "scene", uri: `models/${glbName}`, defaultLoad: true, }]; if (!main && assets.length) { metadata.assets.unshift({ id: "main", label: "Compressed Scene", - type: "model", - url: glbName, - enabled: true, + role: "scene", category: "scene", uri: `models/${glbName}`, defaultLoad: true, }); } if (typeof metadata.cesium_js === "string") { diff --git a/scripts/lib/area-config.js b/scripts/lib/area-config.js index 1b57535..07638fd 100644 --- a/scripts/lib/area-config.js +++ b/scripts/lib/area-config.js @@ -26,6 +26,9 @@ function normalizeAreaConfig(raw, options = {}) { const compress = normalizeCompressConfig(raw.compress); const budget = normalizeBudgetConfig(raw.budget); const pipelineDir = path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline")); + const packageDir = path.resolve(outputOverrides.packageDir || path.join(areaDir, "package")); + const packageStagingDir = path.resolve(outputOverrides.packageStagingDir || path.join(pipelineDir, "package-staging")); + const previewDir = path.resolve(outputOverrides.previewDir || path.join(areaDir, "_preview")); const geojsonDir = path.resolve(outputOverrides.geojsonDir || path.join(areaDir, "osm2streets_web_out")); const outputs = { areaDir, @@ -35,18 +38,28 @@ function normalizeAreaConfig(raw, options = {}) { qgisPreview: path.resolve(outputOverrides.qgisPreview || path.join(areaDir, `${fileStem}-preview.png`)), blend: path.resolve(outputOverrides.blend || path.join(areaDir, `${fileStem}.blend`)), render: path.resolve(outputOverrides.render || path.join(areaDir, `${fileStem}.png`)), - glb: path.resolve(outputOverrides.glb || path.join(areaDir, `${fileStem}.glb`)), + // Published static assets are staged first and promoted by the package stage. + packageDir, + packageStagingDir, + packageManifest: path.resolve(outputOverrides.packageManifest || path.join(packageDir, "manifest.json")), + packageStagingManifest: path.resolve(outputOverrides.packageStagingManifest || path.join(packageStagingDir, "manifest.json")), + packageModelDir: path.resolve(outputOverrides.packageModelDir || path.join(packageDir, "models")), + packageStagingModelDir: path.resolve(outputOverrides.packageStagingModelDir || path.join(packageStagingDir, "models")), + packagePrimaryGlb: path.resolve(outputOverrides.packagePrimaryGlb || path.join(packageDir, "models", `${fileStem}.glb`)), + glb: path.resolve(outputOverrides.glb || path.join(packageStagingDir, "models", `${fileStem}.glb`)), trafficSignalsDynamicGlb: path.resolve( - outputOverrides.trafficSignalsDynamicGlb || path.join(areaDir, `${fileStem}-traffic-signals-dynamic.glb`), + outputOverrides.trafficSignalsDynamicGlb || path.join(previewDir, `${fileStem}-traffic-signals-dynamic.glb`), ), - trafficSignalsCountdown0Glb: path.resolve(outputOverrides.trafficSignalsCountdown0Glb || path.join(areaDir, `${fileStem}-traffic-signals-countdown-0.glb`)), - trafficSignalsCountdown1Glb: path.resolve(outputOverrides.trafficSignalsCountdown1Glb || path.join(areaDir, `${fileStem}-traffic-signals-countdown-1.glb`)), - metadata: path.resolve(outputOverrides.metadata || path.join(areaDir, `${fileStem}.json`)), + trafficSignalsCountdown0Glb: path.resolve(outputOverrides.trafficSignalsCountdown0Glb || path.join(previewDir, `${fileStem}-traffic-signals-countdown-0.glb`)), + trafficSignalsCountdown1Glb: path.resolve(outputOverrides.trafficSignalsCountdown1Glb || path.join(previewDir, `${fileStem}-traffic-signals-countdown-1.glb`)), + metadata: path.resolve(outputOverrides.metadata || path.join(packageStagingDir, "manifest.json")), cesiumPreview: path.resolve( outputOverrides.cesiumPreview || path.join(areaDir, `${fileStem}-cesium-preview.html`), ), - vehicleRoute: path.resolve(outputOverrides.vehicleRoute || path.join(areaDir, `${fileStem}-vehicle-route.json`)), - vehicleModel: path.resolve(outputOverrides.vehicleModel || path.join(areaDir, `${fileStem}-vehicle-car.gltf`)), + previewDir, + previewDescriptor: path.resolve(outputOverrides.previewDescriptor || path.join(previewDir, "descriptor.json")), + vehicleRoute: path.resolve(outputOverrides.vehicleRoute || path.join(previewDir, `${fileStem}-vehicle-route.json`)), + vehicleModel: path.resolve(outputOverrides.vehicleModel || path.join(previewDir, `${fileStem}-vehicle-car.gltf`)), trafficSignalAssemblies: path.resolve( outputOverrides.trafficSignalAssemblies || path.join(geojsonDir, "traffic_signal_assemblies.geojson"), ), @@ -68,8 +81,9 @@ function normalizeAreaConfig(raw, options = {}) { blender: raw.stages?.blender ?? true, cesium: raw.stages?.cesium ?? true, reimport: false, - preview: false, + preview: true, compress: true, + package: true, }, qgis: { arrowScale: raw.qgis?.arrowScale ?? raw.arrowScale ?? 0.8, diff --git a/scripts/lib/area-diagnostics.js b/scripts/lib/area-diagnostics.js index 7e8d9cb..87c8073 100644 --- a/scripts/lib/area-diagnostics.js +++ b/scripts/lib/area-diagnostics.js @@ -10,6 +10,7 @@ const { layerFile, } = require("./scene-layers"); const { digest: glbDigest } = require("../glb-digest"); +const { validateManifest } = require("./package-contract"); const { BUDGETS, evaluateGlbBudget, @@ -31,8 +32,8 @@ function analyzeArea(configPath, options = {}) { const artifacts = artifactStatus(area); const manifests = stageManifestStatus(area, resolvedConfig); const metadataWarnings = []; - const metadata = metadataSummary(area.outputs.metadata, metadataWarnings); - const glb = fs.existsSync(area.outputs.glb) ? glbDigest(area.outputs.glb) : null; + const metadata = metadataSummary(area.outputs.packageManifest, metadataWarnings); + const glb = fs.existsSync(area.outputs.packagePrimaryGlb) ? glbDigest(area.outputs.packagePrimaryGlb) : null; const warnings = [ ...metadataWarnings, ...collectWarnings(area, osm, artifacts, manifests, glb, metadata), @@ -331,8 +332,8 @@ function artifactStatus(area) { ["Traffic signal runtime", area.outputs.trafficSignals, true, "file"], ["Blend scene", area.outputs.blend, true, "file"], ["Render PNG", area.outputs.render, true, "file"], - ["Cesium GLB", area.outputs.glb, true, "file"], - ["Cesium metadata", area.outputs.metadata, true, "file"], + ["Package manifest", area.outputs.packageManifest, true, "file"], + ["Package primary GLB", area.outputs.packagePrimaryGlb, true, "file"], ["Cesium preview", area.outputs.cesiumPreview, true, "file"], ]; return entries.map(([label, file, expected, type]) => { @@ -359,6 +360,7 @@ function metadataSummary(file, warnings) { if (!fs.existsSync(file)) return null; try { const metadata = JSON.parse(fs.readFileSync(file, "utf8")); + if (metadata.schema === "osm-asset-package/v1") validateManifest(metadata, path.dirname(file)); return { asset: metadata.asset || null, assets: Array.isArray(metadata.assets) ? metadata.assets.length : 0, @@ -748,7 +750,7 @@ function classifyAreaQuality(result) { warnings.push(`OSM ways reference ${osm.ways.missingNodeRefs} missing node(s).`); } - const fatalArtifacts = new Set(["Cesium GLB", "Cesium metadata", "Cesium preview"]); + const fatalArtifacts = new Set(["Package manifest", "Package primary GLB", "Cesium preview"]); for (const artifact of artifacts) { if (fatalArtifacts.has(artifact.label)) { if (!artifact.exists) { diff --git a/scripts/lib/area-preview.js b/scripts/lib/area-preview.js index 472ea41..2e811da 100644 --- a/scripts/lib/area-preview.js +++ b/scripts/lib/area-preview.js @@ -13,7 +13,7 @@ function writeCesiumPreviewSupportFiles(outDir) { } } -function cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, areaId, vehicleModelNames = [], trafficSignalsName = null) { +function cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, areaId, vehicleModelNames = [], trafficSignalsName = null, previewDescriptorName = null) { const previewConfig = { areaId, glbName, @@ -22,6 +22,7 @@ function cesiumPreviewHtml(glbName, metadataName, routeName, vehicleModelName, a vehicleModelName, vehicleModelNames, trafficSignalsName, + previewDescriptorName, }; return ` @@ -105,10 +106,10 @@ function escapeScriptJson(value) { function previewSummary(area) { const route = JSON.parse(fs.readFileSync(area.outputs.vehicleRoute, "utf8")); return { - glbName: path.basename(area.outputs.glb), - metadataName: path.basename(area.outputs.metadata), - routeName: path.basename(area.outputs.vehicleRoute), - vehicleModelName: path.basename(area.outputs.vehicleModel), + glbName: "package/manifest.json", + metadataName: "package/manifest.json", + routeName: path.relative(area.outputs.areaDir, area.outputs.vehicleRoute).split(path.sep).join("/"), + vehicleModelName: path.relative(area.outputs.areaDir, area.outputs.vehicleModel).split(path.sep).join("/"), trafficSignalsName: path.relative(area.outputs.areaDir, area.outputs.trafficSignals).split(path.sep).join("/"), routeSegments: Array.isArray(route.segments) ? route.segments.length : null, }; diff --git a/scripts/lib/build-stages.js b/scripts/lib/build-stages.js index c185295..3fcb40c 100644 --- a/scripts/lib/build-stages.js +++ b/scripts/lib/build-stages.js @@ -4,17 +4,19 @@ const STAGES = [ { id: "intermediates", label: "OSM / QGIS intermediates", description: "Rebuild GeoJSON and GeoPackage from OSM" }, { id: "reimport", label: "Reimport QGIS edits", description: "Copy GeoPackage layers back to GeoJSON" }, { id: "blender", label: "Blender scene", description: "Generate the editable scene and render" }, - { id: "cesium", label: "Cesium export", description: "Export GLB, metadata, and preview" }, - { id: "preview", label: "Preview refresh", description: "Regenerate preview HTML and vehicles" }, - { id: "compress", label: "Compress delivery assets", description: "Replace standard GLB, metadata, and preview with compressed deliverables" }, + { id: "cesium", label: "Cesium export", description: "Export static package staging assets" }, + { id: "compress", label: "Compress staged package", description: "Compress the staged primary GLB" }, + { id: "package", label: "Publish asset package", description: "Validate and atomically publish package/" }, + { id: "preview", label: "Preview refresh", description: "Regenerate local verification preview and dynamic assets" }, ]; const ALIASES = { - all: ["intermediates", "blender", "cesium", "compress"], + all: ["intermediates", "blender", "cesium", "compress", "package", "preview"], qgis: ["intermediates"], osm2streets: ["intermediates"], geojson: ["intermediates"], intermediate: ["intermediates"], intermediates: ["intermediates"], reimport: ["reimport"], gpkg: ["reimport"], blender: ["blender"], scene: ["blender"], cesium: ["cesium"], glb: ["cesium"], preview: ["preview"], html: ["preview"], cesiumPreview: ["preview"], compress: ["compress"], compression: ["compress"], compressedCesium: ["compress"], + package: ["package"], publish: ["package"], assetPackage: ["package"], }; function resolveStages(defaults, requested) { diff --git a/scripts/lib/cesium-preview.js b/scripts/lib/cesium-preview.js index cefde37..dec6d65 100644 --- a/scripts/lib/cesium-preview.js +++ b/scripts/lib/cesium-preview.js @@ -40,6 +40,8 @@ async function main() { setLoadingMessage("Loading scene", config.areaId || ""); const metadata = await fetchJson(config.metadataName); + const descriptor = config.previewDescriptorName ? await fetchOptionalJson(config.previewDescriptorName) : null; + adaptPackageManifest(metadata, descriptor); const routeData = await fetchOptionalJson(config.routeName); const signalData = await fetchOptionalJson(config.trafficSignalsName); const placement = scenePlacement(metadata); @@ -145,11 +147,11 @@ } function scenePlacement(metadata) { - const anchor = metadata.anchor || {}; + const anchor = metadata.anchor || metadata.placement || {}; const longitude = Number(anchor.longitude || 0); const latitude = Number(anchor.latitude || 0); const height = Number(anchor.height || 0); - const heading = Number(metadata.heading_correction_degrees || 0); + const heading = Number(metadata.heading_correction_degrees ?? metadata.placement?.headingCorrectionDegrees ?? 0); const position = Cesium.Cartesian3.fromDegrees(longitude, latitude, height); const enu = Cesium.Transforms.eastNorthUpToFixedFrame(position); const correction = Cesium.Matrix3.fromRotationZ(Cesium.Math.toRadians(heading)); @@ -164,6 +166,10 @@ } function normalizedAssets(metadata) { + if (metadata.schema === "osm-asset-package/v1") { + const staticAssets = metadata.assets.map((asset) => ({ id: asset.id, label: asset.id, type: "model", url: asset.uri, enabled: asset.defaultLoad, category: asset.category, semantic: asset.role === "layer" })); + return staticAssets.concat(metadata.previewAssets || []); + } const assets = Array.isArray(metadata.assets) && metadata.assets.length ? metadata.assets : [{ @@ -176,6 +182,15 @@ return assets.filter((asset) => (asset.type || "model") === "model" && asset.url); } + function adaptPackageManifest(metadata, descriptor) { + if (metadata.schema !== "osm-asset-package/v1") return; + metadata.anchor = metadata.placement; + metadata.heading_correction_degrees = metadata.placement.headingCorrectionDegrees; + metadata.scene_stats = metadata.sceneStats || {}; + metadata.bounds = { min_lon: metadata.bounds.minLon, min_lat: metadata.bounds.minLat, max_lon: metadata.bounds.maxLon, max_lat: metadata.bounds.maxLat }; + if (descriptor?.assets) metadata.previewAssets = descriptor.assets; + } + // One broken entry in metadata.assets should not blank the whole preview, so // failures are collected and surfaced in the diagnostics panel instead. async function loadSceneAssets(viewer, metadata, placement) { @@ -187,7 +202,7 @@ loaded.push(entry); // Semantic assets are inspection aids. Defer their network and GPU cost // until the user explicitly switches out of the normal scene view. - if (entry.category === "semantic") continue; + if (entry.semantic || entry.enabled === false) continue; await loadAsset(viewer, entry, placement); } if (!loaded.some((asset) => asset.model && asset.category !== "semantic")) { diff --git a/scripts/lib/package-contract.js b/scripts/lib/package-contract.js new file mode 100644 index 0000000..48b6267 --- /dev/null +++ b/scripts/lib/package-contract.js @@ -0,0 +1,66 @@ +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const crypto = require("crypto"); + +const SCHEMA = "osm-asset-package/v1"; +const CATEGORIES = new Set(["scene", "roads", "buildings", "vegetation", "water"]); +const ROLES = new Set(["scene", "layer"]); + +function relativeUri(uri) { + if (typeof uri !== "string" || !uri || uri.startsWith("/") || path.isAbsolute(uri) || uri.includes("\\") || uri.split("/").includes("..")) { + throw new Error(`Asset URI must be a package-relative forward-slash path: ${uri}`); + } + return uri; +} + +function validateManifest(manifest, packageDir, options = {}) { + const errors = []; + const fail = (message) => errors.push(message); + if (!manifest || manifest.schema !== SCHEMA) fail(`schema must be '${SCHEMA}'`); + if (typeof manifest?.packageVersion !== "string" || !manifest.packageVersion) fail("packageVersion is required"); + if (typeof manifest?.areaId !== "string" || !manifest.areaId) fail("areaId is required"); + const coordinateSystem = manifest?.coordinateSystem || {}; + if (coordinateSystem.axes !== "ENU" || coordinateSystem.units !== "meters" || coordinateSystem.x !== "east" || coordinateSystem.y !== "north" || coordinateSystem.z !== "up") fail("coordinateSystem must declare ENU meters (east/north/up)"); + const placement = manifest?.placement || {}; + for (const key of ["longitude", "latitude", "height", "headingCorrectionDegrees"]) if (!Number.isFinite(placement[key])) fail(`placement.${key} must be finite`); + if (Number.isFinite(placement.longitude) && (placement.longitude < -180 || placement.longitude > 180)) fail("placement.longitude is out of range"); + if (Number.isFinite(placement.latitude) && (placement.latitude < -90 || placement.latitude > 90)) fail("placement.latitude is out of range"); + const bounds = manifest?.bounds || {}; + for (const key of ["minLon", "minLat", "maxLon", "maxLat"]) if (!Number.isFinite(bounds[key])) fail(`bounds.${key} must be finite`); + if (Number.isFinite(bounds.minLon) && Number.isFinite(bounds.maxLon) && bounds.minLon > bounds.maxLon) fail("bounds longitude order is invalid"); + if (Number.isFinite(bounds.minLat) && Number.isFinite(bounds.maxLat) && bounds.minLat > bounds.maxLat) fail("bounds latitude order is invalid"); + if (!Array.isArray(manifest?.assets) || !manifest.assets.length) fail("assets must be a non-empty array"); + const ids = new Set(); let sceneCount = 0; + for (const asset of manifest?.assets || []) { + if (!asset || typeof asset.id !== "string" || !asset.id) { fail("asset id is required"); continue; } + if (ids.has(asset.id)) fail(`duplicate asset id '${asset.id}'`); ids.add(asset.id); + if (!ROLES.has(asset.role)) fail(`asset '${asset.id}' has invalid role`); + if (!CATEGORIES.has(asset.category)) fail(`asset '${asset.id}' has invalid category`); + if (typeof asset.defaultLoad !== "boolean") fail(`asset '${asset.id}' defaultLoad must be boolean`); + try { relativeUri(asset.uri); } catch (error) { fail(error.message); continue; } + if (asset.role === "scene") { + sceneCount += 1; + if (asset.category !== "scene" || !asset.defaultLoad) fail("scene asset must be category scene and defaultLoad true"); + } else if (asset.category === "scene" || asset.defaultLoad) fail("layer assets must use a semantic category and defaultLoad false"); + if (packageDir && options.requireFiles !== false) { + const resolved = path.resolve(packageDir, asset.uri); + if (!resolved.startsWith(`${path.resolve(packageDir)}${path.sep}`) || !fs.existsSync(resolved)) fail(`asset '${asset.id}' is missing from package: ${asset.uri}`); + } + } + if (sceneCount !== 1) fail("exactly one scene asset is required"); + if (errors.length) throw new Error(`Invalid asset package manifest: ${errors.join("; ")}`); + return manifest; +} + +function addIntegrity(manifest, packageDir) { + for (const asset of manifest.assets) { + const file = path.resolve(packageDir, asset.uri); + const buffer = fs.readFileSync(file); + asset.integrity = { bytes: buffer.length, sha256: crypto.createHash("sha256").update(buffer).digest("hex") }; + } + return manifest; +} + +module.exports = { SCHEMA, CATEGORIES, ROLES, relativeUri, validateManifest, addIntegrity }; diff --git a/scripts/test-asset-budgets.js b/scripts/test-asset-budgets.js index 71500b1..3e8d1c6 100644 --- a/scripts/test-asset-budgets.js +++ b/scripts/test-asset-budgets.js @@ -77,6 +77,7 @@ assert.equal("compressedMetadata" in area.outputs, false); assert.equal("compressedCesiumPreview" in area.outputs, false); fs.mkdirSync(area.outputs.geojsonDir, { recursive: true }); for (const file of [area.outputs.glb, area.outputs.metadata, area.outputs.cesiumPreview, area.outputs.vehicleRoute, area.outputs.vehicleModel]) { + fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, "fixture\n"); } const lanePolygons = path.join(area.outputs.geojsonDir, "lane_polygons.geojson"); diff --git a/scripts/test-build-stages.js b/scripts/test-build-stages.js index bcfeb9b..6818c19 100644 --- a/scripts/test-build-stages.js +++ b/scripts/test-build-stages.js @@ -2,8 +2,8 @@ "use strict"; const assert = require("assert"); const { resolveStages, canonicalStages } = require("./lib/build-stages"); -assert.deepEqual(canonicalStages(resolveStages({}, ["compress", "blender", "preview"])), ["blender", "preview", "compress"]); -assert.deepEqual(canonicalStages(resolveStages({}, ["all"])), ["intermediates", "blender", "cesium", "compress"]); -assert.deepEqual(canonicalStages(resolveStages({ intermediates: true, blender: true, cesium: true, compress: true })), ["intermediates", "blender", "cesium", "compress"]); +assert.deepEqual(canonicalStages(resolveStages({}, ["compress", "blender", "preview"])), ["blender", "compress", "preview"]); +assert.deepEqual(canonicalStages(resolveStages({}, ["all"])), ["intermediates", "blender", "cesium", "compress", "package", "preview"]); +assert.deepEqual(canonicalStages(resolveStages({ intermediates: true, blender: true, cesium: true, compress: true, package: true })), ["intermediates", "blender", "cesium", "compress", "package"]); assert.throws(() => resolveStages({}, ["intermediates", "reimport"]), /mutually exclusive/); console.log("Build stage tests passed."); diff --git a/scripts/test-package-contract.js b/scripts/test-package-contract.js new file mode 100644 index 0000000..d7c079c --- /dev/null +++ b/scripts/test-package-contract.js @@ -0,0 +1,24 @@ +#!/usr/bin/env node +"use strict"; + +const assert = require("assert"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { SCHEMA, validateManifest } = require("./lib/package-contract"); + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "asset-package-")); +fs.mkdirSync(path.join(dir, "models")); +fs.writeFileSync(path.join(dir, "models", "area.glb"), "glb"); +const valid = { schema: SCHEMA, packageVersion: "1.0.0", areaId: "area", coordinateSystem: { axes: "ENU", units: "meters", x: "east", y: "north", z: "up" }, placement: { longitude: 120, latitude: 30, height: 0, headingCorrectionDegrees: -90 }, bounds: { minLon: 119, minLat: 29, maxLon: 121, maxLat: 31 }, assets: [{ id: "main", role: "scene", category: "scene", uri: "models/area.glb", defaultLoad: true }] }; +assert.doesNotThrow(() => validateManifest(valid, dir)); +for (const change of [ + (m) => { m.schema = "bad"; }, (m) => { m.assets[0].uri = "../area.glb"; }, + (m) => { m.assets[0].category = "dynamic"; }, (m) => { m.assets.push({ ...m.assets[0] }); }, + (m) => { m.placement.longitude = 900; }, +]) { + const manifest = JSON.parse(JSON.stringify(valid)); change(manifest); + assert.throws(() => validateManifest(manifest, dir)); +} +fs.rmSync(dir, { recursive: true, force: true }); +console.log("Asset package contract tests passed."); diff --git a/scripts/test-package-examples.js b/scripts/test-package-examples.js new file mode 100644 index 0000000..67b8b97 --- /dev/null +++ b/scripts/test-package-examples.js @@ -0,0 +1,9 @@ +#!/usr/bin/env node +"use strict"; +const assert = require("assert"); +const { resolveAsset, placement } = require("../examples/asset-package-resolver"); +const manifest = { coordinateSystem: { axes: "ENU" }, placement: { longitude: 120, latitude: 30, height: 2, headingCorrectionDegrees: -90 }, assets: [{ id: "main", uri: "models/a.glb" }, { id: "roads", uri: "models/roads.glb" }] }; +assert.equal(resolveAsset("https://consumer.example/copied/manifest.json", manifest), "https://consumer.example/copied/models/a.glb"); +assert.equal(resolveAsset("https://consumer.example/copied/manifest.json", manifest, "roads"), "https://consumer.example/copied/models/roads.glb"); +assert.equal(placement(manifest).coordinateSystem.axes, "ENU"); +console.log("Asset package example resolver tests passed.");