diff --git a/config/areas/fengshu-er-road.json b/config/areas/fengshu-er-road.json index e1382d2..c748cf2 100644 --- a/config/areas/fengshu-er-road.json +++ b/config/areas/fengshu-er-road.json @@ -1,6 +1,7 @@ { "id": "fengshu-er-road", "input": "/Users/que01/osm2streets-qgis-workflow/inputs/osm/枫树二路.osm", + "nativeRoadPackage": "../../inputs/native-road-packages/fengshu-er-road.zip", "outputRoot": "/Users/que01/osm2streets-qgis-workflow/outputs", "qgisApp": "/Applications/QGIS.app", "blenderApp": "/Applications/Blender.app", @@ -24,31 +25,6 @@ "turnLaneArrows": { "enabled": false }, - "nativeRoad": { - "edgeLines": false, - "junctionTemplates": { - "enabled": true, - "references": [], - "clusters": [ - { - "id": "zhushanhu-fengshu-complex", - "template": "complex-junction-v1", - "referenceFile": "../../inputs/osm/珠山湖大道（枫树二路）口.geojson", - "approachWidthMultiplier": 1.45, - "approachLengthMeters": 32, - "coreRadiusMeters": 28, - "cornerRadiusMeters": 25, - "outerRadiusExtraMeters": 24, - "nodeIds": [ - "8005332807", - "8024512135", - "8024512145", - "8024512147" - ] - } - ] - } - }, "osm2streets": { "debug_each_step": false, "dual_carriageway_experiment": false, diff --git a/config/areas/nantaizi-lake-innovation-valley.json b/config/areas/nantaizi-lake-innovation-valley.json index 959caed..da3fcb9 100644 --- a/config/areas/nantaizi-lake-innovation-valley.json +++ b/config/areas/nantaizi-lake-innovation-valley.json @@ -1,6 +1,7 @@ { "id": "nantaizi-lake-innovation-valley", "input": "/Users/que01/Desktop/南台子湖创新谷OSM.osm", + "nativeRoadPackage": "../../inputs/native-road-packages/nantaizi-lake-innovation-valley.zip", "outputRoot": "/Users/que01/osm2streets-qgis-workflow/outputs", "qgisApp": "/Applications/QGIS.app", "blenderApp": "/Applications/Blender.app", @@ -24,9 +25,6 @@ "turnLaneArrows": { "enabled": true }, - "nativeRoad": { - "edgeLines": false - }, "osm2streets": { "debug_each_step": false, "dual_carriageway_experiment": false, diff --git a/config/examples/template.json b/config/examples/template.json index bb84a57..9be08a7 100644 --- a/config/examples/template.json +++ b/config/examples/template.json @@ -1,6 +1,7 @@ { "id": "my-area", "input": "/absolute/path/to/input.osm", + "nativeRoadPackage": "/absolute/path/to/exported-native-road-package.zip", "outputRoot": "/absolute/path/to/outputs", "qgisApp": "/Applications/QGIS.app", "blenderApp": "/Applications/Blender.app", @@ -24,14 +25,6 @@ "turnLaneArrows": { "enabled": false }, - "nativeRoad": { - "edgeLines": false, - "junctionTemplates": { - "enabled": false, - "references": [], - "clusters": [] - } - }, "osm2streets": { "debug_each_step": false, "dual_carriageway_experiment": false, diff --git a/inputs/native-road-packages/fengshu-er-road.zip b/inputs/native-road-packages/fengshu-er-road.zip new file mode 100644 index 0000000..3d048a3 Binary files /dev/null and b/inputs/native-road-packages/fengshu-er-road.zip differ diff --git a/inputs/native-road-packages/nantaizi-lake-innovation-valley.zip b/inputs/native-road-packages/nantaizi-lake-innovation-valley.zip new file mode 100644 index 0000000..f6eb4d2 Binary files /dev/null and b/inputs/native-road-packages/nantaizi-lake-innovation-valley.zip differ diff --git a/package-lock.json b/package-lock.json index 3cdc36f..4a12f8f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@inquirer/prompts": "^8.5.2", "@osm-asset/road-compiler": "git+https://git.app.que01.top/que01/road-compiler.git#v0.2.2", + "fflate": "0.8.3", "ol": "^10.10.0", "osm2streets-js-node": "0.1.4" } diff --git a/package.json b/package.json index 398af2f..8a8faf8 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "serve:v2x-preview": "node scripts/v2x-preview-server.js", "test:road-workbench": "node scripts/test-road-workbench.js", "test:native-road": "node scripts/test-native-road.js", + "test:native-road-package": "node scripts/test-native-road-package.js", "test:road-compiler-cli": "node scripts/test-road-compiler-cli.js", "test:road-parity": "node scripts/test-road-parity.js", "test:gaode-junction-reference": "node scripts/test-gaode-junction-reference.js", @@ -38,6 +39,7 @@ "dependencies": { "@inquirer/prompts": "^8.5.2", "@osm-asset/road-compiler": "git+https://git.app.que01.top/que01/road-compiler.git#v0.2.2", + "fflate": "0.8.3", "ol": "^10.10.0", "osm2streets-js-node": "0.1.4" } diff --git a/scripts/build-area.js b/scripts/build-area.js index 04322f5..e03990b 100755 --- a/scripts/build-area.js +++ b/scripts/build-area.js @@ -4,7 +4,7 @@ const fs = require("fs"); const path = require("path"); const { spawnSync } = require("child_process"); const { readAreaConfig } = require("./lib/area-config"); -const { compileArea: compileNativeRoads } = require("./lib/road-compiler-cli"); +const { importNativeRoadPackage } = require("./lib/native-road-package"); const { resolveStages } = require("./lib/build-stages"); const { validateManifest, addIntegrity } = require("./lib/package-contract"); const { blenderExecutable: resolveBlenderExecutable } = require("./lib/tool-paths"); @@ -225,10 +225,7 @@ function buildBlenderScene(area, roadProvider) { } fs.mkdirSync(path.dirname(area.outputs.blend), { recursive: true }); fs.mkdirSync(path.dirname(area.outputs.render), { recursive: true }); - if (roadProvider === "native") { - compileNativeRoads(area); - ensureNativeRoadLayers(area); - } + const nativePackage = roadProvider === "native" ? importNativeRoadPackage(area) : null; const blenderArgs = [ "--background", @@ -246,8 +243,8 @@ function buildBlenderScene(area, roadProvider) { area.blender.treeStyle, ]; if (roadProvider === "native") { - blenderArgs.push("--native-road", area.outputs.nativeRoadDir); - blenderArgs.push("--traffic-signals", path.join(area.outputs.nativeRoadDir, "traffic-signals.json")); + blenderArgs.push("--native-road", nativePackage.rootDir); + blenderArgs.push("--traffic-signals", path.join(nativePackage.rootDir, "traffic-signals.json")); } else { blenderArgs.push("--geojson", area.outputs.geojsonDir); } @@ -270,7 +267,7 @@ function buildBlenderScene(area, roadProvider) { inputs: { config: fileRecord(configPath), osm: fileRecord(area.input), - ...(roadProvider === "native" ? nativeRoadRecords(area) : sceneGeojsonRecords(area)), + ...(roadProvider === "native" ? nativeRoadRecords(nativePackage) : sceneGeojsonRecords(area)), ...(roadProvider === "osm2streets" ? { geojsonDir: fileRecord(area.outputs.geojsonDir), trafficSignalAssemblies: fileRecord(area.outputs.trafficSignalAssemblies), @@ -282,7 +279,7 @@ function buildBlenderScene(area, roadProvider) { render: fileRecord(area.outputs.render), }, summary: { - ...(roadProvider === "native" ? { nativeRoad: nativeRoadFeatureCounts(area) } : { geojson: geojsonFeatureCounts(area) }), + ...(roadProvider === "native" ? { nativeRoad: nativeRoadFeatureCounts(nativePackage) } : { geojson: geojsonFeatureCounts(area) }), roadProvider, blendBytes: fileRecord(area.outputs.blend).bytes, renderBytes: fileRecord(area.outputs.render).bytes, @@ -291,17 +288,13 @@ function buildBlenderScene(area, roadProvider) { }); } -function ensureNativeRoadLayers(area) { - ensureFile(path.join(area.outputs.nativeRoadDir, "compiled.json"), "Native road compilation"); - for (const file of ["road_surface.geojson", "edge_lines.geojson", "intersection_surface.geojson", "sidewalk_surface.geojson", "lane_separators.geojson", "center_lines.geojson", "direction_arrows.geojson", "turn_arrows.geojson", "crosswalks.geojson", "vehicle_stop_lines.geojson"]) { - ensureFile(path.join(area.outputs.nativeRoadDir, "layers", file), `Native road layer ${file}`); - } -} - -function nativeRoadRecords(area) { - const root = path.join(area.outputs.nativeRoadDir, "layers"); +function nativeRoadRecords(nativePackage) { + const root = path.join(nativePackage.rootDir, "layers"); return { - nativeRoadCompiled: fileRecord(path.join(area.outputs.nativeRoadDir, "compiled.json")), + nativeRoadZip: nativePackage.zipRecord, + nativeRoadImport: fileRecord(nativePackage.rootDir), + nativeRoadCompiled: fileRecord(path.join(nativePackage.rootDir, "compiled.json")), + nativeRoadSignals: fileRecord(path.join(nativePackage.rootDir, "traffic-signals.json")), nativeRoadSurface: fileRecord(path.join(root, "road_surface.geojson")), nativeEdgeLines: fileRecord(path.join(root, "edge_lines.geojson")), nativeIntersectionSurface: fileRecord(path.join(root, "intersection_surface.geojson")), @@ -315,8 +308,8 @@ function nativeRoadRecords(area) { }; } -function nativeRoadFeatureCounts(area) { - const root = path.join(area.outputs.nativeRoadDir, "layers"); +function nativeRoadFeatureCounts(nativePackage) { + const root = path.join(nativePackage.rootDir, "layers"); return { roadSurface: featureCount(path.join(root, "road_surface.geojson")), edgeLines: featureCount(path.join(root, "edge_lines.geojson")), @@ -487,8 +480,9 @@ function compressCesiumGlb(area) { function publishPackage(area, roadProvider) { ensureFile(area.outputs.packageStagingManifest, "Staged package manifest"); - const trafficSignalsSource = roadProvider === "native" - ? path.join(area.outputs.nativeRoadDir, "traffic-signals.json") + const nativePackage = roadProvider === "native" ? importNativeRoadPackage(area) : null; + const trafficSignalsSource = nativePackage + ? path.join(nativePackage.rootDir, "traffic-signals.json") : area.outputs.trafficSignals; ensureFile(trafficSignalsSource, "Traffic signal anchors"); const started = Date.now(); @@ -574,7 +568,7 @@ function writeCesiumPreview(area, roadProvider) { intersectionSurface: fileRecord(intersectionSurface), }); } else { - Object.assign(previewInputs, nativeRoadRecords(area)); + Object.assign(previewInputs, nativeRoadRecords(importNativeRoadPackage(area))); // This operational preview intentionally has no synthetic vehicle source. // A stale descriptor from an older build must not resurrect simulated routes. fs.rmSync(area.outputs.trafficSimulation, { force: true }); diff --git a/scripts/lib/area-config.js b/scripts/lib/area-config.js index 4eee8db..ef233b9 100644 --- a/scripts/lib/area-config.js +++ b/scripts/lib/area-config.js @@ -25,6 +25,10 @@ function normalizeAreaConfig(raw, options = {}) { const fileStem = outputOverrides.fileStem || id; const compress = normalizeCompressConfig(raw.compress); const budget = normalizeBudgetConfig(raw.budget); + const roadProvider = roadProviderOption(raw.blender?.roadProvider ?? "native"); + if (roadProvider === "native" && raw.nativeRoadPackage === undefined) { + throw new Error("Missing config key: nativeRoadPackage"); + } const pipelineDir = path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline")); const packageDir = path.resolve(outputOverrides.packageDir || path.join(areaDir, "package")); const packageStagingDir = path.resolve(outputOverrides.packageStagingDir || path.join(pipelineDir, "package-staging")); @@ -32,13 +36,9 @@ function normalizeAreaConfig(raw, options = {}) { const packageStagingRuntimeDir = path.resolve(outputOverrides.packageStagingRuntimeDir || path.join(packageStagingDir, "runtime")); const previewDir = path.resolve(outputOverrides.previewDir || path.join(areaDir, "_preview")); const geojsonDir = path.resolve(outputOverrides.geojsonDir || path.join(areaDir, "osm2streets_web_out")); - const nativeRoadDir = path.resolve(outputOverrides.nativeRoadDir || path.join(areaDir, "native-road")); const outputs = { areaDir, geojsonDir, - nativeRoadDir, - nativeRoadOverrides: path.resolve(outputOverrides.nativeRoadOverrides || path.join(areaDir, "native-road-overrides.json")), - nativeTrafficSignals: path.resolve(outputOverrides.nativeTrafficSignals || path.join(areaDir, "native-traffic-signals.json")), gpkg: path.resolve(outputOverrides.gpkg || path.join(areaDir, `${fileStem}.gpkg`)), qgisProject: path.resolve(outputOverrides.qgisProject || path.join(areaDir, `${fileStem}.qgz`)), qgisPreview: path.resolve(outputOverrides.qgisPreview || path.join(areaDir, `${fileStem}-preview.png`)), @@ -81,6 +81,9 @@ function normalizeAreaConfig(raw, options = {}) { return { id, + nativeRoadPackage: raw.nativeRoadPackage === undefined + ? null + : path.resolve(options.configDir || repoRoot, requireText(raw.nativeRoadPackage, "nativeRoadPackage")), input, outputRoot, qgisApp: raw.qgisApp || (process.platform === "darwin" ? "/Applications/QGIS.app" : "/usr"), @@ -111,10 +114,6 @@ function normalizeAreaConfig(raw, options = {}) { turnLaneArrows: { enabled: booleanOption(raw.turnLaneArrows?.enabled, false, "turnLaneArrows.enabled"), }, - nativeRoad: { - edgeLines: booleanOption(raw.nativeRoad?.edgeLines, false, "nativeRoad.edgeLines"), - junctionTemplates: normalizeJunctionTemplates(raw.nativeRoad?.junctionTemplates, repoRoot, options.configDir || repoRoot), - }, osm2streets: raw.osm2streets || { debug_each_step: false, dual_carriageway_experiment: false, @@ -125,7 +124,7 @@ function normalizeAreaConfig(raw, options = {}) { blender: { treeStyle: raw.blender?.treeStyle || "natural", officeOverrides: raw.blender?.officeOverrides || raw.blender?.office_overrides || "", - roadProvider: roadProviderOption(raw.blender?.roadProvider ?? "native"), + roadProvider, }, v2xPreview: normalizeV2xPreviewConfig(raw.v2xPreview), compress, @@ -272,24 +271,8 @@ function booleanOption(value, fallback, label) { throw new Error(`${label} must be boolean`); } -function toRoadCompilerInput(area) { - return { - areaId: area.id, - osmFile: area.input, - outDir: area.outputs.nativeRoadDir, - stagingDir: area.outputs.pipelineDir, - overridesFile: area.outputs.nativeRoadOverrides, - trafficSignalsFile: area.outputs.nativeTrafficSignals, - comparisonDir: area.outputs.geojsonDir, - options: { - edgeLines: area.nativeRoad.edgeLines, - junctionTemplates: area.nativeRoad.junctionTemplates, - }, - }; -} module.exports = { normalizeAreaConfig, readAreaConfig, - toRoadCompilerInput, }; diff --git a/scripts/lib/area-diagnostics.js b/scripts/lib/area-diagnostics.js index 4d502f6..b0ddbde 100644 --- a/scripts/lib/area-diagnostics.js +++ b/scripts/lib/area-diagnostics.js @@ -3,6 +3,7 @@ const fs = require("fs"); const path = require("path"); const { readAreaConfig } = require("./area-config"); +const { importNativeRoadPackage } = require("./native-road-package"); const { SCENE_LAYERS, SCENE_FILE, @@ -324,6 +325,11 @@ function addEndpoint(map, ref) { function artifactStatus(area) { const native = area.blender.roadProvider === "native"; + let nativePackage = null; + let nativePackageError = null; + if (native) { + try { nativePackage = importNativeRoadPackage(area); } catch (error) { nativePackageError = error.message; } + } const entries = [ ...(native ? [] : [ ["GeoJSON dir", area.outputs.geojsonDir, true, "dir"], @@ -332,15 +338,16 @@ function artifactStatus(area) { ["QGIS preview", area.outputs.qgisPreview, true, "file"], ["Traffic signal assemblies", area.outputs.trafficSignalAssemblies, true, "file"], ]), - ["Native road directory", area.outputs.nativeRoadDir, native, "dir"], - ["Traffic signal runtime", native ? path.join(area.outputs.nativeRoadDir, "traffic-signals.json") : area.outputs.packageTrafficSignals, true, "file"], + ["Native road ZIP", area.nativeRoadPackage, native, "file"], + ["Native road import", nativePackage?.rootDir || path.join(area.outputs.pipelineDir, "native-road-import"), native, "dir"], + ["Traffic signal runtime", native ? path.join(nativePackage?.rootDir || "", "traffic-signals.json") : area.outputs.packageTrafficSignals, true, "file"], ["Blend scene", area.outputs.blend, true, "file"], ["Render PNG", area.outputs.render, true, "file"], ["Package manifest", area.outputs.packageManifest, true, "file"], ["Package primary GLB", area.outputs.packagePrimaryGlb, true, "file"], ["Cesium preview", area.outputs.cesiumPreview, true, "file"], ]; - return entries.map(([label, file, expected, type]) => { + const result = entries.map(([label, file, expected, type]) => { const exists = fs.existsSync(file); const stat = exists ? fs.statSync(file) : null; const validType = !exists || (type === "dir" ? stat.isDirectory() : stat.isFile()); @@ -358,6 +365,8 @@ function artifactStatus(area) { geojsonFiles, }; }); + if (nativePackageError) result.push({ label: "Native road ZIP validation", path: area.nativeRoadPackage, expected: true, type: "file", exists: false, validType: false, error: nativePackageError }); + return result; } function metadataSummary(file, warnings) { @@ -378,6 +387,10 @@ function metadataSummary(file, warnings) { function stageManifestStatus(area, configPath = null) { const native = area.blender.roadProvider === "native"; + let nativePackage = null; + if (native) { + try { nativePackage = importNativeRoadPackage(area); } catch (_) { /* artifact status reports the validation failure */ } + } const compressionComplete = fs.existsSync(stageManifestPath(area, "compress")); const reimportManifest = stageManifestPath(area, "reimport"); const hasReimportManifest = fs.existsSync(reimportManifest); @@ -435,7 +448,7 @@ function stageManifestStatus(area, configPath = null) { inputs: { ...(configPath ? { config: configPath } : {}), osm: area.input, - ...(native ? nativeRoadRecords(area) : { + ...(native && nativePackage ? nativeRoadRecords(nativePackage) : { geojsonDir: area.outputs.geojsonDir, ...sceneGeojsonFiles(area), trafficSignalAssemblies: area.outputs.trafficSignalAssemblies, @@ -468,7 +481,7 @@ function stageManifestStatus(area, configPath = null) { glb: area.outputs.glb, metadata: area.outputs.metadata, }), - ...(native ? nativeRoadRecords(area) : { + ...(native && nativePackage ? nativeRoadRecords(nativePackage) : { lanePolygons: path.join(area.outputs.geojsonDir, "lane_polygons.geojson"), network: path.join(area.outputs.geojsonDir, "network.json"), intersectionSurface: path.join(area.outputs.geojsonDir, "intersection_surface.geojson"), @@ -609,11 +622,13 @@ function sceneGeojsonFiles(area) { return files; } -function nativeRoadRecords(area) { - const root = path.join(area.outputs.nativeRoadDir, "layers"); +function nativeRoadRecords(nativePackage) { + const root = path.join(nativePackage.rootDir, "layers"); return { - nativeRoadCompiled: path.join(area.outputs.nativeRoadDir, "compiled.json"), - nativeRoadSignals: path.join(area.outputs.nativeRoadDir, "traffic-signals.json"), + nativeRoadZip: nativePackage.zipFile, + nativeRoadImport: nativePackage.rootDir, + nativeRoadCompiled: path.join(nativePackage.rootDir, "compiled.json"), + nativeRoadSignals: path.join(nativePackage.rootDir, "traffic-signals.json"), nativeRoadSurface: path.join(root, "road_surface.geojson"), nativeEdgeLines: path.join(root, "edge_lines.geojson"), nativeIntersectionSurface: path.join(root, "intersection_surface.geojson"), diff --git a/scripts/lib/native-preview-traffic-simulation.js b/scripts/lib/native-preview-traffic-simulation.js index a37415b..47b76a7 100644 --- a/scripts/lib/native-preview-traffic-simulation.js +++ b/scripts/lib/native-preview-traffic-simulation.js @@ -19,7 +19,8 @@ const DEFAULT_SETTINGS = { }; function buildNativeTrafficSimulation(area, options = {}) { - const root = area.outputs.nativeRoadDir; + const root = options.nativePackage?.rootDir; + if (!root) throw new Error("Native traffic simulation requires an imported native road package"); const compiledPath = path.join(root, "compiled.json"); const signalPath = path.join(root, "traffic-signals.json"); const connectorPath = path.join(root, "layers", "connectors.geojson"); diff --git a/scripts/lib/native-road-package.js b/scripts/lib/native-road-package.js new file mode 100644 index 0000000..d53e4a7 --- /dev/null +++ b/scripts/lib/native-road-package.js @@ -0,0 +1,87 @@ +"use strict"; + +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); +const { unzipSync } = require("fflate"); + +const CONTRACT = "native-road-package/v1.1"; +const ROOT_FILES = ["manifest.json", "compiled.json", "diagnostics.json", "comparison.json", "traffic-signal-assemblies.json", "traffic-signals.json"]; + +function sha256(file) { return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); } + +function readCentralEntries(bytes) { + const entries = []; + for (let i = 0; i + 46 <= bytes.length; i += 1) { + if (bytes.readUInt32LE(i) !== 0x02014b50) continue; + const nameLength = bytes.readUInt16LE(i + 28); + const extraLength = bytes.readUInt16LE(i + 30); + const commentLength = bytes.readUInt16LE(i + 32); + const name = bytes.subarray(i + 46, i + 46 + nameLength).toString("utf8"); + const externalAttrs = bytes.readUInt32LE(i + 38); + entries.push({ name, externalAttrs, directory: name.endsWith("/") }); + i += 45 + nameLength + extraLength + commentLength; + } + if (!entries.length) throw new Error("ZIP central directory is missing"); + return entries; +} + +function validateName(name) { + if (!name || path.posix.isAbsolute(name) || name.split("/").includes("..") || name.includes("\\")) throw new Error(`Unsafe ZIP entry path: ${name}`); + if (name.startsWith("./") || (name.includes("/") && !name.startsWith("layers/")) || (name.startsWith("layers/") && name.split("/").length !== 2)) throw new Error(`Native road ZIP must be root-flat: ${name}`); +} + +function validateManifest(rootDir, areaId) { + let manifest; + try { manifest = JSON.parse(fs.readFileSync(path.join(rootDir, "manifest.json"), "utf8")); } catch (error) { throw new Error(`Invalid native road manifest: ${error.message}`); } + if (!manifest || manifest.contract !== CONTRACT) throw new Error(`Unsupported native road contract: ${manifest?.contract || "missing"}`); + if (manifest.areaId !== areaId) throw new Error(`Native road areaId mismatch: expected ${areaId}, got ${manifest.areaId}`); + if (!Array.isArray(manifest.layers) || !manifest.layers.length) throw new Error("Native road manifest layers are missing"); + const declared = new Set(); + for (const layer of manifest.layers) { + if (!layer || typeof layer.source !== "string" || layer.source !== path.posix.basename(layer.source) || declared.has(layer.source)) throw new Error("Invalid native road layer declaration"); + if (!["surface", "marking", "semantic"].includes(layer.role)) throw new Error(`Invalid native road layer role: ${layer.role}`); + if (layer.role !== "semantic" && typeof layer.materialLayer !== "string") throw new Error(`Renderable layer ${layer.source} has no materialLayer`); + declared.add(layer.source); + } + const published = new Set(fs.readdirSync(path.join(rootDir, "layers")).filter((name) => name.endsWith(".geojson")).map((name) => name.slice(0, -8))); + if (declared.size !== published.size || [...declared].some((name) => !published.has(name))) throw new Error("Native road manifest/source mismatch"); + return manifest; +} + +function importNativeRoadPackage(area) { + const zipFile = area.nativeRoadPackage; + if (!zipFile || !fs.existsSync(zipFile) || !fs.statSync(zipFile).isFile()) throw new Error(`Native road ZIP not found: ${zipFile || "missing nativeRoadPackage"}`); + const bytes = fs.readFileSync(zipFile); + const hash = crypto.createHash("sha256").update(bytes).digest("hex"); + const entries = readCentralEntries(bytes); + const names = new Set(); + for (const entry of entries) { + validateName(entry.name); + if (names.has(entry.name)) throw new Error(`Duplicate ZIP entry: ${entry.name}`); + names.add(entry.name); + const unixMode = entry.externalAttrs >>> 16; + if ((unixMode & 0xf000) === 0xa000) throw new Error(`Symbolic links are not allowed: ${entry.name}`); + } + let files; + try { files = unzipSync(bytes); } catch (error) { throw new Error(`Invalid ZIP archive: ${error.message}`); } + const expected = new Set([...ROOT_FILES, ...[...names].filter((name) => name.startsWith("layers/") && name.endsWith(".geojson"))]); + for (const name of ROOT_FILES) if (!names.has(name)) throw new Error(`Missing native road ZIP entry: ${name}`); + for (const name of names) if (!expected.has(name) || (!name.startsWith("layers/") && !ROOT_FILES.includes(name))) throw new Error(`Unexpected native road ZIP entry: ${name}`); + const rootDir = path.join(area.outputs.pipelineDir, "native-road-import", hash); + if (!fs.existsSync(rootDir)) { + fs.mkdirSync(path.join(rootDir, "layers"), { recursive: true }); + for (const [name, data] of Object.entries(files)) { + if (!names.has(name) || name.endsWith("/")) continue; + const destination = path.join(rootDir, name); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.writeFileSync(destination, data); + } + } + const manifest = validateManifest(rootDir, area.id); + const records = {}; + for (const name of names) { const file = path.join(rootDir, name); if (fs.existsSync(file) && fs.statSync(file).isFile()) records[name] = { path: file, bytes: fs.statSync(file).size, sha256: sha256(file) }; } + return Object.freeze({ zipFile, zipRecord: { path: zipFile, bytes: bytes.length, sha256: hash }, rootDir, manifest, records }); +} + +module.exports = { CONTRACT, importNativeRoadPackage }; diff --git a/scripts/test-asset-budgets.js b/scripts/test-asset-budgets.js index f0b8bef..5c335f5 100644 --- a/scripts/test-asset-budgets.js +++ b/scripts/test-asset-budgets.js @@ -19,7 +19,7 @@ assert.match(areaBuildSource, /exportCesium\(area, roadProvider\)/); assert.match(areaBuildSource, /"--dynamic-glb", area\.outputs\.trafficSignalsDynamicGlb/); assert.match(areaBuildSource, /trafficSignalsCountdown0Glb: fileRecord/); assert.match(areaBuildSource, /if \(roadProvider === "osm2streets"\) \{[\s\S]*?buildPreviewVehicleRoute/); -assert.match(areaBuildSource, /Object\.assign\(previewInputs, nativeRoadRecords\(area\)\)/); +assert.match(areaBuildSource, /Object\.assign\(previewInputs, nativeRoadRecords\(/); assert.match(areaBuildSource, /vehicleRoute: optionalFileRecord\(area\.outputs\.vehicleRoute\)/); const gltf = { @@ -48,7 +48,7 @@ assert.equal(evaluateGlbBudget(digest, budget).violations[0].key, "glbBytes"); const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "asset-budget-")); const input = path.join(tempDir, "input.osm"); fs.writeFileSync(input, ""); -const base = { id: "test-area", input, outputRoot: tempDir }; +const base = { id: "test-area", input, outputRoot: tempDir, blender: { roadProvider: "osm2streets" } }; assert.equal( normalizeAreaConfig(base).outputs.trafficSignals, path.join(tempDir, "test-area", "osm2streets_web_out", "traffic_signals.json"), @@ -63,7 +63,7 @@ assert.equal( ); assert.equal(normalizeAreaConfig({ ...base, budget: { nodes: 800 } }).budget.glbNodes, 800); assert.equal(normalizeAreaConfig(base).stages.intermediates, false); -assert.equal(normalizeAreaConfig(base).blender.roadProvider, "native"); +assert.equal(normalizeAreaConfig(base).blender.roadProvider, "osm2streets"); assert.deepEqual(normalizeAreaConfig(base).v2xPreview, { enabled: false, apiBaseUrl: "/api", @@ -80,7 +80,7 @@ assert.throws( () => normalizeAreaConfig({ ...base, v2xPreview: "enabled" }), /v2xPreview must be an object/, ); -assert.equal(normalizeAreaConfig({ ...base, blender: { roadProvider: "native" } }).blender.roadProvider, "native"); +assert.equal(normalizeAreaConfig({ ...base, nativeRoadPackage: path.join(tempDir, "roads.zip"), blender: { roadProvider: "native" } }).blender.roadProvider, "native"); assert.throws( () => normalizeAreaConfig({ ...base, blender: { roadProvider: "other" } }), /blender\.roadProvider/, diff --git a/scripts/test-native-preview-traffic.js b/scripts/test-native-preview-traffic.js index 7d0c7d3..3941500 100644 --- a/scripts/test-native-preview-traffic.js +++ b/scripts/test-native-preview-traffic.js @@ -25,7 +25,7 @@ fs.writeFileSync(path.join(layersDir, "connectors.geojson"), JSON.stringify({ ty fs.writeFileSync(path.join(layersDir, "vehicle_stop_lines.geojson"), JSON.stringify({ type: "FeatureCollection", features: [{ type: "Feature", properties: { road_id: "road:a" }, geometry: { type: "Polygon", coordinates: [] } }] })); fs.writeFileSync(path.join(layersDir, "lane_centerlines.geojson"), JSON.stringify({ type: "FeatureCollection", features: roads.map((road) => ({ type: "Feature", properties: { road_id: road.id, lane_index: 1 }, geometry: { type: "LineString", coordinates: road.centerline } })) })); -const descriptor = buildNativeTrafficSimulation({ id: "fixture", outputs: { areaDir, nativeRoadDir } }); +const descriptor = buildNativeTrafficSimulation({ id: "fixture", outputs: { areaDir } }, { nativePackage: { rootDir: nativeRoadDir } }); assert.equal(descriptor.schema, SCHEMA); assert.equal(descriptor.coordinateSystem.route, "WGS84"); assert.ok(descriptor.routes.length >= 1); diff --git a/scripts/test-native-road-package.js b/scripts/test-native-road-package.js new file mode 100644 index 0000000..e0c69ed --- /dev/null +++ b/scripts/test-native-road-package.js @@ -0,0 +1,50 @@ +#!/usr/bin/env node +"use strict"; + +const assert = require("assert"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { zipSync } = require("fflate"); +const { importNativeRoadPackage } = require("./lib/native-road-package"); + +const temp = fs.mkdtempSync(path.join(os.tmpdir(), "native-road-package-")); +const area = { id: "fixture", nativeRoadPackage: "", outputs: { pipelineDir: path.join(temp, "pipeline") } }; +const layers = ["road_surface", "connectors"]; +const manifest = { contract: "native-road-package/v1.1", areaId: "fixture", layers: [ + { source: "road_surface", role: "surface", materialLayer: "road_surface" }, + { source: "connectors", role: "semantic" }, +] }; +function validEntries() { + return { + "manifest.json": Buffer.from(JSON.stringify(manifest)), + "compiled.json": Buffer.from("{}"), "diagnostics.json": Buffer.from("{}"), "comparison.json": Buffer.from("{}"), + "traffic-signal-assemblies.json": Buffer.from("{}"), "traffic-signals.json": Buffer.from("{}"), + ...Object.fromEntries(layers.map((layer) => [`layers/${layer}.geojson`, Buffer.from('{"type":"FeatureCollection","features":[]}')])), + }; +} +function write(name, entries = validEntries()) { const file = path.join(temp, name); fs.writeFileSync(file, zipSync(entries)); area.nativeRoadPackage = file; return file; } +write("valid.zip"); +const first = importNativeRoadPackage(area); +assert.equal(first.manifest.areaId, "fixture"); +assert.equal(Object.keys(first.records).length, 8); +assert.ok(first.rootDir.includes(first.zipRecord.sha256)); +assert.equal(importNativeRoadPackage(area).rootDir, first.rootDir); +assert.throws(() => { area.nativeRoadPackage = path.join(temp, "missing.zip"); importNativeRoadPackage(area); }, /ZIP not found/); +write("corrupt.zip", { "manifest.json": Buffer.from("{") }); +assert.throws(() => importNativeRoadPackage(area), /ZIP central directory|Invalid ZIP|Invalid native road manifest|Missing native road ZIP entry/); +write("nested.zip", { "nested/manifest.json": Buffer.from("{}") }); +assert.throws(() => importNativeRoadPackage(area), /root-flat/); +write("traversal.zip", { "../manifest.json": Buffer.from("{}") }); +assert.throws(() => importNativeRoadPackage(area), /Unsafe ZIP entry/); +const wrongContract = validEntries(); wrongContract["manifest.json"] = Buffer.from(JSON.stringify({ ...manifest, contract: "v1" })); write("contract.zip", wrongContract); +assert.throws(() => importNativeRoadPackage(area), /Unsupported native road contract/); +const wrongArea = validEntries(); wrongArea["manifest.json"] = Buffer.from(JSON.stringify({ ...manifest, areaId: "other" })); write("area.zip", wrongArea); +assert.throws(() => importNativeRoadPackage(area), /areaId mismatch/); +const mismatch = validEntries(); delete mismatch["layers/connectors.geojson"]; write("mismatch.zip", mismatch); +assert.throws(() => importNativeRoadPackage(area), /manifest\/source mismatch/); +const symlink = write("symlink.zip"); +const data = fs.readFileSync(symlink); const index = data.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); data.writeUInt32LE((0xa000 << 16) >>> 0, index + 38); fs.writeFileSync(symlink, data); +assert.throws(() => importNativeRoadPackage(area), /Symbolic links/); +fs.rmSync(temp, { recursive: true, force: true }); +console.log("Native road ZIP package tests passed.");