que01/osmWorkflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: import native road packages from ZIP

Browse Source
This commit is contained in:
que01
2026-08-26 12:51:58 +08:00
parent 009fe34f79
commit 113ea54535
15 changed files with 200 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
scripts/lib/area-config.js
View File

@@ -25,6 +25,10 @@ function normalizeAreaConfig(raw, options = {}) {
const fileStem = outputOverrides.fileStem || id;
const compress = normalizeCompressConfig(raw.compress);
const budget = normalizeBudgetConfig(raw.budget);
const roadProvider = roadProviderOption(raw.blender?.roadProvider ?? "native");
if (roadProvider === "native" && raw.nativeRoadPackage === undefined) {
throw new Error("Missing config key: nativeRoadPackage");
}
const pipelineDir = path.resolve(outputOverrides.pipelineDir || path.join(areaDir, "_pipeline"));
const packageDir = path.resolve(outputOverrides.packageDir || path.join(areaDir, "package"));
const packageStagingDir = path.resolve(outputOverrides.packageStagingDir || path.join(pipelineDir, "package-staging"));
@@ -32,13 +36,9 @@ function normalizeAreaConfig(raw, options = {}) {
const packageStagingRuntimeDir = path.resolve(outputOverrides.packageStagingRuntimeDir || path.join(packageStagingDir, "runtime"));
const previewDir = path.resolve(outputOverrides.previewDir || path.join(areaDir, "_preview"));
const geojsonDir = path.resolve(outputOverrides.geojsonDir || path.join(areaDir, "osm2streets_web_out"));
const nativeRoadDir = path.resolve(outputOverrides.nativeRoadDir || path.join(areaDir, "native-road"));
const outputs = {
areaDir,
geojsonDir,
nativeRoadDir,
nativeRoadOverrides: path.resolve(outputOverrides.nativeRoadOverrides || path.join(areaDir, "native-road-overrides.json")),
nativeTrafficSignals: path.resolve(outputOverrides.nativeTrafficSignals || path.join(areaDir, "native-traffic-signals.json")),
gpkg: path.resolve(outputOverrides.gpkg || path.join(areaDir, `${fileStem}.gpkg`)),
qgisProject: path.resolve(outputOverrides.qgisProject || path.join(areaDir, `${fileStem}.qgz`)),
qgisPreview: path.resolve(outputOverrides.qgisPreview || path.join(areaDir, `${fileStem}-preview.png`)),
@@ -81,6 +81,9 @@ function normalizeAreaConfig(raw, options = {}) {
return {
id,
nativeRoadPackage: raw.nativeRoadPackage === undefined
? null
: path.resolve(options.configDir || repoRoot, requireText(raw.nativeRoadPackage, "nativeRoadPackage")),
input,
outputRoot,
qgisApp: raw.qgisApp || (process.platform === "darwin" ? "/Applications/QGIS.app" : "/usr"),
@@ -111,10 +114,6 @@ function normalizeAreaConfig(raw, options = {}) {
turnLaneArrows: {
enabled: booleanOption(raw.turnLaneArrows?.enabled, false, "turnLaneArrows.enabled"),
},
nativeRoad: {
edgeLines: booleanOption(raw.nativeRoad?.edgeLines, false, "nativeRoad.edgeLines"),
junctionTemplates: normalizeJunctionTemplates(raw.nativeRoad?.junctionTemplates, repoRoot, options.configDir || repoRoot),
},
osm2streets: raw.osm2streets || {
debug_each_step: false,
dual_carriageway_experiment: false,
@@ -125,7 +124,7 @@ function normalizeAreaConfig(raw, options = {}) {
blender: {
treeStyle: raw.blender?.treeStyle || "natural",
officeOverrides: raw.blender?.officeOverrides || raw.blender?.office_overrides || "",
roadProvider: roadProviderOption(raw.blender?.roadProvider ?? "native"),
roadProvider,
},
v2xPreview: normalizeV2xPreviewConfig(raw.v2xPreview),
compress,
@@ -272,24 +271,8 @@ function booleanOption(value, fallback, label) {
throw new Error(`${label} must be boolean`);
}
function toRoadCompilerInput(area) {
return {
areaId: area.id,
osmFile: area.input,
outDir: area.outputs.nativeRoadDir,
stagingDir: area.outputs.pipelineDir,
overridesFile: area.outputs.nativeRoadOverrides,
trafficSignalsFile: area.outputs.nativeTrafficSignals,
comparisonDir: area.outputs.geojsonDir,
options: {
edgeLines: area.nativeRoad.edgeLines,
junctionTemplates: area.nativeRoad.junctionTemplates,
},
};
}
module.exports = {
normalizeAreaConfig,
readAreaConfig,
toRoadCompilerInput,
};

33
scripts/lib/area-diagnostics.js
View File

@@ -3,6 +3,7 @@
const fs = require("fs");
const path = require("path");
const { readAreaConfig } = require("./area-config");
const { importNativeRoadPackage } = require("./native-road-package");
const {
SCENE_LAYERS,
SCENE_FILE,
@@ -324,6 +325,11 @@ function addEndpoint(map, ref) {
function artifactStatus(area) {
const native = area.blender.roadProvider === "native";
let nativePackage = null;
let nativePackageError = null;
if (native) {
try { nativePackage = importNativeRoadPackage(area); } catch (error) { nativePackageError = error.message; }
}
const entries = [
...(native ? [] : [
["GeoJSON dir", area.outputs.geojsonDir, true, "dir"],
@@ -332,15 +338,16 @@ function artifactStatus(area) {
["QGIS preview", area.outputs.qgisPreview, true, "file"],
["Traffic signal assemblies", area.outputs.trafficSignalAssemblies, true, "file"],
]),
["Native road directory", area.outputs.nativeRoadDir, native, "dir"],
["Traffic signal runtime", native ? path.join(area.outputs.nativeRoadDir, "traffic-signals.json") : area.outputs.packageTrafficSignals, true, "file"],
["Native road ZIP", area.nativeRoadPackage, native, "file"],
["Native road import", nativePackage?.rootDir || path.join(area.outputs.pipelineDir, "native-road-import"), native, "dir"],
["Traffic signal runtime", native ? path.join(nativePackage?.rootDir || "", "traffic-signals.json") : area.outputs.packageTrafficSignals, true, "file"],
["Blend scene", area.outputs.blend, true, "file"],
["Render PNG", area.outputs.render, true, "file"],
["Package manifest", area.outputs.packageManifest, true, "file"],
["Package primary GLB", area.outputs.packagePrimaryGlb, true, "file"],
["Cesium preview", area.outputs.cesiumPreview, true, "file"],
];
return entries.map(([label, file, expected, type]) => {
const result = entries.map(([label, file, expected, type]) => {
const exists = fs.existsSync(file);
const stat = exists ? fs.statSync(file) : null;
const validType = !exists || (type === "dir" ? stat.isDirectory() : stat.isFile());
@@ -358,6 +365,8 @@ function artifactStatus(area) {
geojsonFiles,
};
});
if (nativePackageError) result.push({ label: "Native road ZIP validation", path: area.nativeRoadPackage, expected: true, type: "file", exists: false, validType: false, error: nativePackageError });
return result;
}
function metadataSummary(file, warnings) {
@@ -378,6 +387,10 @@ function metadataSummary(file, warnings) {
function stageManifestStatus(area, configPath = null) {
const native = area.blender.roadProvider === "native";
let nativePackage = null;
if (native) {
try { nativePackage = importNativeRoadPackage(area); } catch (_) { /* artifact status reports the validation failure */ }
}
const compressionComplete = fs.existsSync(stageManifestPath(area, "compress"));
const reimportManifest = stageManifestPath(area, "reimport");
const hasReimportManifest = fs.existsSync(reimportManifest);
@@ -435,7 +448,7 @@ function stageManifestStatus(area, configPath = null) {
inputs: {
...(configPath ? { config: configPath } : {}),
osm: area.input,
...(native ? nativeRoadRecords(area) : {
...(native && nativePackage ? nativeRoadRecords(nativePackage) : {
geojsonDir: area.outputs.geojsonDir,
...sceneGeojsonFiles(area),
trafficSignalAssemblies: area.outputs.trafficSignalAssemblies,
@@ -468,7 +481,7 @@ function stageManifestStatus(area, configPath = null) {
glb: area.outputs.glb,
metadata: area.outputs.metadata,
}),
...(native ? nativeRoadRecords(area) : {
...(native && nativePackage ? nativeRoadRecords(nativePackage) : {
lanePolygons: path.join(area.outputs.geojsonDir, "lane_polygons.geojson"),
network: path.join(area.outputs.geojsonDir, "network.json"),
intersectionSurface: path.join(area.outputs.geojsonDir, "intersection_surface.geojson"),
@@ -609,11 +622,13 @@ function sceneGeojsonFiles(area) {
return files;
}
function nativeRoadRecords(area) {
const root = path.join(area.outputs.nativeRoadDir, "layers");
function nativeRoadRecords(nativePackage) {
const root = path.join(nativePackage.rootDir, "layers");
return {
nativeRoadCompiled: path.join(area.outputs.nativeRoadDir, "compiled.json"),
nativeRoadSignals: path.join(area.outputs.nativeRoadDir, "traffic-signals.json"),
nativeRoadZip: nativePackage.zipFile,
nativeRoadImport: nativePackage.rootDir,
nativeRoadCompiled: path.join(nativePackage.rootDir, "compiled.json"),
nativeRoadSignals: path.join(nativePackage.rootDir, "traffic-signals.json"),
nativeRoadSurface: path.join(root, "road_surface.geojson"),
nativeEdgeLines: path.join(root, "edge_lines.geojson"),
nativeIntersectionSurface: path.join(root, "intersection_surface.geojson"),

3
scripts/lib/native-preview-traffic-simulation.js
View File

@@ -19,7 +19,8 @@ const DEFAULT_SETTINGS = {
};
function buildNativeTrafficSimulation(area, options = {}) {
const root = area.outputs.nativeRoadDir;
const root = options.nativePackage?.rootDir;
if (!root) throw new Error("Native traffic simulation requires an imported native road package");
const compiledPath = path.join(root, "compiled.json");
const signalPath = path.join(root, "traffic-signals.json");
const connectorPath = path.join(root, "layers", "connectors.geojson");

87
scripts/lib/native-road-package.js Normal file
View File

@@ -0,0 +1,87 @@
"use strict";
const crypto = require("crypto");
const fs = require("fs");
const path = require("path");
const { unzipSync } = require("fflate");
const CONTRACT = "native-road-package/v1.1";
const ROOT_FILES = ["manifest.json", "compiled.json", "diagnostics.json", "comparison.json", "traffic-signal-assemblies.json", "traffic-signals.json"];
function sha256(file) { return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); }
function readCentralEntries(bytes) {
const entries = [];
for (let i = 0; i + 46 <= bytes.length; i += 1) {
if (bytes.readUInt32LE(i) !== 0x02014b50) continue;
const nameLength = bytes.readUInt16LE(i + 28);
const extraLength = bytes.readUInt16LE(i + 30);
const commentLength = bytes.readUInt16LE(i + 32);
const name = bytes.subarray(i + 46, i + 46 + nameLength).toString("utf8");
const externalAttrs = bytes.readUInt32LE(i + 38);
entries.push({ name, externalAttrs, directory: name.endsWith("/") });
i += 45 + nameLength + extraLength + commentLength;
}
if (!entries.length) throw new Error("ZIP central directory is missing");
return entries;
}
function validateName(name) {
if (!name || path.posix.isAbsolute(name) || name.split("/").includes("..") || name.includes("\\")) throw new Error(`Unsafe ZIP entry path: ${name}`);
if (name.startsWith("./") || (name.includes("/") && !name.startsWith("layers/")) || (name.startsWith("layers/") && name.split("/").length !== 2)) throw new Error(`Native road ZIP must be root-flat: ${name}`);
}
function validateManifest(rootDir, areaId) {
let manifest;
try { manifest = JSON.parse(fs.readFileSync(path.join(rootDir, "manifest.json"), "utf8")); } catch (error) { throw new Error(`Invalid native road manifest: ${error.message}`); }
if (!manifest || manifest.contract !== CONTRACT) throw new Error(`Unsupported native road contract: ${manifest?.contract || "missing"}`);
if (manifest.areaId !== areaId) throw new Error(`Native road areaId mismatch: expected ${areaId}, got ${manifest.areaId}`);
if (!Array.isArray(manifest.layers) || !manifest.layers.length) throw new Error("Native road manifest layers are missing");
const declared = new Set();
for (const layer of manifest.layers) {
if (!layer || typeof layer.source !== "string" || layer.source !== path.posix.basename(layer.source) || declared.has(layer.source)) throw new Error("Invalid native road layer declaration");
if (!["surface", "marking", "semantic"].includes(layer.role)) throw new Error(`Invalid native road layer role: ${layer.role}`);
if (layer.role !== "semantic" && typeof layer.materialLayer !== "string") throw new Error(`Renderable layer ${layer.source} has no materialLayer`);
declared.add(layer.source);
}
const published = new Set(fs.readdirSync(path.join(rootDir, "layers")).filter((name) => name.endsWith(".geojson")).map((name) => name.slice(0, -8)));
if (declared.size !== published.size || [...declared].some((name) => !published.has(name))) throw new Error("Native road manifest/source mismatch");
return manifest;
}
function importNativeRoadPackage(area) {
const zipFile = area.nativeRoadPackage;
if (!zipFile || !fs.existsSync(zipFile) || !fs.statSync(zipFile).isFile()) throw new Error(`Native road ZIP not found: ${zipFile || "missing nativeRoadPackage"}`);
const bytes = fs.readFileSync(zipFile);
const hash = crypto.createHash("sha256").update(bytes).digest("hex");
const entries = readCentralEntries(bytes);
const names = new Set();
for (const entry of entries) {
validateName(entry.name);
if (names.has(entry.name)) throw new Error(`Duplicate ZIP entry: ${entry.name}`);
names.add(entry.name);
const unixMode = entry.externalAttrs >>> 16;
if ((unixMode & 0xf000) === 0xa000) throw new Error(`Symbolic links are not allowed: ${entry.name}`);
}
let files;
try { files = unzipSync(bytes); } catch (error) { throw new Error(`Invalid ZIP archive: ${error.message}`); }
const expected = new Set([...ROOT_FILES, ...[...names].filter((name) => name.startsWith("layers/") && name.endsWith(".geojson"))]);
for (const name of ROOT_FILES) if (!names.has(name)) throw new Error(`Missing native road ZIP entry: ${name}`);
for (const name of names) if (!expected.has(name) || (!name.startsWith("layers/") && !ROOT_FILES.includes(name))) throw new Error(`Unexpected native road ZIP entry: ${name}`);
const rootDir = path.join(area.outputs.pipelineDir, "native-road-import", hash);
if (!fs.existsSync(rootDir)) {
fs.mkdirSync(path.join(rootDir, "layers"), { recursive: true });
for (const [name, data] of Object.entries(files)) {
if (!names.has(name) || name.endsWith("/")) continue;
const destination = path.join(rootDir, name);
fs.mkdirSync(path.dirname(destination), { recursive: true });
fs.writeFileSync(destination, data);
}
}
const manifest = validateManifest(rootDir, area.id);
const records = {};
for (const name of names) { const file = path.join(rootDir, name); if (fs.existsSync(file) && fs.statSync(file).isFile()) records[name] = { path: file, bytes: fs.statSync(file).size, sha256: sha256(file) }; }
return Object.freeze({ zipFile, zipRecord: { path: zipFile, bytes: bytes.length, sha256: hash }, rootDir, manifest, records });
}
module.exports = { CONTRACT, importNativeRoadPackage };