Initial osm2streets QGIS workflow

This commit is contained in:
2026-07-21 17:03:27 +08:00
commit dc22543e9a
10 changed files with 846 additions and 0 deletions

521
scripts/build-osm2streets-qgis.js Executable file
View File

@@ -0,0 +1,521 @@
#!/usr/bin/env node
const fs = require("fs");
const path = require("path");
const os = require("os");
const { execFileSync } = require("child_process");
const { JsStreetNetwork } = require("osm2streets-js-node");
const repoRoot = path.resolve(__dirname, "..");
const args = parseArgs(process.argv.slice(2));
const configPath = path.resolve(args.config || path.join(repoRoot, "config", "default.json"));
const config = loadConfig(configPath, args);
const qgisApp = config.qgisApp;
const qgisMacOS = path.join(qgisApp, "Contents", "MacOS");
const qgisPython = path.join(qgisMacOS, "python3.12");
const ogr2ogr = path.join(qgisMacOS, "ogr2ogr");
const inputPath = path.resolve(config.input);
const outDir = path.resolve(config.outDir);
const gpkgPath = path.resolve(config.gpkg);
const projectPath = path.resolve(config.project);
const previewPath = path.resolve(config.preview);
const arrowScale = Number(config.arrowScale);
const clipPad = Number(config.clipPad);
const canvasPad = Number(config.canvasPad);
const previewPad = Number(config.previewPad);
if (!Number.isFinite(arrowScale) || arrowScale <= 0) {
throw new Error(`Invalid arrowScale: ${config.arrowScale}`);
}
for (const [key, value] of [["clipPad", clipPad], ["canvasPad", canvasPad], ["previewPad", previewPad]]) {
if (!Number.isFinite(value) || value < 0) {
throw new Error(`Invalid ${key}: ${config[key]}`);
}
}
if (!fs.existsSync(inputPath)) {
throw new Error(`Input OSM XML not found: ${inputPath}`);
}
for (const exe of [ogr2ogr, qgisPython]) {
if (!fs.existsSync(exe)) {
throw new Error(`QGIS executable not found: ${exe}`);
}
}
fs.mkdirSync(outDir, { recursive: true });
fs.mkdirSync(path.dirname(gpkgPath), { recursive: true });
fs.mkdirSync(path.dirname(projectPath), { recursive: true });
fs.mkdirSync(path.dirname(previewPath), { recursive: true });
const xml = fs.readFileSync(inputPath, "utf8");
const bbox = getOsmBounds(xml);
const clip = makeClipPolygon(bbox, clipPad);
const network = new JsStreetNetwork(xml, JSON.stringify(clip), config.osm2streets);
writeGeoJson(outDir, "plain.geojson", network.toGeojsonPlain());
writeGeoJson(outDir, "lane_polygons.geojson", network.toLanePolygonsGeojson());
writeGeoJson(outDir, "lane_markings.geojson", network.toLaneMarkingsGeojson());
writeGeoJson(outDir, "intersection_markings.geojson", network.toIntersectionMarkingsGeojson());
fs.writeFileSync(path.join(outDir, "network.json"), network.toJson());
const split = splitLayers(outDir, arrowScale);
writeJson(path.join(outDir, "road_surface.geojson"), split.roadSurface);
writeJson(path.join(outDir, "sidewalks.geojson"), split.sidewalks);
writeJson(path.join(outDir, "lane_separators.geojson"), split.laneSeparators);
writeJson(path.join(outDir, "center_lines.geojson"), split.centerLines);
writeJson(path.join(outDir, "vehicle_stop_lines.geojson"), split.vehicleStopLines);
writeJson(path.join(outDir, "lane_arrows_webscale.geojson"), split.laneArrows);
writeJson(path.join(outDir, "sidewalk_corners.geojson"), split.sidewalkCorners);
if (fs.existsSync(gpkgPath)) {
fs.unlinkSync(gpkgPath);
}
const ogrEnv = qgisEnv();
importLayer(gpkgPath, path.join(outDir, "road_surface.geojson"), "road_surface", false, ogrEnv);
importLayer(gpkgPath, path.join(outDir, "sidewalks.geojson"), "sidewalks", true, ogrEnv);
importLayer(gpkgPath, path.join(outDir, "sidewalk_corners.geojson"), "sidewalk_corners", true, ogrEnv);
importLayer(gpkgPath, path.join(outDir, "lane_separators.geojson"), "lane_separators", true, ogrEnv);
importLayer(gpkgPath, path.join(outDir, "center_lines.geojson"), "center_lines", true, ogrEnv);
importLayer(gpkgPath, path.join(outDir, "vehicle_stop_lines.geojson"), "vehicle_stop_lines", true, ogrEnv);
importLayer(gpkgPath, path.join(outDir, "lane_arrows_webscale.geojson"), "lane_arrows_webscale", true, ogrEnv);
const qgisScript = path.join(outDir, "_create_qgis_project.py");
const defaultPreviewExtent = extentString(expandBounds(
featureBounds(split.laneArrows.features[0] || split.roadSurface.features[0]),
previewPad,
));
fs.writeFileSync(qgisScript, makeQgisScript({
qgisMacOS,
gpkgPath,
projectPath,
previewPath,
canvasExtent: config.canvasExtent || extentString(expandBounds(bbox, canvasPad)),
previewExtent: config.previewExtent || defaultPreviewExtent,
}));
execFileSync(qgisPython, [qgisScript], {
stdio: "inherit",
env: {
...process.env,
...qgisEnv(),
QT_QPA_PLATFORM: "offscreen",
PYTHONHOME: path.join(qgisApp, "Contents", "Frameworks"),
PYTHONPATH: [
path.join(qgisApp, "Contents", "Resources", "python"),
path.join(qgisApp, "Contents", "Resources", "python3.11", "site-packages"),
].join(":"),
DYLD_LIBRARY_PATH: [
qgisMacOS,
path.join(qgisApp, "Contents", "Frameworks"),
].join(":"),
},
});
fixCanvas(projectPath, config.canvasExtent || extentString(expandBounds(bbox, canvasPad)));
console.log(`Config: ${configPath}`);
console.log(`GeoPackage: ${gpkgPath}`);
console.log(`QGIS project: ${projectPath}`);
console.log(`Preview PNG: ${previewPath}`);
function parseArgs(argv) {
const out = {};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (!arg.startsWith("--")) continue;
const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase());
const next = argv[i + 1];
if (!next || next.startsWith("--")) {
out[key] = "true";
} else {
out[key] = next;
i += 1;
}
}
return out;
}
function loadConfig(file, cliArgs) {
if (!fs.existsSync(file)) {
throw new Error(`Config file not found: ${file}`);
}
const base = JSON.parse(fs.readFileSync(file, "utf8"));
const overrides = {};
const mapping = {
qgisApp: "qgisApp",
input: "input",
outDir: "outDir",
gpkg: "gpkg",
project: "project",
preview: "preview",
arrowScale: "arrowScale",
clipPad: "clipPad",
pad: "clipPad",
canvasPad: "canvasPad",
previewPad: "previewPad",
canvasExtent: "canvasExtent",
previewExtent: "previewExtent",
};
for (const [argKey, configKey] of Object.entries(mapping)) {
if (cliArgs[argKey] !== undefined) {
overrides[configKey] = cliArgs[argKey];
}
}
if (process.env.QGIS_APP && overrides.qgisApp === undefined) {
overrides.qgisApp = process.env.QGIS_APP;
}
const merged = deepMerge(base, overrides);
const required = ["qgisApp", "input", "outDir", "gpkg", "project", "preview", "osm2streets"];
for (const key of required) {
if (merged[key] === undefined || merged[key] === null || merged[key] === "") {
throw new Error(`Missing config key: ${key}`);
}
}
return merged;
}
function deepMerge(base, overrides) {
const out = { ...base };
for (const [key, value] of Object.entries(overrides)) {
if (
value &&
typeof value === "object" &&
!Array.isArray(value) &&
base[key] &&
typeof base[key] === "object" &&
!Array.isArray(base[key])
) {
out[key] = deepMerge(base[key], value);
} else {
out[key] = value;
}
}
return out;
}
function qgisEnv() {
return {
PROJ_LIB: path.join(qgisApp, "Contents", "Resources", "qgis", "proj"),
GDAL_DATA: path.join(qgisApp, "Contents", "Resources", "qgis", "gdal"),
};
}
function getOsmBounds(xmlText) {
let minLon = Infinity;
let minLat = Infinity;
let maxLon = -Infinity;
let maxLat = -Infinity;
for (const match of xmlText.matchAll(/<node\b([^>]*)>/g)) {
const attrs = match[1];
const latMatch = attrs.match(/\blat=(["'])(.*?)\1/);
const lonMatch = attrs.match(/\blon=(["'])(.*?)\1/);
if (!latMatch || !lonMatch) continue;
const lat = Number(latMatch[2]);
const lon = Number(lonMatch[2]);
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
minLon = Math.min(minLon, lon);
maxLon = Math.max(maxLon, lon);
minLat = Math.min(minLat, lat);
maxLat = Math.max(maxLat, lat);
}
if (!Number.isFinite(minLon)) {
throw new Error("No OSM node coordinates found");
}
return { minLon, minLat, maxLon, maxLat };
}
function makeClipPolygon(bbox, pad) {
const b = expandBounds(bbox, pad);
return {
type: "FeatureCollection",
features: [{
type: "Feature",
properties: {},
geometry: {
type: "Polygon",
coordinates: [[
[b.minLon, b.minLat],
[b.maxLon, b.minLat],
[b.maxLon, b.maxLat],
[b.minLon, b.maxLat],
[b.minLon, b.minLat],
]],
},
}],
};
}
function expandBounds(bbox, pad) {
return {
minLon: bbox.minLon - pad,
minLat: bbox.minLat - pad,
maxLon: bbox.maxLon + pad,
maxLat: bbox.maxLat + pad,
};
}
function centeredExtent(bbox, width, height) {
const cx = (bbox.minLon + bbox.maxLon) / 2;
const cy = (bbox.minLat + bbox.maxLat) / 2;
return {
minLon: cx - width / 2,
minLat: cy - height / 2,
maxLon: cx + width / 2,
maxLat: cy + height / 2,
};
}
function extentString(bbox) {
return `${bbox.minLon},${bbox.minLat},${bbox.maxLon},${bbox.maxLat}`;
}
function featureBounds(feature) {
if (!feature?.geometry?.coordinates) {
throw new Error("No feature available for preview extent");
}
const coords = [];
collectCoords(feature.geometry.coordinates, coords);
if (!coords.length) {
throw new Error("Preview feature has no coordinates");
}
return {
minLon: Math.min(...coords.map((p) => p[0])),
minLat: Math.min(...coords.map((p) => p[1])),
maxLon: Math.max(...coords.map((p) => p[0])),
maxLat: Math.max(...coords.map((p) => p[1])),
};
}
function writeGeoJson(dir, name, content) {
const file = path.join(dir, name);
fs.writeFileSync(file, content);
const count = JSON.parse(content).features?.length ?? 0;
console.log(`${file}\tfeatures=${count}`);
}
function writeJson(file, value) {
fs.writeFileSync(file, JSON.stringify(value));
console.log(`${file}\tfeatures=${value.features.length}`);
}
function emptyCollection() {
return { type: "FeatureCollection", features: [] };
}
function splitLayers(dir, arrowScaleValue) {
const lanePolygons = JSON.parse(fs.readFileSync(path.join(dir, "lane_polygons.geojson"), "utf8"));
const markings = JSON.parse(fs.readFileSync(path.join(dir, "lane_markings.geojson"), "utf8"));
const intersections = JSON.parse(fs.readFileSync(path.join(dir, "intersection_markings.geojson"), "utf8"));
const out = {
roadSurface: emptyCollection(),
sidewalks: emptyCollection(),
laneSeparators: emptyCollection(),
centerLines: emptyCollection(),
vehicleStopLines: emptyCollection(),
laneArrows: emptyCollection(),
sidewalkCorners: intersections,
};
for (const feature of lanePolygons.features || []) {
const type = feature.properties?.type;
if (type === "Sidewalk" || type === "Footway") {
out.sidewalks.features.push(feature);
} else {
out.roadSurface.features.push(feature);
}
}
for (const feature of markings.features || []) {
const type = feature.properties?.type;
if (type === "lane separator") out.laneSeparators.features.push(feature);
if (type === "center line") out.centerLines.features.push(feature);
if (type === "vehicle stop line") out.vehicleStopLines.features.push(feature);
if (type === "lane arrow") out.laneArrows.features.push(scaleFeature(feature, arrowScaleValue));
}
return out;
}
function scaleFeature(feature, scale) {
const coords = [];
collectCoords(feature.geometry.coordinates, coords);
if (!coords.length) return feature;
const xs = coords.map((p) => p[0]);
const ys = coords.map((p) => p[1]);
const cx = (Math.min(...xs) + Math.max(...xs)) / 2;
const cy = (Math.min(...ys) + Math.max(...ys)) / 2;
return {
type: "Feature",
properties: { ...feature.properties, render_scale: scale },
geometry: {
...feature.geometry,
coordinates: scaleCoords(feature.geometry.coordinates, cx, cy, scale),
},
};
}
function collectCoords(obj, acc) {
if (Array.isArray(obj) && obj.length >= 2 && typeof obj[0] === "number" && typeof obj[1] === "number") {
acc.push([obj[0], obj[1]]);
} else if (Array.isArray(obj)) {
for (const item of obj) collectCoords(item, acc);
}
}
function scaleCoords(obj, cx, cy, scale) {
if (Array.isArray(obj) && obj.length >= 2 && typeof obj[0] === "number" && typeof obj[1] === "number") {
return [cx + (obj[0] - cx) * scale, cy + (obj[1] - cy) * scale, ...obj.slice(2)];
}
if (Array.isArray(obj)) {
return obj.map((item) => scaleCoords(item, cx, cy, scale));
}
return obj;
}
function importLayer(gpkg, source, layerName, update, env) {
const args = ["-f", "GPKG"];
if (update) args.push("-update", "-overwrite");
args.push(gpkg, source, "-nln", layerName);
execFileSync(ogr2ogr, args, { stdio: "inherit", env: { ...process.env, ...env } });
}
function makeQgisScript(options) {
return `
from pathlib import Path
from qgis.PyQt.QtCore import QSize
from qgis.PyQt.QtGui import QColor, QImage, QPainter
from qgis.core import (
QgsApplication,
QgsCoordinateReferenceSystem,
QgsFillSymbol,
QgsMapRendererCustomPainterJob,
QgsMapSettings,
QgsProject,
QgsRectangle,
QgsSingleSymbolRenderer,
QgsVectorLayer,
)
QGIS_PREFIX = ${JSON.stringify(options.qgisMacOS)}
GPKG = ${JSON.stringify(options.gpkgPath)}
PROJECT_PATH = ${JSON.stringify(options.projectPath)}
PREVIEW_PATH = ${JSON.stringify(options.previewPath)}
PREVIEW_EXTENT = [${options.previewExtent.split(",").map(Number).join(", ")}]
def fill_symbol(color, outline="0,0,0,0", outline_width="0"):
return QgsFillSymbol.createSimple({
"color": color,
"outline_color": outline,
"outline_width": outline_width,
"outline_width_unit": "MM",
"joinstyle": "round",
})
def make_layer(layer_name, title, color, outline="0,0,0,0", outline_width="0"):
layer = QgsVectorLayer(f"{GPKG}|layername={layer_name}", title, "ogr")
if not layer.isValid():
raise RuntimeError(f"Invalid layer: {title}")
layer.setRenderer(QgsSingleSymbolRenderer(fill_symbol(color, outline, outline_width)))
return layer
QgsApplication.setPrefixPath(QGIS_PREFIX, True)
app = QgsApplication([], False)
app.initQgis()
project = QgsProject.instance()
project.clear()
project.setFileName(PROJECT_PATH)
project.setCrs(QgsCoordinateReferenceSystem("EPSG:4326"))
project.setPresetHomePath(str(Path(PROJECT_PATH).parent))
layers = {
"road_surface": make_layer("road_surface", "osm2streets road surface", "43,43,40,255", "30,30,28,255", "0.04"),
"sidewalks": make_layer("sidewalks", "osm2streets sidewalks", "190,190,182,255", "156,156,148,255", "0.025"),
"sidewalk_corners": make_layer("sidewalk_corners", "osm2streets sidewalk corners", "190,190,182,255", "156,156,148,255", "0.025"),
"lane_separators": make_layer("lane_separators", "osm2streets lane separators", "238,238,230,255"),
"center_lines": make_layer("center_lines", "osm2streets center lines", "245,190,42,255"),
"vehicle_stop_lines": make_layer("vehicle_stop_lines", "osm2streets vehicle stop lines", "255,255,246,255"),
"lane_arrows": make_layer("lane_arrows_webscale", "osm2streets lane arrows", "255,255,246,255", "43,43,40,200", "0.015"),
}
draw_order = ["road_surface", "sidewalks", "sidewalk_corners", "lane_separators", "center_lines", "vehicle_stop_lines", "lane_arrows"]
for key in draw_order:
project.addMapLayer(layers[key], False)
root = project.layerTreeRoot()
for key in draw_order:
root.insertLayer(0, layers[key])
if not project.write(PROJECT_PATH):
raise RuntimeError(f"Failed to write {PROJECT_PATH}")
settings = QgsMapSettings()
settings.setLayers([layers[key] for key in reversed(draw_order)])
settings.setDestinationCrs(QgsCoordinateReferenceSystem("EPSG:4326"))
settings.setExtent(QgsRectangle(*PREVIEW_EXTENT))
settings.setOutputSize(QSize(1600, 1100))
settings.setBackgroundColor(QColor(245, 245, 240))
image = QImage(settings.outputSize(), QImage.Format_ARGB32_Premultiplied)
image.fill(settings.backgroundColor().rgba())
painter = QPainter(image)
job = QgsMapRendererCustomPainterJob(settings, painter)
job.start()
job.waitForFinished()
painter.end()
image.save(PREVIEW_PATH)
print(PROJECT_PATH)
print(PREVIEW_PATH)
app.exitQgis()
`;
}
function fixCanvas(projectFile, extentCsv) {
const fixScript = path.join(os.tmpdir(), `osm2streets_fix_canvas_${process.pid}.py`);
const script = `
from pathlib import Path
import shutil
import zipfile
import xml.etree.ElementTree as ET
project_path = Path(${JSON.stringify(projectFile)})
extent_values = [${extentCsv.split(",").map(Number).join(", ")}]
work_dir = Path(${JSON.stringify(path.join(os.tmpdir(), `osm2streets_qgz_fix_${process.pid}`))})
if work_dir.exists():
shutil.rmtree(work_dir)
work_dir.mkdir(parents=True)
with zipfile.ZipFile(project_path, "r") as zin:
zin.extractall(work_dir)
qgs_files = list(work_dir.glob("*.qgs"))
if not qgs_files:
raise RuntimeError("No .qgs file found inside project")
qgs_path = qgs_files[0]
tree = ET.parse(qgs_path)
root = tree.getroot()
old_canvas = root.find("mapcanvas")
if old_canvas is not None:
root.remove(old_canvas)
project_crs = root.find("projectCrs/spatialrefsys")
canvas = ET.Element("mapcanvas", {"name": "theMapCanvas", "annotationsVisible": "1"})
ET.SubElement(canvas, "units").text = "degrees"
extent = ET.SubElement(canvas, "extent")
for key, value in zip(["xmin", "ymin", "xmax", "ymax"], extent_values):
ET.SubElement(extent, key).text = str(value)
ET.SubElement(canvas, "rotation").text = "0"
dest = ET.SubElement(canvas, "destinationsrs")
if project_crs is not None:
dest.append(ET.fromstring(ET.tostring(project_crs, encoding="unicode")))
ET.SubElement(canvas, "rendermaptile").text = "0"
ET.SubElement(canvas, "expressionContextScope")
layer_tree = root.find("layer-tree-group")
insert_at = list(root).index(layer_tree) + 1 if layer_tree is not None else 1
root.insert(insert_at, canvas)
ET.indent(tree, space=" ")
tree.write(qgs_path, encoding="UTF-8", xml_declaration=True)
tmp = project_path.with_suffix(".qgz.tmp")
with zipfile.ZipFile(tmp, "w", compression=zipfile.ZIP_DEFLATED) as zout:
for f in sorted(work_dir.iterdir()):
zout.write(f, f.name)
tmp.replace(project_path)
shutil.rmtree(work_dir)
`;
fs.writeFileSync(fixScript, script);
execFileSync("python3", [fixScript], { stdio: "inherit" });
fs.unlinkSync(fixScript);
}