osmWorkflow/scripts/reimport-gpkg.js

#!/usr/bin/env node

"use strict";

// Reverse of the intermediates stage: pull manually-edited layers back out of
// <area>.gpkg into osm2streets_web_out/*.geojson and rebuild the merged scene.
//
// Use after hand-fixing geometry in QGIS. Re-running intermediates would
// regenerate the GeoPackage from OSM and throw those edits away.
//
// Every layer is exported to a staging directory and parsed before anything in
// outDir is touched: ogr2ogr exits non-zero on a missing layer but still leaves
// a zero-byte file behind, so a partial export must not reach the output tree.

const fs = require("fs");
const path = require("path");
const os = require("os");
const { execFileSync } = require("child_process");
const { qgisPaths } = require("./lib/tool-paths");
const { parseOsm } = require("../packages/road-compiler/src/osm");
const {
  validateTrafficSignalSourceReferences,
} = require("./lib/traffic-signals");
const {
  SCENE_LAYERS,
  AUXILIARY_EDIT_LAYERS,
  SCENE_FILE,
  SCENE_STYLE_FILE,
  layerFile,
  mergeScene,
  sceneStyle,
} = require("./lib/scene-layers");

const args = parseArgs(process.argv.slice(2));
const config = loadConfig(args);
const qgisApp = config.qgisApp || (process.platform === "darwin" ? "/Applications/QGIS.app" : "/usr");
const qgis = qgisPaths(qgisApp);
const ogr2ogr = qgis.ogr2ogr;
const ogrinfo = qgis.ogrinfo;
const outDir = path.resolve(requireText(config.outDir, "outDir"));
const gpkgPath = path.resolve(requireText(config.gpkg, "gpkg"));
const inputPath = path.resolve(requireText(config.input, "input"));
const trafficSignalAssembliesPath = path.resolve(
  config.trafficSignalAssemblies || path.join(outDir, "traffic_signal_assemblies.geojson"),
);

for (const exe of [ogr2ogr, ogrinfo]) {
  if (!fs.existsSync(exe)) {
    throw new Error(`QGIS executable not found: ${exe}`);
  }
}
if (!fs.existsSync(gpkgPath)) {
  throw new Error(`GeoPackage not found: ${gpkgPath}\nRun the intermediates stage first.`);
}
if (!fs.existsSync(inputPath)) {
  throw new Error(`Input OSM XML not found: ${inputPath}`);
}
if (!fs.existsSync(outDir)) {
  throw new Error(`GeoJSON output directory not found: ${outDir}`);
}

console.log(`Reimport: ${gpkgPath}`);
console.log(`Target:   ${outDir}`);

const present = gpkgLayers();
const trafficSignalControls = parseOsm(fs.readFileSync(inputPath, "utf8")).trafficSignalControls;
const missing = SCENE_LAYERS.filter((layer) => !present.has(layer.id)).map((layer) => layer.id);
if (missing.length) {
  throw new Error(
    `GeoPackage is missing ${missing.length} layer(s): ${missing.join(", ")}\n` +
    `Present: ${[...present].join(", ") || "(none)"}`,
  );
}

const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), "osm2streets-reimport-"));
try {
  const staged = SCENE_LAYERS.map((layer) => {
    const stagedPath = path.join(stagingDir, layerFile(layer));
    exportLayer(layer.id, stagedPath);
    const collection = readCollection(stagedPath, layer.id);
    if (layer.id === "intersection_surface") {
      restoreIntersectionSurfaceIds(collection);
      fs.writeFileSync(stagedPath, JSON.stringify(collection));
    }
    console.log(`${layer.id}\tfeatures=${collection.features.length}`);
    return { layer, stagedPath, collection };
  });
  const auxiliary = AUXILIARY_EDIT_LAYERS.map((layer) => {
    if (!present.has(layer.id)) throw new Error(`GeoPackage is missing auxiliary layer '${layer.id}'`);
    const stagedPath = path.join(stagingDir, layer.file);
    exportLayer(layer.id, stagedPath);
    const collection = readCollection(stagedPath, layer.id);
    const validated = validateTrafficSignalSourceReferences(collection, trafficSignalControls);
    console.log(`${layer.id}\tfeatures=${validated.features.length}`);
    return { layer, stagedPath, collection: validated };
  });

  for (const item of staged) {
    // Copy rather than rename: the staging dir may be on another filesystem.
    fs.copyFileSync(item.stagedPath, path.join(outDir, layerFile(item.layer)));
  }
  for (const item of auxiliary) {
    const destination = item.layer.id === "traffic_signal_assemblies"
      ? trafficSignalAssembliesPath : path.join(outDir, item.layer.file);
    fs.copyFileSync(item.stagedPath, destination);
  }

  const byId = new Map(staged.map((item) => [item.layer.id, item.collection]));
  const scene = mergeScene((layer) => byId.get(layer.id));
  fs.writeFileSync(path.join(outDir, SCENE_FILE), JSON.stringify(scene));
  fs.writeFileSync(
    path.join(outDir, SCENE_STYLE_FILE),
    JSON.stringify(sceneStyle(), null, 2),
  );
  console.log(`${path.join(outDir, SCENE_FILE)}\tfeatures=${scene.features.length}`);

  const empty = staged.filter((item) => item.collection.features.length === 0);
  if (empty.length) {
    console.warn(`Warning: empty layer(s): ${empty.map((item) => item.layer.id).join(", ")}`);
  }
} finally {
  fs.rmSync(stagingDir, { recursive: true, force: true });
}

function parseArgs(argv) {
  const out = {};
  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (!arg.startsWith("--")) continue;
    const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase());
    const next = argv[i + 1];
    if (!next || next.startsWith("--")) {
      out[key] = "true";
    } else {
      out[key] = next;
      i += 1;
    }
  }
  return out;
}

function loadConfig(cliArgs) {
  const base = {};
  if (cliArgs.config) {
    const file = path.resolve(cliArgs.config);
    if (!fs.existsSync(file)) {
      throw new Error(`Config file not found: ${file}`);
    }
    Object.assign(base, JSON.parse(fs.readFileSync(file, "utf8")));
  }
  for (const key of ["qgisApp", "input", "outDir", "gpkg", "trafficSignalAssemblies"]) {
    if (cliArgs[key] !== undefined) base[key] = cliArgs[key];
  }
  return base;
}

function requireText(value, key) {
  if (typeof value !== "string" || value.trim() === "") {
    throw new Error(`Missing config key: ${key}`);
  }
  return value;
}

function gdalEnv() {
  return qgis.env;
}

function gpkgLayers() {
  const output = execFileSync(ogrinfo, ["-q", gpkgPath], {
    encoding: "utf8",
    env: { ...process.env, ...gdalEnv() },
  });
  const names = new Set();
  for (const line of output.split("\n")) {
    const match = /^\s*\d+:\s+(\S+)/.exec(line);
    if (match) names.add(match[1]);
  }
  return names;
}

function exportLayer(layerName, destination) {
  // No COORDINATE_PRECISION here on purpose: the default already round-trips
  // full double precision, and setting it explicitly makes GDAL run its
  // precision-reduction pass, which drops vertices that collapse at the given
  // resolution (measured: 28 points lost across 7 lane-arrow polygons).
  execFileSync(ogr2ogr, [
    // GeoPackage reserves "id" as its FID column. Preserve it so the
    // intersection surface can retain its osm2streets internal ID on export.
    "-preserve_fid",
    "-f", "GeoJSON",
    destination,
    gpkgPath,
    layerName,
  ], {
    stdio: "inherit",
    env: { ...process.env, ...gdalEnv() },
  });
}

function restoreIntersectionSurfaceIds(collection) {
  for (const [index, feature] of collection.features.entries()) {
    if (!feature || typeof feature !== "object") {
      throw new Error(`intersection_surface feature ${index + 1} is invalid`);
    }
    const existingId = Number(feature.properties?.id);
    if (Number.isInteger(existingId)) continue;
    const fid = Number(feature.id);
    if (!Number.isInteger(fid)) {
      throw new Error(
        `intersection_surface feature ${index + 1} is missing its osm2streets internal ID`,
      );
    }
    feature.properties = { ...(feature.properties || {}), id: fid };
  }
}

function readCollection(file, layerName) {
  let parsed;
  try {
    parsed = JSON.parse(fs.readFileSync(file, "utf8"));
  } catch (error) {
    throw new Error(`Layer '${layerName}' did not export valid GeoJSON: ${error.message}`);
  }
  if (parsed.type !== "FeatureCollection" || !Array.isArray(parsed.features)) {
    throw new Error(`Layer '${layerName}' did not export a FeatureCollection`);
  }
  return parsed;
}