From c0c8a16dd4012b9fdad086c9f6bd22a2c573ba3b Mon Sep 17 00:00:00 2001 From: que01 Date: Wed, 26 Aug 2026 17:54:38 +0800 Subject: [PATCH] feat: add immutable road revision storage --- .trellis/spec/backend/index.md | 5 + .../spec/backend/road-revision-storage.md | 68 ++++++ package.json | 2 +- src/compile/road-revisions.js | 216 ++++++++++++++++++ src/index.js | 1 + test/road-revisions.js | 77 +++++++ workbench/server.js | 5 + 7 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 .trellis/spec/backend/index.md create mode 100644 .trellis/spec/backend/road-revision-storage.md create mode 100644 src/compile/road-revisions.js create mode 100644 test/road-revisions.js diff --git a/.trellis/spec/backend/index.md b/.trellis/spec/backend/index.md new file mode 100644 index 0000000..2f8b289 --- /dev/null +++ b/.trellis/spec/backend/index.md @@ -0,0 +1,5 @@ +# Backend Development Guidelines + +## Revision Storage + +- [Road Revision Storage](./road-revision-storage.md) -- immutable workbench revision layout, validation, and recovery contract. diff --git a/.trellis/spec/backend/road-revision-storage.md b/.trellis/spec/backend/road-revision-storage.md new file mode 100644 index 0000000..c8361eb --- /dev/null +++ b/.trellis/spec/backend/road-revision-storage.md @@ -0,0 +1,68 @@ +# Road Revision Storage + +## 1. Scope / Trigger + +`src/compile/road-revisions.js` owns the persistent v2 workbench layout. It is called after a successful import and when an existing workbench area is read, so legacy `import-*` directories migrate lazily without an administrative command. + +## 2. Signatures + +```js +ensureRevisionStore(workspace) // -> { paths, active, baseline } +createCheckpoint(workspace, label) // -> restored revision +readRevision(workspace, revisionId) // -> { manifest, osm, nativeRoadOverrides, directEdits, trafficSignals } +``` + +## 3. Contracts + +- Legacy inputs remain at `source.osm`, `native-road-overrides.json`, and `native-traffic-signals.json`; migration must never rewrite them. +- OSM bytes are stored once under `osm/.osm`. +- `active/native-road-edits.json` starts as a validated v2 document at version `0`; `active/state.json` points to the active revision. +- A revision is immutable at `revisions/rev-NNNN/`. Its `manifest.json` uses `road-workbench-revision/v1`, references the content-addressed OSM, lists frozen JSON documents, and records SHA-256 digests. +- Area-config snapshots are intentionally owned by the subsequent snapshot step. Do not silently fall back to the external config when that step is introduced. + +## 4. Validation And Error Matrix + +| Condition | Error / behavior | +| --- | --- | +| Required legacy input missing | `Revision source is missing: ` | +| Content-addressed OSM differs from its filename digest | `Content-addressed OSM is corrupt` | +| Revision manifest schema or ID mismatch | `Invalid revision manifest` | +| Revision ID is not `rev-NNNN` | Reject with `Invalid revision id` | +| Manifest path leaves its OSM or revision directory | Reject the read; never resolve an external path | +| Referenced frozen OSM or JSON digest mismatch | Reject the read with a digest mismatch error | +| Empty checkpoint label | Reject with `label must be a non-empty string` | + +## 5. Good / Base / Bad Cases + +- Good: first open of a legacy directory adds `osm/`, `active/`, and `revisions/rev-0001/`, leaving old files byte-identical. +- Base: opening an already migrated directory is idempotent; an interrupted migration with a baseline but no active state repairs only the missing state file. +- Bad: mutating a frozen revision document or its referenced OSM must make `readRevision()` fail rather than returning altered input. + +## 6. Tests Required + +`test/road-revisions.js` must cover: + +- legacy-byte preservation and baseline creation; +- equal OSM bytes yielding one content-addressed copy; +- named checkpoint creation, parent linkage, and complete restore; +- manifest digest equality with the actual frozen files; +- invalid checkpoint labels. + +## 7. Wrong Vs Correct + +### Wrong + +```js +fs.copyFileSync(sourceOsm, path.join(revisionDirectory, 'source.osm')); +``` + +This duplicates equal OSM files and provides no integrity check. + +### Correct + +```js +const { file, digest } = storeOsm(paths); +manifest.source = { osmFile: path.relative(paths.workspace, file), osmSha256: digest }; +``` + +The content address is both the deduplication key and the integrity contract. diff --git a/package.json b/package.json index b008b54..90a40bc 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "road-compiler": "bin/road-compiler.js" }, "scripts": { - "test": "node test/index.js && node test/native-road-edits.js && node test/fixtures.js", + "test": "node test/index.js && node test/native-road-edits.js && node test/road-revisions.js && node test/fixtures.js", "road:workbench": "node bin/road-workbench.js", "road:export": "node bin/road-compiler.js", "build": "vite build --config workbench/client/vite.config.ts", diff --git a/src/compile/road-revisions.js b/src/compile/road-revisions.js new file mode 100644 index 0000000..e8f50b5 --- /dev/null +++ b/src/compile/road-revisions.js @@ -0,0 +1,216 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); +const { emptyEditDocument, validateEditDocument } = require('./native-road-edits'); + +const REVISION_SCHEMA = 'road-workbench-revision/v1'; +const ACTIVE_STATE_SCHEMA = 'road-workbench-active/v1'; +const PACKAGE_VERSION = require('../../package.json').version; +const GEOMETRY_SCHEMA = 'native-road-package/v1.1'; + +function sha256(bytes) { + return crypto.createHash('sha256').update(bytes).digest('hex'); +} + +function readFile(file) { + if (!fs.existsSync(file)) throw new Error(`Revision source is missing: ${file}`); + return fs.readFileSync(file); +} + +function writeFileAtomic(file, bytes) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const staging = `${file}.${process.pid}.tmp`; + fs.writeFileSync(staging, bytes); + fs.renameSync(staging, file); +} + +function writeJsonAtomic(file, value) { + writeFileAtomic(file, `${JSON.stringify(value, null, 2)}\n`); +} + +function pathsFor(workspace) { + return { + workspace, + sourceOsm: path.join(workspace, 'source.osm'), + overrides: path.join(workspace, 'native-road-overrides.json'), + trafficSignals: path.join(workspace, 'native-traffic-signals.json'), + osmDirectory: path.join(workspace, 'osm'), + activeDirectory: path.join(workspace, 'active'), + activeEdits: path.join(workspace, 'active', 'native-road-edits.json'), + activeState: path.join(workspace, 'active', 'state.json'), + revisionsDirectory: path.join(workspace, 'revisions'), + }; +} + +function storeOsm(paths) { + const bytes = readFile(paths.sourceOsm); + const digest = sha256(bytes); + const file = path.join(paths.osmDirectory, `${digest}.osm`); + if (fs.existsSync(file)) { + if (sha256(fs.readFileSync(file)) !== digest) throw new Error(`Content-addressed OSM is corrupt: ${file}`); + } else { + writeFileAtomic(file, bytes); + } + return { file, digest }; +} + +function revisionId(revisionsDirectory) { + const numbers = fs.existsSync(revisionsDirectory) + ? fs + .readdirSync(revisionsDirectory, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => /^rev-(\d{4})$/.exec(entry.name)) + .filter(Boolean) + .map((match) => Number(match[1])) + : []; + return `rev-${String(Math.max(0, ...numbers) + 1).padStart(4, '0')}`; +} + +function activeDocument(paths, osmSha256) { + if (fs.existsSync(paths.activeEdits)) return validateEditDocument(readJson(paths.activeEdits)); + const document = emptyEditDocument({ osmSha256, compilerGeometryVersion: GEOMETRY_SCHEMA }); + writeJsonAtomic(paths.activeEdits, document); + return document; +} + +function ensureRevisionStore(workspace) { + const paths = pathsFor(workspace); + const osm = storeOsm(paths); + const edits = activeDocument(paths, osm.digest); + if (fs.existsSync(path.join(paths.revisionsDirectory, 'rev-0001', 'manifest.json'))) { + const baseline = readRevision(workspace, 'rev-0001'); + const active = fs.existsSync(paths.activeState) + ? readJson(paths.activeState) + : { + schema: ACTIVE_STATE_SCHEMA, + activeRevisionId: baseline.manifest.id, + documentVersion: edits.documentVersion, + }; + if (!fs.existsSync(paths.activeState)) writeJsonAtomic(paths.activeState, active); + return { paths, active, baseline }; + } + const baseline = createRevision(paths, { + id: 'rev-0001', + directEdits: edits, + label: 'Import baseline', + }); + writeJsonAtomic(paths.activeState, { + schema: ACTIVE_STATE_SCHEMA, + activeRevisionId: baseline.manifest.id, + documentVersion: edits.documentVersion, + }); + return { paths, active: readJson(paths.activeState), baseline }; +} + +function createCheckpoint(workspace, label) { + if (typeof label !== 'string' || !label.trim()) + throw new Error('Revision checkpoint label must be a non-empty string.'); + const initialized = ensureRevisionStore(workspace); + const directEdits = validateEditDocument(readJson(initialized.paths.activeEdits)); + const revision = createRevision(initialized.paths, { + id: revisionId(initialized.paths.revisionsDirectory), + directEdits, + label: label.trim(), + parentRevisionId: initialized.active.activeRevisionId, + }); + writeJsonAtomic(initialized.paths.activeState, { + ...initialized.active, + activeRevisionId: revision.manifest.id, + documentVersion: directEdits.documentVersion, + }); + return revision; +} + +function createRevision(paths, { id, directEdits, label, parentRevisionId = undefined }) { + assertRevisionId(id); + const osm = storeOsm(paths); + const overrides = readFile(paths.overrides); + const trafficSignals = readFile(paths.trafficSignals); + const revisions = paths.revisionsDirectory; + fs.mkdirSync(revisions, { recursive: true }); + const destination = path.join(revisions, id); + if (fs.existsSync(destination)) throw new Error(`Revision already exists: ${id}`); + const staging = fs.mkdtempSync(path.join(revisions, '.staging-')); + try { + const files = { + nativeRoadOverrides: 'native-road-overrides.json', + directEdits: 'native-road-edits.json', + trafficSignals: 'native-traffic-signals.json', + }; + writeFileAtomic(path.join(staging, files.nativeRoadOverrides), overrides); + writeJsonAtomic(path.join(staging, files.directEdits), directEdits); + writeFileAtomic(path.join(staging, files.trafficSignals), trafficSignals); + const manifest = { + schema: REVISION_SCHEMA, + id, + createdAt: new Date().toISOString(), + ...(label ? { label } : {}), + ...(parentRevisionId ? { parentRevisionId } : {}), + source: { + osmFile: path.relative(paths.workspace, osm.file), + osmSha256: osm.digest, + }, + documents: files, + digests: { + nativeRoadOverrides: sha256(overrides), + directEdits: sha256(`${JSON.stringify(directEdits, null, 2)}\n`), + trafficSignals: sha256(trafficSignals), + }, + compiler: { packageVersion: PACKAGE_VERSION, geometrySchema: GEOMETRY_SCHEMA }, + }; + writeJsonAtomic(path.join(staging, 'manifest.json'), manifest); + fs.renameSync(staging, destination); + return readRevision(paths.workspace, id); + } catch (error) { + fs.rmSync(staging, { recursive: true, force: true }); + throw error; + } +} + +function readRevision(workspace, id) { + assertRevisionId(id); + const paths = pathsFor(workspace); + const directory = path.join(paths.revisionsDirectory, id); + const manifest = readJson(path.join(directory, 'manifest.json')); + if (manifest.schema !== REVISION_SCHEMA || manifest.id !== id) throw new Error(`Invalid revision manifest: ${id}`); + const osmFile = path.resolve(paths.workspace, manifest.source.osmFile); + if (!isWithin(paths.osmDirectory, osmFile)) throw new Error(`Revision ${id} OSM path escapes the workspace store.`); + const osm = readFile(osmFile); + if (sha256(osm) !== manifest.source.osmSha256) + throw new Error(`Revision ${id} OSM digest does not match its manifest.`); + const documents = Object.fromEntries( + Object.entries(manifest.documents).map(([name, file]) => { + const document = path.resolve(directory, file); + if (!isWithin(directory, document)) throw new Error(`Revision ${id} ${name} path escapes the revision.`); + const bytes = readFile(document); + if (sha256(bytes) !== manifest.digests[name]) + throw new Error(`Revision ${id} ${name} digest does not match its manifest.`); + return [name, JSON.parse(bytes.toString('utf8'))]; + }), + ); + return { manifest, osm: osm.toString('utf8'), ...documents }; +} + +function assertRevisionId(id) { + if (typeof id !== 'string' || !/^rev-\d{4}$/.test(id)) throw new Error(`Invalid revision id: ${id}`); +} + +function isWithin(directory, file) { + const resolvedDirectory = `${path.resolve(directory)}${path.sep}`; + return file.startsWith(resolvedDirectory); +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +module.exports = { + REVISION_SCHEMA, + ACTIVE_STATE_SCHEMA, + sha256, + ensureRevisionStore, + createCheckpoint, + readRevision, +}; diff --git a/src/index.js b/src/index.js index f5a0646..95074d8 100644 --- a/src/index.js +++ b/src/index.js @@ -10,6 +10,7 @@ module.exports = { nativeTrafficSignals: require('./native-traffic-signals'), nativeRoad: require('./compile/native-road'), nativeRoadEdits: require('./compile/native-road-edits'), + roadRevisions: require('./compile/road-revisions'), layerManifest: require('./compile/layer-manifest'), nativeRoadPackage: require('./export/native-road-package'), compiler: require('./compile/compiler'), diff --git a/test/road-revisions.js b/test/road-revisions.js new file mode 100644 index 0000000..4cb43f7 --- /dev/null +++ b/test/road-revisions.js @@ -0,0 +1,77 @@ +'use strict'; + +const assert = require('assert/strict'); +const crypto = require('crypto'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { nativeRoadEdits, roadRevisions } = require('../src'); + +const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'road-revisions-')); +const source = '\n'; +const overrides = { schema: 'native-road-overrides/v1', overrides: [] }; +const signals = { + schema: 'native-traffic-signals/v1', + provenance: 'empty', + assemblies: { type: 'FeatureCollection', features: [] }, +}; + +fs.writeFileSync(path.join(directory, 'source.osm'), source); +fs.writeFileSync(path.join(directory, 'native-road-overrides.json'), `${JSON.stringify(overrides, null, 2)}\n`); +fs.writeFileSync(path.join(directory, 'native-traffic-signals.json'), `${JSON.stringify(signals, null, 2)}\n`); +const legacyBytes = new Map( + ['source.osm', 'native-road-overrides.json', 'native-traffic-signals.json'].map((file) => [ + file, + fs.readFileSync(path.join(directory, file)), + ]), +); + +const first = roadRevisions.ensureRevisionStore(directory); +assert.equal(first.baseline.manifest.id, 'rev-0001'); +assert.equal(first.baseline.manifest.schema, roadRevisions.REVISION_SCHEMA); +assert.equal(first.active.activeRevisionId, 'rev-0001'); +assert.equal(first.baseline.directEdits.schema, nativeRoadEdits.EDITS_SCHEMA); +assert.equal(first.baseline.directEdits.documentVersion, 0); +for (const [file, bytes] of legacyBytes) + assert.deepEqual(fs.readFileSync(path.join(directory, file)), bytes, `${file} remains byte-for-byte unchanged`); + +const digest = crypto.createHash('sha256').update(source).digest('hex'); +assert.equal(first.baseline.manifest.source.osmSha256, digest); +assert.equal(fs.readdirSync(path.join(directory, 'osm')).filter((file) => file.endsWith('.osm')).length, 1); +roadRevisions.ensureRevisionStore(directory); +assert.equal( + fs.readdirSync(path.join(directory, 'osm')).filter((file) => file.endsWith('.osm')).length, + 1, + 'same OSM is stored once', +); + +const activeEdits = nativeRoadEdits.emptyEditDocument({ + osmSha256: digest, + compilerGeometryVersion: 'native-road-package/v1.1', +}); +activeEdits.operations = [{ id: 'op-1', createdAt: '2026-08-26T12:00:00.000Z', constraintIds: [] }]; +fs.writeFileSync(path.join(directory, 'active', 'native-road-edits.json'), `${JSON.stringify(activeEdits, null, 2)}\n`); +const checkpoint = roadRevisions.createCheckpoint(directory, 'Before junction changes'); +assert.equal(checkpoint.manifest.id, 'rev-0002'); +assert.equal(checkpoint.manifest.label, 'Before junction changes'); +assert.equal(checkpoint.manifest.parentRevisionId, 'rev-0001'); +assert.deepEqual(checkpoint.nativeRoadOverrides, overrides); +assert.deepEqual(checkpoint.trafficSignals, signals); +assert.deepEqual(checkpoint.directEdits, activeEdits); +assert.equal( + checkpoint.manifest.digests.nativeRoadOverrides, + crypto + .createHash('sha256') + .update(fs.readFileSync(path.join(directory, 'revisions', 'rev-0002', 'native-road-overrides.json'))) + .digest('hex'), +); +assert.deepEqual( + roadRevisions.readRevision(directory, 'rev-0002'), + checkpoint, + 'a checkpoint can be completely restored', +); +assert.throws(() => roadRevisions.createCheckpoint(directory, ''), /label must be a non-empty string/); +assert.throws(() => roadRevisions.readRevision(directory, '../rev-0002'), /Invalid revision id/); + +fs.rmSync(directory, { recursive: true, force: true }); +console.log('road revision storage tests passed'); diff --git a/workbench/server.js b/workbench/server.js index d8a143b..6b1ab18 100644 --- a/workbench/server.js +++ b/workbench/server.js @@ -9,6 +9,7 @@ const { generate, validateDocument, runtime } = require('../src/native-traffic-s const { convertGeoJson } = require('../src/reference/gaode'); const { exportNativeRoadPackage } = require('../src/export/native-road-package'); const { compileInput } = require('../src/compile/compiler'); +const { ensureRevisionStore } = require('../src/compile/road-revisions'); function startWorkbench({ area = null, @@ -167,6 +168,9 @@ function handle(request, response, session) { } function state(area, junctionReference = null, debug = false) { + // This is the read entry point for both a fresh import and a reopened legacy + // workspace. Initialization only adds the v2 layout; it never rewrites v1 files. + ensureRevisionStore(path.dirname(area.input)); const nativeDir = area.outputs.nativeRoadDir; const osm2streetsRoadSurface = area.outputs.geojsonDir ? path.join(area.outputs.geojsonDir, 'road_surface.geojson') @@ -325,6 +329,7 @@ function readUpload(request, session) { ); try { const compiled = compileInput(input); + ensureRevisionStore(root); session.area = compiled.area; return compiled; } catch (error) {