feat: add immutable road revision storage
This commit is contained in:
216
src/compile/road-revisions.js
Normal file
216
src/compile/road-revisions.js
Normal file
@@ -0,0 +1,216 @@
|
||||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { emptyEditDocument, validateEditDocument } = require('./native-road-edits');
|
||||
|
||||
const REVISION_SCHEMA = 'road-workbench-revision/v1';
|
||||
const ACTIVE_STATE_SCHEMA = 'road-workbench-active/v1';
|
||||
const PACKAGE_VERSION = require('../../package.json').version;
|
||||
const GEOMETRY_SCHEMA = 'native-road-package/v1.1';
|
||||
|
||||
function sha256(bytes) {
|
||||
return crypto.createHash('sha256').update(bytes).digest('hex');
|
||||
}
|
||||
|
||||
function readFile(file) {
|
||||
if (!fs.existsSync(file)) throw new Error(`Revision source is missing: ${file}`);
|
||||
return fs.readFileSync(file);
|
||||
}
|
||||
|
||||
function writeFileAtomic(file, bytes) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const staging = `${file}.${process.pid}.tmp`;
|
||||
fs.writeFileSync(staging, bytes);
|
||||
fs.renameSync(staging, file);
|
||||
}
|
||||
|
||||
function writeJsonAtomic(file, value) {
|
||||
writeFileAtomic(file, `${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function pathsFor(workspace) {
|
||||
return {
|
||||
workspace,
|
||||
sourceOsm: path.join(workspace, 'source.osm'),
|
||||
overrides: path.join(workspace, 'native-road-overrides.json'),
|
||||
trafficSignals: path.join(workspace, 'native-traffic-signals.json'),
|
||||
osmDirectory: path.join(workspace, 'osm'),
|
||||
activeDirectory: path.join(workspace, 'active'),
|
||||
activeEdits: path.join(workspace, 'active', 'native-road-edits.json'),
|
||||
activeState: path.join(workspace, 'active', 'state.json'),
|
||||
revisionsDirectory: path.join(workspace, 'revisions'),
|
||||
};
|
||||
}
|
||||
|
||||
function storeOsm(paths) {
|
||||
const bytes = readFile(paths.sourceOsm);
|
||||
const digest = sha256(bytes);
|
||||
const file = path.join(paths.osmDirectory, `${digest}.osm`);
|
||||
if (fs.existsSync(file)) {
|
||||
if (sha256(fs.readFileSync(file)) !== digest) throw new Error(`Content-addressed OSM is corrupt: ${file}`);
|
||||
} else {
|
||||
writeFileAtomic(file, bytes);
|
||||
}
|
||||
return { file, digest };
|
||||
}
|
||||
|
||||
function revisionId(revisionsDirectory) {
|
||||
const numbers = fs.existsSync(revisionsDirectory)
|
||||
? fs
|
||||
.readdirSync(revisionsDirectory, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => /^rev-(\d{4})$/.exec(entry.name))
|
||||
.filter(Boolean)
|
||||
.map((match) => Number(match[1]))
|
||||
: [];
|
||||
return `rev-${String(Math.max(0, ...numbers) + 1).padStart(4, '0')}`;
|
||||
}
|
||||
|
||||
function activeDocument(paths, osmSha256) {
|
||||
if (fs.existsSync(paths.activeEdits)) return validateEditDocument(readJson(paths.activeEdits));
|
||||
const document = emptyEditDocument({ osmSha256, compilerGeometryVersion: GEOMETRY_SCHEMA });
|
||||
writeJsonAtomic(paths.activeEdits, document);
|
||||
return document;
|
||||
}
|
||||
|
||||
function ensureRevisionStore(workspace) {
|
||||
const paths = pathsFor(workspace);
|
||||
const osm = storeOsm(paths);
|
||||
const edits = activeDocument(paths, osm.digest);
|
||||
if (fs.existsSync(path.join(paths.revisionsDirectory, 'rev-0001', 'manifest.json'))) {
|
||||
const baseline = readRevision(workspace, 'rev-0001');
|
||||
const active = fs.existsSync(paths.activeState)
|
||||
? readJson(paths.activeState)
|
||||
: {
|
||||
schema: ACTIVE_STATE_SCHEMA,
|
||||
activeRevisionId: baseline.manifest.id,
|
||||
documentVersion: edits.documentVersion,
|
||||
};
|
||||
if (!fs.existsSync(paths.activeState)) writeJsonAtomic(paths.activeState, active);
|
||||
return { paths, active, baseline };
|
||||
}
|
||||
const baseline = createRevision(paths, {
|
||||
id: 'rev-0001',
|
||||
directEdits: edits,
|
||||
label: 'Import baseline',
|
||||
});
|
||||
writeJsonAtomic(paths.activeState, {
|
||||
schema: ACTIVE_STATE_SCHEMA,
|
||||
activeRevisionId: baseline.manifest.id,
|
||||
documentVersion: edits.documentVersion,
|
||||
});
|
||||
return { paths, active: readJson(paths.activeState), baseline };
|
||||
}
|
||||
|
||||
function createCheckpoint(workspace, label) {
|
||||
if (typeof label !== 'string' || !label.trim())
|
||||
throw new Error('Revision checkpoint label must be a non-empty string.');
|
||||
const initialized = ensureRevisionStore(workspace);
|
||||
const directEdits = validateEditDocument(readJson(initialized.paths.activeEdits));
|
||||
const revision = createRevision(initialized.paths, {
|
||||
id: revisionId(initialized.paths.revisionsDirectory),
|
||||
directEdits,
|
||||
label: label.trim(),
|
||||
parentRevisionId: initialized.active.activeRevisionId,
|
||||
});
|
||||
writeJsonAtomic(initialized.paths.activeState, {
|
||||
...initialized.active,
|
||||
activeRevisionId: revision.manifest.id,
|
||||
documentVersion: directEdits.documentVersion,
|
||||
});
|
||||
return revision;
|
||||
}
|
||||
|
||||
function createRevision(paths, { id, directEdits, label, parentRevisionId = undefined }) {
|
||||
assertRevisionId(id);
|
||||
const osm = storeOsm(paths);
|
||||
const overrides = readFile(paths.overrides);
|
||||
const trafficSignals = readFile(paths.trafficSignals);
|
||||
const revisions = paths.revisionsDirectory;
|
||||
fs.mkdirSync(revisions, { recursive: true });
|
||||
const destination = path.join(revisions, id);
|
||||
if (fs.existsSync(destination)) throw new Error(`Revision already exists: ${id}`);
|
||||
const staging = fs.mkdtempSync(path.join(revisions, '.staging-'));
|
||||
try {
|
||||
const files = {
|
||||
nativeRoadOverrides: 'native-road-overrides.json',
|
||||
directEdits: 'native-road-edits.json',
|
||||
trafficSignals: 'native-traffic-signals.json',
|
||||
};
|
||||
writeFileAtomic(path.join(staging, files.nativeRoadOverrides), overrides);
|
||||
writeJsonAtomic(path.join(staging, files.directEdits), directEdits);
|
||||
writeFileAtomic(path.join(staging, files.trafficSignals), trafficSignals);
|
||||
const manifest = {
|
||||
schema: REVISION_SCHEMA,
|
||||
id,
|
||||
createdAt: new Date().toISOString(),
|
||||
...(label ? { label } : {}),
|
||||
...(parentRevisionId ? { parentRevisionId } : {}),
|
||||
source: {
|
||||
osmFile: path.relative(paths.workspace, osm.file),
|
||||
osmSha256: osm.digest,
|
||||
},
|
||||
documents: files,
|
||||
digests: {
|
||||
nativeRoadOverrides: sha256(overrides),
|
||||
directEdits: sha256(`${JSON.stringify(directEdits, null, 2)}\n`),
|
||||
trafficSignals: sha256(trafficSignals),
|
||||
},
|
||||
compiler: { packageVersion: PACKAGE_VERSION, geometrySchema: GEOMETRY_SCHEMA },
|
||||
};
|
||||
writeJsonAtomic(path.join(staging, 'manifest.json'), manifest);
|
||||
fs.renameSync(staging, destination);
|
||||
return readRevision(paths.workspace, id);
|
||||
} catch (error) {
|
||||
fs.rmSync(staging, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function readRevision(workspace, id) {
|
||||
assertRevisionId(id);
|
||||
const paths = pathsFor(workspace);
|
||||
const directory = path.join(paths.revisionsDirectory, id);
|
||||
const manifest = readJson(path.join(directory, 'manifest.json'));
|
||||
if (manifest.schema !== REVISION_SCHEMA || manifest.id !== id) throw new Error(`Invalid revision manifest: ${id}`);
|
||||
const osmFile = path.resolve(paths.workspace, manifest.source.osmFile);
|
||||
if (!isWithin(paths.osmDirectory, osmFile)) throw new Error(`Revision ${id} OSM path escapes the workspace store.`);
|
||||
const osm = readFile(osmFile);
|
||||
if (sha256(osm) !== manifest.source.osmSha256)
|
||||
throw new Error(`Revision ${id} OSM digest does not match its manifest.`);
|
||||
const documents = Object.fromEntries(
|
||||
Object.entries(manifest.documents).map(([name, file]) => {
|
||||
const document = path.resolve(directory, file);
|
||||
if (!isWithin(directory, document)) throw new Error(`Revision ${id} ${name} path escapes the revision.`);
|
||||
const bytes = readFile(document);
|
||||
if (sha256(bytes) !== manifest.digests[name])
|
||||
throw new Error(`Revision ${id} ${name} digest does not match its manifest.`);
|
||||
return [name, JSON.parse(bytes.toString('utf8'))];
|
||||
}),
|
||||
);
|
||||
return { manifest, osm: osm.toString('utf8'), ...documents };
|
||||
}
|
||||
|
||||
function assertRevisionId(id) {
|
||||
if (typeof id !== 'string' || !/^rev-\d{4}$/.test(id)) throw new Error(`Invalid revision id: ${id}`);
|
||||
}
|
||||
|
||||
function isWithin(directory, file) {
|
||||
const resolvedDirectory = `${path.resolve(directory)}${path.sep}`;
|
||||
return file.startsWith(resolvedDirectory);
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
REVISION_SCHEMA,
|
||||
ACTIVE_STATE_SCHEMA,
|
||||
sha256,
|
||||
ensureRevisionStore,
|
||||
createCheckpoint,
|
||||
readRevision,
|
||||
};
|
||||
@@ -10,6 +10,7 @@ module.exports = {
|
||||
nativeTrafficSignals: require('./native-traffic-signals'),
|
||||
nativeRoad: require('./compile/native-road'),
|
||||
nativeRoadEdits: require('./compile/native-road-edits'),
|
||||
roadRevisions: require('./compile/road-revisions'),
|
||||
layerManifest: require('./compile/layer-manifest'),
|
||||
nativeRoadPackage: require('./export/native-road-package'),
|
||||
compiler: require('./compile/compiler'),
|
||||
|
||||
Reference in New Issue
Block a user