Merge pull request #816 from objectstack-ai/copilot/add-persistence-support

Author: hotlong

.gitignore

@@ -6,6 +6,9 @@ package-lock.json
 dist/
 *.tsbuildinfo
 
+# ObjectStack data directory (persistence)
+**/.objectstack/data/
+
 # IDE
 .vscode/
 .idea/

ROADMAP.md

@@ -395,6 +395,7 @@ business/custom objects, aligning with industry best practices (e.g., ServiceNow
 - [x] **ObjectQL Engine** — CRUD, hooks (before/after), middleware chain, action registry
 - [x] **Schema Registry** — FQN namespacing, multi-package contribution, priority resolution
 - [x] **In-Memory Driver** — Full CRUD, bulk ops, transactions, aggregation pipeline (Mingo), streaming
+- [x] **In-Memory Driver Persistence** — File-system (Node.js) and localStorage (Browser) persistence adapters with auto-save, custom adapter support
 - [x] **Metadata Service** — CRUD, query, bulk ops, overlay system, dependency tracking, import/export, file watching
 - [x] **Serializers** — JSON, YAML, TypeScript format support
 - [x] **Loaders** — Memory, Filesystem, Remote (HTTP) loaders

packages/plugins/driver-memory/src/index.ts

@@ -3,7 +3,10 @@
 import { InMemoryDriver } from './memory-driver.js';
 
 export { InMemoryDriver }; // Export class for direct usage
-export type { InMemoryDriverConfig } from './memory-driver.js';
+export type { InMemoryDriverConfig, PersistenceAdapterInterface } from './memory-driver.js';
+
+export { FileSystemPersistenceAdapter } from './persistence/file-adapter.js';
+export { LocalStoragePersistenceAdapter } from './persistence/local-storage-adapter.js';
 
 export { MemoryAnalyticsService } from './memory-analytics.js';
 export type { MemoryAnalyticsConfig } from './memory-analytics.js';

packages/plugins/driver-memory/src/memory-driver.ts

@@ -6,6 +6,20 @@ import { Logger, createLogger } from '@objectstack/core';
 import { Query, Aggregator } from 'mingo';
 import { getValueByPath } from './memory-matcher.js';
 
+/**
+ * Persistence adapter interface.
+ * Matches the PersistenceAdapterSchema contract from @objectstack/spec.
+ */
+export interface PersistenceAdapterInterface {
+  load(): Promise<Record<string, any[]> | null>;
+  save(db: Record<string, any[]>): Promise<void>;
+  flush(): Promise<void>;
+  /** Optional: Start periodic auto-save (used by FileSystemPersistenceAdapter). */
+  startAutoSave?(): void;
+  /** Optional: Stop auto-save timer and flush pending writes. */
+  stopAutoSave?(): Promise<void>;
+}
+
 /**
  * Configuration options for the InMemory driver.
  * Aligned with @objectstack/spec MemoryConfigSchema.
@@ -17,6 +31,21 @@ export interface InMemoryDriverConfig {
   strictMode?: boolean;
   /** Optional: Logger instance */
   logger?: Logger;
+  /**
+   * Optional persistence configuration.
+   * - `'file'` — File-system persistence with defaults (Node.js only)
+   * - `'local'` — localStorage persistence with defaults (Browser only)
+   * - `{ type: 'file', path?: string, autoSaveInterval?: number }` — File-system with options
+   * - `{ type: 'local', key?: string }` — localStorage with options
+   * - `{ adapter: PersistenceAdapterInterface }` — Custom adapter
+   */
+  persistence?: string | {
+    type?: 'file' | 'local';
+    path?: string;
+    key?: string;
+    autoSaveInterval?: number;
+    adapter?: PersistenceAdapterInterface;
+  };
 }
 
 /**
@@ -51,6 +80,7 @@ export class InMemoryDriver implements DriverInterface {
   private logger: Logger;
   private idCounters: Map<string, number> = new Map();
   private transactions: Map<string, MemoryTransaction> = new Map();
+  private persistenceAdapter: PersistenceAdapterInterface | null = null;
 
   constructor(config?: InMemoryDriverConfig) {
     this.config = config || {};
@@ -100,6 +130,37 @@ export class InMemoryDriver implements DriverInterface {
   // ===================================
 
   async connect() {
+    // Initialize persistence adapter if configured
+    await this.initPersistence();
+
+    // Load persisted data if available
+    if (this.persistenceAdapter) {
+      const persisted = await this.persistenceAdapter.load();
+      if (persisted) {
+        for (const [objectName, records] of Object.entries(persisted)) {
+          this.db[objectName] = records;
+          // Update ID counters based on persisted data
+          for (const record of records) {
+            if (record.id && typeof record.id === 'string') {
+              // ID format: {objectName}-{timestamp}-{counter}
+              const parts = record.id.split('-');
+              const lastPart = parts[parts.length - 1];
+              const counter = parseInt(lastPart, 10);
+              if (!isNaN(counter)) {
+                const current = this.idCounters.get(objectName) || 0;
+                if (counter > current) {
+                  this.idCounters.set(objectName, counter);
+                }
+              }
+            }
+          }
+        }
+        this.logger.info('InMemory Database restored from persistence', {
+          tables: Object.keys(persisted).length,
+        });
+      }
+    }
+
     // Load initial data if provided
     if (this.config.initialData) {
       for (const [objectName, records] of Object.entries(this.config.initialData)) {
@@ -115,9 +176,22 @@ export class InMemoryDriver implements DriverInterface {
     } else {
       this.logger.info('InMemory Database Connected (Virtual)');
     }
+
+    // Start auto-save if using file adapter
+    if (this.persistenceAdapter?.startAutoSave) {
+      this.persistenceAdapter.startAutoSave();
+    }
   }
 
   async disconnect() {
+    // Stop auto-save and flush pending writes
+    if (this.persistenceAdapter) {
+      if (this.persistenceAdapter.stopAutoSave) {
+        await this.persistenceAdapter.stopAutoSave();
+      }
+      await this.persistenceAdapter.flush();
+    }
+
     const tableCount = Object.keys(this.db).length;
     const recordCount = Object.values(this.db).reduce((sum, table) => sum + table.length, 0);
 
@@ -226,6 +300,7 @@ export class InMemoryDriver implements DriverInterface {
     };
 
     table.push(newRecord);
+    this.markDirty();
     this.logger.debug('Record created', { object, id: newRecord.id, tableSize: table.length });
     return { ...newRecord };
   }
@@ -253,6 +328,7 @@ export class InMemoryDriver implements DriverInterface {
     };
 
     table[index] = updatedRecord;
+    this.markDirty();
     this.logger.debug('Record updated', { object, id });
     return { ...updatedRecord };
   }
@@ -293,6 +369,7 @@ export class InMemoryDriver implements DriverInterface {
     }
 
     table.splice(index, 1);
+    this.markDirty();
     this.logger.debug('Record deleted', { object, id, tableSize: table.length });
     return true;
   }
@@ -350,6 +427,7 @@ export class InMemoryDriver implements DriverInterface {
           }
       }
 
+      if (count > 0) this.markDirty();
       this.logger.debug('UpdateMany completed', { object, count });
       return { count };
   }
@@ -377,6 +455,7 @@ export class InMemoryDriver implements DriverInterface {
       }
 
       const count = initialLength - this.db[object].length;
+      if (count > 0) this.markDirty();
       this.logger.debug('DeleteMany completed', { object, count });
       return { count };
   }
@@ -435,6 +514,7 @@ export class InMemoryDriver implements DriverInterface {
     // Restore the snapshot
     this.db = tx.snapshot;
     this.transactions.delete(txId);
+    this.markDirty();
     this.logger.debug('Transaction rolled back', { txId });
   }
 
@@ -448,6 +528,7 @@ export class InMemoryDriver implements DriverInterface {
   async clear() {
     this.db = {};
     this.idCounters.clear();
+    this.markDirty();
     this.logger.debug('All data cleared');
   }
 
@@ -818,4 +899,65 @@ export class InMemoryDriver implements DriverInterface {
     const timestamp = Date.now();
     return `${key}-${timestamp}-${counter}`;
   }
+
+  // ===================================
+  // Persistence
+  // ===================================
+
+  /**
+   * Mark the database as dirty, triggering persistence save.
+   */
+  private markDirty(): void {
+    if (this.persistenceAdapter) {
+      this.persistenceAdapter.save(this.db);
+    }
+  }
+
+  /**
+   * Flush pending persistence writes to ensure data is safely stored.
+   */
+  async flush(): Promise<void> {
+    if (this.persistenceAdapter) {
+      await this.persistenceAdapter.flush();
+    }
+  }
+
+  /**
+   * Initialize the persistence adapter based on configuration.
+   */
+  private async initPersistence(): Promise<void> {
+    const persistence = this.config.persistence;
+    if (!persistence) return;
+
+    if (typeof persistence === 'string') {
+      if (persistence === 'file') {
+        const { FileSystemPersistenceAdapter } = await import('./persistence/file-adapter.js');
+        this.persistenceAdapter = new FileSystemPersistenceAdapter();
+      } else if (persistence === 'local') {
+        const { LocalStoragePersistenceAdapter } = await import('./persistence/local-storage-adapter.js');
+        this.persistenceAdapter = new LocalStoragePersistenceAdapter();
+      } else {
+        throw new Error(`Unknown persistence type: "${persistence}". Use 'file' or 'local'.`);
+      }
+    } else if ('adapter' in persistence && persistence.adapter) {
+      this.persistenceAdapter = persistence.adapter;
+    } else if ('type' in persistence) {
+      if (persistence.type === 'file') {
+        const { FileSystemPersistenceAdapter } = await import('./persistence/file-adapter.js');
+        this.persistenceAdapter = new FileSystemPersistenceAdapter({
+          path: persistence.path,
+          autoSaveInterval: persistence.autoSaveInterval,
+        });
+      } else if (persistence.type === 'local') {
+        const { LocalStoragePersistenceAdapter } = await import('./persistence/local-storage-adapter.js');
+        this.persistenceAdapter = new LocalStoragePersistenceAdapter({
+          key: persistence.key,
+        });
+      }
+    }
+
+    if (this.persistenceAdapter) {
+      this.logger.debug('Persistence adapter initialized');
+    }
+  }
 }

packages/plugins/driver-memory/src/persistence/file-adapter.ts

@@ -0,0 +1,103 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+/**
+ * FileSystemPersistenceAdapter
+ *
+ * Persists the in-memory database to a JSON file on disk.
+ * Supports atomic writes (write to temp file then rename) and auto-save with dirty tracking.
+ *
+ * Node.js only — will throw if used in non-Node.js environments.
+ */
+export class FileSystemPersistenceAdapter {
+  private readonly filePath: string;
+  private readonly autoSaveInterval: number;
+  private dirty = false;
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private currentDb: Record<string, any[]> | null = null;
+
+  constructor(options?: { path?: string; autoSaveInterval?: number }) {
+    this.filePath = options?.path || path.join('.objectstack', 'data', 'memory-driver.json');
+    this.autoSaveInterval = options?.autoSaveInterval ?? 2000;
+  }
+
+  /**
+   * Load persisted data from disk.
+   * Returns null if no file exists.
+   */
+  async load(): Promise<Record<string, any[]> | null> {
+    try {
+      if (!fs.existsSync(this.filePath)) {
+        return null;
+      }
+      const raw = fs.readFileSync(this.filePath, 'utf-8');
+      const data = JSON.parse(raw);
+      return data as Record<string, any[]>;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Save data to disk using atomic write (temp file + rename).
+   */
+  async save(db: Record<string, any[]>): Promise<void> {
+    this.currentDb = db;
+    this.dirty = true;
+  }
+
+  /**
+   * Flush pending writes to disk immediately.
+   */
+  async flush(): Promise<void> {
+    if (!this.dirty || !this.currentDb) return;
+    await this.writeToDisk(this.currentDb);
+    this.dirty = false;
+  }
+
+  /**
+   * Start the auto-save timer.
+   */
+  startAutoSave(): void {
+    if (this.timer) return;
+    this.timer = setInterval(async () => {
+      if (this.dirty && this.currentDb) {
+        await this.writeToDisk(this.currentDb);
+        this.dirty = false;
+      }
+    }, this.autoSaveInterval);
+
+    // Allow process to exit even if timer is running
+    if (this.timer) {
+      this.timer.unref();
+    }
+  }
+
+  /**
+   * Stop the auto-save timer and flush pending writes.
+   */
+  async stopAutoSave(): Promise<void> {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    await this.flush();
+  }
+
+  /**
+   * Atomic write: write to temp file, then rename.
+   */
+  private async writeToDisk(db: Record<string, any[]>): Promise<void> {
+    const dir = path.dirname(this.filePath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    const tmpPath = this.filePath + '.tmp';
+    const json = JSON.stringify(db, null, 2);
+    fs.writeFileSync(tmpPath, json, 'utf-8');
+    fs.renameSync(tmpPath, this.filePath);
+  }
+}

packages/plugins/driver-memory/src/persistence/index.ts

@@ -0,0 +1,4 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+export { FileSystemPersistenceAdapter } from './file-adapter.js';
+export { LocalStoragePersistenceAdapter } from './local-storage-adapter.js';