From 2e6d1b3d7f29329477369ef0b2f47eae7f0e5e05 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 13 Feb 2026 13:58:17 +0000 Subject: [PATCH] rely on the yaci store flyaway migration for db setup, remove redundant migrations --- README.md | 35 +- api/src/db/connection.rs | 353 ---------------- api/src/db/mod.rs | 6 - api/src/db/queries.rs | 1 - api/src/main.rs | 7 - api/src/services/sync.rs | 24 ++ database/init/01-create-schema.sql | 5 - database/init/02-treasury-schema.sql | 388 ++++++++++++++++++ database/init/02-yaci-store-tables.sql | 292 ------------- .../migrations/V1__create_treasury_tables.sql | 106 ----- dev.sh | 20 +- indexer/Dockerfile.render | 20 - indexer/README.md | 2 +- indexer/SETUP.md | 5 +- indexer/add-missing-tables.sql | 31 -- indexer/application.properties | 7 +- indexer/application.render.properties | 91 ---- indexer/create-yaci-tables.sql | 92 ----- indexer/init-schema.sql | 292 ------------- indexer/render-entrypoint.sh | 73 ---- render.yaml | 64 --- 21 files changed, 442 insertions(+), 1472 deletions(-) delete mode 100644 api/src/db/connection.rs delete mode 100644 api/src/db/mod.rs delete mode 100644 api/src/db/queries.rs delete mode 100644 database/init/01-create-schema.sql create mode 100644 database/init/02-treasury-schema.sql delete mode 100644 database/init/02-yaci-store-tables.sql delete mode 100644 database/migrations/V1__create_treasury_tables.sql delete mode 100644 indexer/Dockerfile.render delete mode 100644 indexer/add-missing-tables.sql delete mode 100644 indexer/application.render.properties delete mode 100644 indexer/create-yaci-tables.sql delete mode 100644 indexer/init-schema.sql delete mode 100644 indexer/render-entrypoint.sh delete mode 100644 render.yaml diff --git a/README.md b/README.md index 9ad38d0..720c8a6 100644 --- a/README.md +++ b/README.md @@ -33,31 +33,34 @@ API instance hosted at ``` administration-data/ ├── .env # Environment variables (not committed) -├── indexer/ # YACI Store indexer configuration +├── .env.example # Environment variable template +├── indexer/ # YACI Store indexer configuration │ ├── application.properties │ ├── config/ │ │ └── application-plugins.yml # Plugin filter configuration │ ├── plugins/ │ │ └── scripts/ -│ │ └── treasury-filter.mvel # UTXO & metadata filter logic +│ │ └── treasury-filter.mvel # Metadata filter logic │ └── README.md -├── api/ # Rust API backend +├── api/ # Rust API backend │ ├── src/ │ │ ├── main.rs -│ │ ├── routes/v1/ # V1 API endpoints -│ │ ├── models/v1.rs # API models with OpenAPI -│ │ ├── openapi.rs # Swagger/OpenAPI config -│ │ └── db/ # Database utilities +│ │ ├── routes/v1/ # V1 API endpoints +│ │ ├── models/v1.rs # API models with OpenAPI +│ │ ├── openapi.rs # Swagger/OpenAPI config +│ │ └── services/ # Background sync & event processing │ ├── Cargo.toml -│ └── README.md # Full API documentation -├── docs/ # Documentation -│ └── architecture.md # Data flow diagrams +│ └── README.md # Full API documentation ├── database/ -│ └── schema/ # Schema definitions -├── scripts/ # Utility shell scripts -├── .github/ # CI/CD workflows +│ ├── init/ # Docker PostgreSQL init scripts (run on first start) +│ │ └── 02-treasury-schema.sql +│ └── schema/ +│ └── treasury.sql # Treasury schema (single source of truth) +├── docs/ # Documentation +│ └── architecture.md # Data flow diagrams +├── .github/ # CI/CD workflows ├── docker-compose.yml -└── dev.sh # Development helper script +└── dev.sh # Development helper script ``` ## Quick Start @@ -206,7 +209,7 @@ Network settings (host, port, protocol magic) are in `indexer/application.proper The system uses two schemas: -**yaci_store** - Raw blockchain data from YACI Store indexer: +**yaci_store** - Raw blockchain data, managed automatically by YACI Store via Flyway migrations: | Table | Description | Filtering | |-------|-------------|-----------| @@ -214,7 +217,7 @@ The system uses two schemas: | `yaci_store.address_utxo` | Treasury UTXOs | Only treasury stake credential | | `yaci_store.transaction_metadata` | TOM metadata | Only label 1694 | -**treasury** - Normalized application data: +**treasury** - Normalized application data (defined in [`database/schema/treasury.sql`](database/schema/treasury.sql)): | Table | Description | |-------|-------------| diff --git a/api/src/db/connection.rs b/api/src/db/connection.rs deleted file mode 100644 index ade28e8..0000000 --- a/api/src/db/connection.rs +++ /dev/null @@ -1,353 +0,0 @@ -// Database connection utilities - -use sqlx::PgPool; - -/// Initialize the administration schema if it doesn't exist -/// This ensures all required tables, indexes, and views are created -pub async fn init_administration_schema(pool: &PgPool) -> Result<(), sqlx::Error> { - tracing::info!("Initializing administration schema..."); - - // Create schema - sqlx::query("CREATE SCHEMA IF NOT EXISTS treasury") - .execute(pool) - .await?; - - // Create treasury_contracts table - sqlx::query(r#" - CREATE TABLE IF NOT EXISTS treasury.treasury_contracts ( - id SERIAL PRIMARY KEY, - contract_instance TEXT UNIQUE NOT NULL, - contract_address TEXT, - stake_credential TEXT, - name TEXT, - publish_tx_hash VARCHAR(64), - publish_time BIGINT, - initialized_tx_hash VARCHAR(64), - initialized_at BIGINT, - permissions JSONB, - status TEXT DEFAULT 'active', - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() - ) - "#).execute(pool).await?; - - // Create vendor_contracts table - sqlx::query(r#" - CREATE TABLE IF NOT EXISTS treasury.vendor_contracts ( - id SERIAL PRIMARY KEY, - treasury_id INT REFERENCES treasury.treasury_contracts(id), - project_id TEXT UNIQUE NOT NULL, - other_identifiers TEXT[], - project_name TEXT, - description TEXT, - vendor_name TEXT, - vendor_address TEXT, - contract_url TEXT, - contract_address TEXT, - fund_tx_hash VARCHAR(64) NOT NULL, - fund_slot BIGINT, - fund_block_time BIGINT, - initial_amount_lovelace BIGINT, - status TEXT DEFAULT 'active', - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() - ) - "#).execute(pool).await?; - - // Create milestones table - sqlx::query(r#" - CREATE TABLE IF NOT EXISTS treasury.milestones ( - id SERIAL PRIMARY KEY, - vendor_contract_id INT NOT NULL REFERENCES treasury.vendor_contracts(id) ON DELETE CASCADE, - milestone_id TEXT NOT NULL, - milestone_order INT NOT NULL, - label TEXT, - description TEXT, - acceptance_criteria TEXT, - amount_lovelace BIGINT, - status TEXT DEFAULT 'pending', - complete_tx_hash VARCHAR(64), - complete_time BIGINT, - complete_description TEXT, - evidence JSONB, - disburse_tx_hash VARCHAR(64), - disburse_time BIGINT, - disburse_amount BIGINT, - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(vendor_contract_id, milestone_id) - ) - "#).execute(pool).await?; - - // Create events table - sqlx::query(r#" - CREATE TABLE IF NOT EXISTS treasury.events ( - id SERIAL PRIMARY KEY, - tx_hash VARCHAR(64) UNIQUE NOT NULL, - slot BIGINT, - block_number BIGINT, - block_time BIGINT, - event_type TEXT NOT NULL, - treasury_id INT REFERENCES treasury.treasury_contracts(id), - vendor_contract_id INT REFERENCES treasury.vendor_contracts(id), - milestone_id INT REFERENCES treasury.milestones(id), - amount_lovelace BIGINT, - reason TEXT, - destination TEXT, - metadata JSONB, - created_at TIMESTAMPTZ DEFAULT NOW() - ) - "#).execute(pool).await?; - - // Create utxos table - sqlx::query(r#" - CREATE TABLE IF NOT EXISTS treasury.utxos ( - id SERIAL PRIMARY KEY, - tx_hash VARCHAR(64) NOT NULL, - output_index SMALLINT NOT NULL, - address TEXT, - address_type TEXT, - vendor_contract_id INT REFERENCES treasury.vendor_contracts(id), - lovelace_amount BIGINT, - slot BIGINT, - block_number BIGINT, - spent BOOLEAN DEFAULT FALSE, - spent_tx_hash VARCHAR(64), - spent_slot BIGINT, - UNIQUE(tx_hash, output_index) - ) - "#).execute(pool).await?; - - // Create sync_status table - sqlx::query(r#" - CREATE TABLE IF NOT EXISTS treasury.sync_status ( - id SERIAL PRIMARY KEY, - sync_type TEXT UNIQUE NOT NULL, - last_slot BIGINT DEFAULT 0, - last_block BIGINT, - last_tx_hash VARCHAR(64), - updated_at TIMESTAMPTZ DEFAULT NOW() - ) - "#).execute(pool).await?; - - // Insert initial sync status records - sqlx::query(r#" - INSERT INTO treasury.sync_status (sync_type, last_slot) - VALUES ('events', 0), ('utxos', 0) - ON CONFLICT (sync_type) DO NOTHING - "#).execute(pool).await?; - - // Create indexes - sqlx::query("CREATE INDEX IF NOT EXISTS idx_treasury_instance ON treasury.treasury_contracts(contract_instance)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_treasury_address ON treasury.treasury_contracts(contract_address)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_treasury_status ON treasury.treasury_contracts(status)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_vendor_treasury ON treasury.vendor_contracts(treasury_id)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_vendor_project_id ON treasury.vendor_contracts(project_id)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_vendor_status ON treasury.vendor_contracts(status)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_vendor_fund_time ON treasury.vendor_contracts(fund_block_time DESC)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_vendor_contract_address ON treasury.vendor_contracts(contract_address)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_milestone_vendor ON treasury.milestones(vendor_contract_id)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_milestone_status ON treasury.milestones(status)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_milestone_order ON treasury.milestones(vendor_contract_id, milestone_order)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_event_type ON treasury.events(event_type)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_event_vendor ON treasury.events(vendor_contract_id)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_event_treasury ON treasury.events(treasury_id)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_event_slot ON treasury.events(slot DESC)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_event_block_time ON treasury.events(block_time DESC)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_utxo_address ON treasury.utxos(address)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_utxo_vendor ON treasury.utxos(vendor_contract_id)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_utxo_unspent ON treasury.utxos(address) WHERE NOT spent").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_utxo_slot ON treasury.utxos(slot DESC)").execute(pool).await?; - - // Create additional indexes for new views - sqlx::query("CREATE INDEX IF NOT EXISTS idx_utxo_vendor_unspent ON treasury.utxos(vendor_contract_id) WHERE NOT spent").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_event_milestone ON treasury.events(milestone_id)").execute(pool).await?; - sqlx::query("CREATE INDEX IF NOT EXISTS idx_event_type_time ON treasury.events(event_type, block_time DESC)").execute(pool).await?; - - // Create views - v_vendor_contracts_summary with extended fields - sqlx::query(r#" - CREATE OR REPLACE VIEW treasury.v_vendor_contracts_summary AS - SELECT - vc.id, - vc.treasury_id, - vc.project_id, - vc.other_identifiers, - vc.project_name, - vc.description, - vc.vendor_name, - vc.vendor_address, - vc.contract_url, - vc.contract_address, - vc.fund_tx_hash, - vc.fund_slot, - vc.fund_block_time, - vc.initial_amount_lovelace, - vc.status, - vc.created_at, - vc.updated_at, - tc.contract_instance as treasury_instance, - tc.name as treasury_name, - COUNT(DISTINCT m.id) as total_milestones, - COUNT(DISTINCT m.id) FILTER (WHERE m.status = 'pending') as pending_milestones, - COUNT(DISTINCT m.id) FILTER (WHERE m.status = 'completed') as completed_milestones, - COUNT(DISTINCT m.id) FILTER (WHERE m.status = 'disbursed') as disbursed_milestones, - COALESCE(SUM(DISTINCT m.disburse_amount), 0)::BIGINT as total_disbursed_lovelace, - COALESCE(SUM(u.lovelace_amount) FILTER (WHERE NOT u.spent), 0)::BIGINT as current_balance_lovelace, - COUNT(u.id) FILTER (WHERE NOT u.spent) as utxo_count, - (SELECT MAX(e.block_time) FROM treasury.events e WHERE e.vendor_contract_id = vc.id) as last_event_time, - (SELECT COUNT(*) FROM treasury.events e WHERE e.vendor_contract_id = vc.id) as event_count - FROM treasury.vendor_contracts vc - LEFT JOIN treasury.treasury_contracts tc ON tc.id = vc.treasury_id - LEFT JOIN treasury.milestones m ON m.vendor_contract_id = vc.id - LEFT JOIN treasury.utxos u ON u.vendor_contract_id = vc.id - GROUP BY vc.id, tc.contract_instance, tc.name - "#).execute(pool).await?; - - sqlx::query(r#" - CREATE OR REPLACE VIEW treasury.v_milestone_timeline AS - SELECT - m.id, - m.milestone_id, - m.milestone_order, - m.label, - m.description, - m.acceptance_criteria, - m.amount_lovelace, - m.status, - m.complete_tx_hash, - m.complete_time, - m.complete_description, - m.evidence, - m.disburse_tx_hash, - m.disburse_time, - m.disburse_amount, - vc.project_id, - vc.project_name, - vc.vendor_address - FROM treasury.milestones m - JOIN treasury.vendor_contracts vc ON vc.id = m.vendor_contract_id - ORDER BY vc.project_id, m.milestone_order - "#).execute(pool).await?; - - sqlx::query(r#" - CREATE OR REPLACE VIEW treasury.v_recent_events AS - SELECT - e.id, - e.tx_hash, - e.slot, - e.block_number, - e.block_time, - e.event_type, - e.amount_lovelace, - e.reason, - e.destination, - e.metadata, - e.created_at, - tc.contract_instance as treasury_instance, - vc.project_id, - vc.project_name, - m.label as milestone_label, - m.milestone_order - FROM treasury.events e - LEFT JOIN treasury.treasury_contracts tc ON tc.id = e.treasury_id - LEFT JOIN treasury.vendor_contracts vc ON vc.id = e.vendor_contract_id - LEFT JOIN treasury.milestones m ON m.id = e.milestone_id - ORDER BY e.slot DESC - "#).execute(pool).await?; - - // v_treasury_summary with extended fields - sqlx::query(r#" - CREATE OR REPLACE VIEW treasury.v_treasury_summary AS - SELECT - tc.id as treasury_id, - tc.contract_instance, - tc.contract_address, - tc.stake_credential, - tc.name, - tc.status, - tc.publish_tx_hash, - tc.publish_time, - tc.initialized_tx_hash, - tc.initialized_at, - tc.permissions, - COUNT(DISTINCT vc.id) as vendor_contract_count, - COUNT(DISTINCT vc.id) FILTER (WHERE vc.status = 'active') as active_contracts, - COUNT(DISTINCT vc.id) FILTER (WHERE vc.status = 'completed') as completed_contracts, - COUNT(DISTINCT vc.id) FILTER (WHERE vc.status = 'cancelled') as cancelled_contracts, - COALESCE(SUM(u.lovelace_amount) FILTER (WHERE NOT u.spent AND u.address = tc.contract_address), 0)::BIGINT as treasury_balance, - COUNT(u.id) FILTER (WHERE NOT u.spent AND u.address = tc.contract_address) as utxo_count, - (SELECT COUNT(*) FROM treasury.events WHERE treasury_id = tc.id) as total_events, - (SELECT MAX(block_time) FROM treasury.events WHERE treasury_id = tc.id) as last_event_time, - tc.created_at, - tc.updated_at - FROM treasury.treasury_contracts tc - LEFT JOIN treasury.vendor_contracts vc ON vc.treasury_id = tc.id - LEFT JOIN treasury.utxos u ON u.address = tc.contract_address - GROUP BY tc.id - "#).execute(pool).await?; - - // v_events_with_context - events with full context - sqlx::query(r#" - CREATE OR REPLACE VIEW treasury.v_events_with_context AS - SELECT - e.id, - e.tx_hash, - e.slot, - e.block_number, - e.block_time, - e.event_type, - e.amount_lovelace, - e.reason, - e.destination, - e.metadata, - e.created_at, - tc.contract_instance as treasury_instance, - tc.name as treasury_name, - vc.project_id, - vc.project_name, - vc.vendor_name, - vc.contract_address as project_address, - m.milestone_id, - m.label as milestone_label, - m.milestone_order - FROM treasury.events e - LEFT JOIN treasury.treasury_contracts tc ON tc.id = e.treasury_id - LEFT JOIN treasury.vendor_contracts vc ON vc.id = e.vendor_contract_id - LEFT JOIN treasury.milestones m ON m.id = e.milestone_id - "#).execute(pool).await?; - - // v_financial_summary - allocated vs disbursed vs remaining - sqlx::query(r#" - CREATE OR REPLACE VIEW treasury.v_financial_summary AS - SELECT - tc.id as treasury_id, - tc.contract_instance, - tc.name as treasury_name, - COALESCE(SUM(vc.initial_amount_lovelace), 0)::BIGINT as total_allocated_lovelace, - COALESCE(SUM(m_totals.total_disbursed), 0)::BIGINT as total_disbursed_lovelace, - (COALESCE(SUM(vc.initial_amount_lovelace), 0) - COALESCE(SUM(m_totals.total_disbursed), 0))::BIGINT as total_remaining_lovelace, - COALESCE(SUM(u.lovelace_amount) FILTER (WHERE NOT u.spent AND u.address = tc.contract_address), 0)::BIGINT as treasury_balance_lovelace, - COALESCE(( - SELECT SUM(u2.lovelace_amount) - FROM treasury.utxos u2 - JOIN treasury.vendor_contracts vc2 ON vc2.id = u2.vendor_contract_id - WHERE vc2.treasury_id = tc.id AND NOT u2.spent - ), 0)::BIGINT as project_balance_lovelace, - COUNT(DISTINCT vc.id) as project_count, - COUNT(DISTINCT CASE WHEN vc.status = 'active' THEN vc.id END) as active_project_count - FROM treasury.treasury_contracts tc - LEFT JOIN treasury.vendor_contracts vc ON vc.treasury_id = tc.id - LEFT JOIN ( - SELECT - m.vendor_contract_id, - SUM(COALESCE(m.disburse_amount, 0)) as total_disbursed - FROM treasury.milestones m - GROUP BY m.vendor_contract_id - ) m_totals ON m_totals.vendor_contract_id = vc.id - LEFT JOIN treasury.utxos u ON u.address = tc.contract_address - GROUP BY tc.id - "#).execute(pool).await?; - - tracing::info!("Administration schema initialized successfully"); - Ok(()) -} diff --git a/api/src/db/mod.rs b/api/src/db/mod.rs deleted file mode 100644 index b183bac..0000000 --- a/api/src/db/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -// Database connection and query utilities - -pub mod connection; -pub mod queries; - -pub use connection::init_administration_schema; diff --git a/api/src/db/queries.rs b/api/src/db/queries.rs deleted file mode 100644 index 008122d..0000000 --- a/api/src/db/queries.rs +++ /dev/null @@ -1 +0,0 @@ -// Database query functions diff --git a/api/src/main.rs b/api/src/main.rs index 017b1f6..e4cc0d5 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -10,7 +10,6 @@ use tower_http::cors::{Any, CorsLayer}; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; -mod db; mod models; mod openapi; mod routes; @@ -64,12 +63,6 @@ async fn main() -> anyhow::Result<()> { let pool = pool.expect("Pool should be initialized after retries"); tracing::info!("Database connection established"); - // Initialize administration schema (creates tables if they don't exist) - if let Err(e) = db::init_administration_schema(&pool).await { - tracing::error!("Failed to initialize administration schema: {}", e); - return Err(e.into()); - } - // Spawn background sync task let sync_pool = pool.clone(); tokio::spawn(async move { diff --git a/api/src/services/sync.rs b/api/src/services/sync.rs index e5a498f..95a6d00 100644 --- a/api/src/services/sync.rs +++ b/api/src/services/sync.rs @@ -13,6 +13,30 @@ use super::event_processor::EventProcessor; pub async fn run_sync_loop(pool: PgPool) { let processor = EventProcessor::new(pool.clone()); + // Wait for yaci_store tables to be created by the indexer's Flyway migrations + tracing::info!("Waiting for yaci_store tables to be available..."); + for attempt in 1..=30 { + let table_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'yaci_store' AND table_name = 'transaction_metadata')" + ) + .fetch_one(&pool) + .await + .unwrap_or(false); + + if table_exists { + tracing::info!("yaci_store tables are ready"); + break; + } + + if attempt == 30 { + tracing::warn!("yaci_store tables not found after 30 attempts, proceeding anyway"); + break; + } + + tracing::info!("yaci_store tables not ready yet, retrying in 5s... (attempt {}/30)", attempt); + tokio::time::sleep(Duration::from_secs(5)).await; + } + // Initial sync: process all events from beginning tracing::info!("Starting initial TOM event sync..."); if let Err(e) = processor.sync_all_events().await { diff --git a/database/init/01-create-schema.sql b/database/init/01-create-schema.sql deleted file mode 100644 index 1174b19..0000000 --- a/database/init/01-create-schema.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Create yaci_store schema for YACI Store -CREATE SCHEMA IF NOT EXISTS yaci_store; - --- Grant permissions -GRANT ALL PRIVILEGES ON SCHEMA yaci_store TO postgres; diff --git a/database/init/02-treasury-schema.sql b/database/init/02-treasury-schema.sql new file mode 100644 index 0000000..56b6e4e --- /dev/null +++ b/database/init/02-treasury-schema.sql @@ -0,0 +1,388 @@ +-- Administration Data - Normalized Schema +-- Processes TOM (Treasury Oversight Metadata) events from YACI Store + +-- Create treasury schema +CREATE SCHEMA IF NOT EXISTS treasury; + +-- ============================================================================ +-- TABLES +-- ============================================================================ + +-- Treasury Contracts (TRSC) - Root treasury reserve contracts +CREATE TABLE IF NOT EXISTS treasury.treasury_contracts ( + id SERIAL PRIMARY KEY, + contract_instance TEXT UNIQUE NOT NULL, -- Policy ID (on-chain instance identifier) + contract_address TEXT, -- Script address (addr1x...) + stake_credential TEXT, -- Shared stake credential + name TEXT, -- Human-readable name/label + publish_tx_hash VARCHAR(64), -- First publish event + publish_time BIGINT, -- Block time of publish + initialized_tx_hash VARCHAR(64), -- First initialize event + initialized_at BIGINT, -- Block time of init + permissions JSONB, -- Permission rules from publish metadata + status TEXT DEFAULT 'active', -- active/paused + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +-- Vendor Contracts (PSSC) - Project-specific contracts linked to treasury +CREATE TABLE IF NOT EXISTS treasury.vendor_contracts ( + id SERIAL PRIMARY KEY, + treasury_id INT REFERENCES treasury.treasury_contracts(id), + project_id TEXT UNIQUE NOT NULL, -- Logical identifier (e.g., "EC-0008-25") + other_identifiers TEXT[], -- Related IDs from otherIdentifiers array + project_name TEXT, -- Label from fund event + description TEXT, -- Project description (joined if array) + vendor_name TEXT, -- vendor.name from metadata + vendor_address TEXT, -- Payment destination (vendor.label in metadata) + contract_url TEXT, -- contract - link to agreement document + contract_address TEXT, -- PSSC script address (from fund tx output) + fund_tx_hash VARCHAR(64) NOT NULL, -- Fund transaction + fund_slot BIGINT, -- Blockchain slot + fund_block_time BIGINT, -- Block timestamp + initial_amount_lovelace BIGINT, -- Initial funding amount (from tx output) + status TEXT DEFAULT 'active', -- active/paused/completed/cancelled + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +-- Milestones - Each vendor contract has ordered milestones +CREATE TABLE IF NOT EXISTS treasury.milestones ( + id SERIAL PRIMARY KEY, + vendor_contract_id INT NOT NULL REFERENCES treasury.vendor_contracts(id) ON DELETE CASCADE, + milestone_id TEXT NOT NULL, -- Logical identifier (e.g., "m-0") + milestone_order INT NOT NULL, -- Position (1, 2, 3...) + label TEXT, -- Milestone name + description TEXT, -- Detailed description + acceptance_criteria TEXT, -- Completion criteria + amount_lovelace BIGINT, -- Allocated amount (if specified) + status TEXT DEFAULT 'pending', -- pending/completed/disbursed + complete_tx_hash VARCHAR(64), -- Completion transaction + complete_time BIGINT, -- Completion timestamp + complete_description TEXT, -- Description from complete event + evidence JSONB, -- Evidence array from complete event + disburse_tx_hash VARCHAR(64), -- Disbursement transaction + disburse_time BIGINT, -- Disbursement timestamp + disburse_amount BIGINT, -- Actual disbursed amount + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(vendor_contract_id, milestone_id) +); + +-- Events - Audit log of all TOM events +CREATE TABLE IF NOT EXISTS treasury.events ( + id SERIAL PRIMARY KEY, + tx_hash VARCHAR(64) UNIQUE NOT NULL, -- Transaction hash + slot BIGINT, -- Blockchain slot + block_number BIGINT, -- Block number + block_time BIGINT, -- Block timestamp + event_type TEXT NOT NULL, -- publish/initialize/fund/complete/disburse/etc. + treasury_id INT REFERENCES treasury.treasury_contracts(id), + vendor_contract_id INT REFERENCES treasury.vendor_contracts(id), + milestone_id INT REFERENCES treasury.milestones(id), + amount_lovelace BIGINT, -- Amount involved + reason TEXT, -- Justification (pause/cancel/modify) + destination TEXT, -- Destination label (disburse) + metadata JSONB, -- Original TOM metadata body + created_at TIMESTAMPTZ DEFAULT NOW() +); + +-- UTXOs - Track UTXOs at treasury-related addresses +CREATE TABLE IF NOT EXISTS treasury.utxos ( + id SERIAL PRIMARY KEY, + tx_hash VARCHAR(64) NOT NULL, -- Transaction hash + output_index SMALLINT NOT NULL, -- Output index + address TEXT, -- Owner address (optional for tracking) + address_type TEXT, -- treasury/vendor_contract/vendor + vendor_contract_id INT REFERENCES treasury.vendor_contracts(id), + lovelace_amount BIGINT, -- Amount (optional for tracking) + slot BIGINT, -- Creation slot (optional for tracking) + block_number BIGINT, -- Block number + spent BOOLEAN DEFAULT FALSE, -- Is spent? + spent_tx_hash VARCHAR(64), -- Spending transaction + spent_slot BIGINT, -- When spent + UNIQUE(tx_hash, output_index) +); + +-- Sync Status - Track synchronization progress +CREATE TABLE IF NOT EXISTS treasury.sync_status ( + id SERIAL PRIMARY KEY, + sync_type TEXT UNIQUE NOT NULL, -- events/utxos + last_slot BIGINT DEFAULT 0, -- Last processed slot + last_block BIGINT, -- Last processed block + last_tx_hash VARCHAR(64), -- Last processed tx + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +-- Insert initial sync status records +INSERT INTO treasury.sync_status (sync_type, last_slot) VALUES ('events', 0), ('utxos', 0) +ON CONFLICT (sync_type) DO NOTHING; + +-- ============================================================================ +-- INDEXES +-- ============================================================================ + +-- Treasury contracts +CREATE INDEX IF NOT EXISTS idx_treasury_instance ON treasury.treasury_contracts(contract_instance); +CREATE INDEX IF NOT EXISTS idx_treasury_address ON treasury.treasury_contracts(contract_address); +CREATE INDEX IF NOT EXISTS idx_treasury_status ON treasury.treasury_contracts(status); + +-- Vendor contracts (projects) +CREATE INDEX IF NOT EXISTS idx_vendor_treasury ON treasury.vendor_contracts(treasury_id); +CREATE INDEX IF NOT EXISTS idx_vendor_project_id ON treasury.vendor_contracts(project_id); +CREATE INDEX IF NOT EXISTS idx_vendor_status ON treasury.vendor_contracts(status); +CREATE INDEX IF NOT EXISTS idx_vendor_fund_time ON treasury.vendor_contracts(fund_block_time DESC); +CREATE INDEX IF NOT EXISTS idx_vendor_contract_address ON treasury.vendor_contracts(contract_address); +CREATE INDEX IF NOT EXISTS idx_vendor_search ON treasury.vendor_contracts + USING gin (to_tsvector('english', COALESCE(project_name, '') || ' ' || COALESCE(description, ''))); + +-- Milestones +CREATE INDEX IF NOT EXISTS idx_milestone_vendor ON treasury.milestones(vendor_contract_id); +CREATE INDEX IF NOT EXISTS idx_milestone_status ON treasury.milestones(status); +CREATE INDEX IF NOT EXISTS idx_milestone_order ON treasury.milestones(vendor_contract_id, milestone_order); + +-- Events +CREATE INDEX IF NOT EXISTS idx_event_type ON treasury.events(event_type); +CREATE INDEX IF NOT EXISTS idx_event_vendor ON treasury.events(vendor_contract_id); +CREATE INDEX IF NOT EXISTS idx_event_treasury ON treasury.events(treasury_id); +CREATE INDEX IF NOT EXISTS idx_event_slot ON treasury.events(slot DESC); +CREATE INDEX IF NOT EXISTS idx_event_block_time ON treasury.events(block_time DESC); + +-- UTXOs +CREATE INDEX IF NOT EXISTS idx_utxo_address ON treasury.utxos(address); +CREATE INDEX IF NOT EXISTS idx_utxo_vendor ON treasury.utxos(vendor_contract_id); +CREATE INDEX IF NOT EXISTS idx_utxo_unspent ON treasury.utxos(address) WHERE NOT spent; +CREATE INDEX IF NOT EXISTS idx_utxo_slot ON treasury.utxos(slot DESC); +CREATE INDEX IF NOT EXISTS idx_utxo_vendor_unspent ON treasury.utxos(vendor_contract_id) WHERE NOT spent; + +-- Full-text search across project fields +CREATE INDEX IF NOT EXISTS idx_vendor_fulltext ON treasury.vendor_contracts + USING gin (to_tsvector('english', + COALESCE(project_id, '') || ' ' || + COALESCE(project_name, '') || ' ' || + COALESCE(description, '') || ' ' || + COALESCE(vendor_name, '') + )); + +-- Events by milestone (for milestone event history) +CREATE INDEX IF NOT EXISTS idx_event_milestone ON treasury.events(milestone_id); + +-- Events by type and time (for activity feed filtering) +CREATE INDEX IF NOT EXISTS idx_event_type_time ON treasury.events(event_type, block_time DESC); + +-- ============================================================================ +-- TRIGGER FOR updated_at +-- ============================================================================ + +CREATE OR REPLACE FUNCTION treasury.update_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_treasury_contracts_updated_at + BEFORE UPDATE ON treasury.treasury_contracts + FOR EACH ROW EXECUTE FUNCTION treasury.update_updated_at(); + +CREATE TRIGGER trg_vendor_contracts_updated_at + BEFORE UPDATE ON treasury.vendor_contracts + FOR EACH ROW EXECUTE FUNCTION treasury.update_updated_at(); + +CREATE TRIGGER trg_milestones_updated_at + BEFORE UPDATE ON treasury.milestones + FOR EACH ROW EXECUTE FUNCTION treasury.update_updated_at(); + +-- ============================================================================ +-- VIEWS +-- ============================================================================ + +-- Vendor contracts with milestone stats, financials, and balance +CREATE OR REPLACE VIEW treasury.v_vendor_contracts_summary AS +SELECT + vc.id, + vc.treasury_id, + vc.project_id, + vc.other_identifiers, + vc.project_name, + vc.description, + vc.vendor_name, + vc.vendor_address, + vc.contract_url, + vc.contract_address, + vc.fund_tx_hash, + vc.fund_slot, + vc.fund_block_time, + vc.initial_amount_lovelace, + vc.status, + vc.created_at, + vc.updated_at, + -- Treasury context + tc.contract_instance as treasury_instance, + tc.name as treasury_name, + -- Milestone counts + COUNT(DISTINCT m.id) as total_milestones, + COUNT(DISTINCT m.id) FILTER (WHERE m.status = 'pending') as pending_milestones, + COUNT(DISTINCT m.id) FILTER (WHERE m.status = 'completed') as completed_milestones, + COUNT(DISTINCT m.id) FILTER (WHERE m.status = 'disbursed') as disbursed_milestones, + -- Financial totals from milestones + COALESCE(SUM(DISTINCT m.disburse_amount), 0)::BIGINT as total_disbursed_lovelace, + -- Current balance from UTXOs + COALESCE(SUM(u.lovelace_amount) FILTER (WHERE NOT u.spent), 0)::BIGINT as current_balance_lovelace, + COUNT(u.id) FILTER (WHERE NOT u.spent) as utxo_count, + -- Last event time + (SELECT MAX(e.block_time) FROM treasury.events e WHERE e.vendor_contract_id = vc.id) as last_event_time, + -- Event count + (SELECT COUNT(*) FROM treasury.events e WHERE e.vendor_contract_id = vc.id) as event_count +FROM treasury.vendor_contracts vc +LEFT JOIN treasury.treasury_contracts tc ON tc.id = vc.treasury_id +LEFT JOIN treasury.milestones m ON m.vendor_contract_id = vc.id +LEFT JOIN treasury.utxos u ON u.vendor_contract_id = vc.id +GROUP BY vc.id, tc.contract_instance, tc.name; + +-- Milestone timeline with vendor context +CREATE OR REPLACE VIEW treasury.v_milestone_timeline AS +SELECT + m.id, + m.milestone_id, + m.milestone_order, + m.label, + m.description, + m.acceptance_criteria, + m.amount_lovelace, + m.status, + m.complete_tx_hash, + m.complete_time, + m.complete_description, + m.evidence, + m.disburse_tx_hash, + m.disburse_time, + m.disburse_amount, + vc.project_id, + vc.project_name, + vc.vendor_address +FROM treasury.milestones m +JOIN treasury.vendor_contracts vc ON vc.id = m.vendor_contract_id +ORDER BY vc.project_id, m.milestone_order; + +-- Recent events with full context +CREATE OR REPLACE VIEW treasury.v_recent_events AS +SELECT + e.id, + e.tx_hash, + e.slot, + e.block_number, + e.block_time, + e.event_type, + e.amount_lovelace, + e.reason, + e.destination, + e.metadata, + e.created_at, + tc.contract_instance as treasury_instance, + vc.project_id, + vc.project_name, + m.label as milestone_label, + m.milestone_order +FROM treasury.events e +LEFT JOIN treasury.treasury_contracts tc ON tc.id = e.treasury_id +LEFT JOIN treasury.vendor_contracts vc ON vc.id = e.vendor_contract_id +LEFT JOIN treasury.milestones m ON m.id = e.milestone_id +ORDER BY e.slot DESC; + +-- Treasury summary stats +CREATE OR REPLACE VIEW treasury.v_treasury_summary AS +SELECT + tc.id as treasury_id, + tc.contract_instance, + tc.contract_address, + tc.stake_credential, + tc.name, + tc.status, + tc.publish_tx_hash, + tc.publish_time, + tc.initialized_tx_hash, + tc.initialized_at, + tc.permissions, + COUNT(DISTINCT vc.id) as vendor_contract_count, + COUNT(DISTINCT vc.id) FILTER (WHERE vc.status = 'active') as active_contracts, + COUNT(DISTINCT vc.id) FILTER (WHERE vc.status = 'completed') as completed_contracts, + COUNT(DISTINCT vc.id) FILTER (WHERE vc.status = 'cancelled') as cancelled_contracts, + COALESCE(SUM(u.lovelace_amount) FILTER (WHERE NOT u.spent AND u.address = tc.contract_address), 0)::BIGINT as treasury_balance, + COUNT(u.id) FILTER (WHERE NOT u.spent AND u.address = tc.contract_address) as utxo_count, + (SELECT COUNT(*) FROM treasury.events WHERE treasury_id = tc.id) as total_events, + (SELECT MAX(block_time) FROM treasury.events WHERE treasury_id = tc.id) as last_event_time, + tc.created_at, + tc.updated_at +FROM treasury.treasury_contracts tc +LEFT JOIN treasury.vendor_contracts vc ON vc.treasury_id = tc.id +LEFT JOIN treasury.utxos u ON u.address = tc.contract_address +GROUP BY tc.id; + +-- Events with full context (treasury, project, milestone info) +CREATE OR REPLACE VIEW treasury.v_events_with_context AS +SELECT + e.id, + e.tx_hash, + e.slot, + e.block_number, + e.block_time, + e.event_type, + e.amount_lovelace, + e.reason, + e.destination, + e.metadata, + e.created_at, + -- Treasury context + tc.contract_instance as treasury_instance, + tc.name as treasury_name, + -- Project context + vc.project_id, + vc.project_name, + vc.vendor_name, + vc.contract_address as project_address, + -- Milestone context + m.milestone_id, + m.label as milestone_label, + m.milestone_order +FROM treasury.events e +LEFT JOIN treasury.treasury_contracts tc ON tc.id = e.treasury_id +LEFT JOIN treasury.vendor_contracts vc ON vc.id = e.vendor_contract_id +LEFT JOIN treasury.milestones m ON m.id = e.milestone_id; + +-- Financial summary view (allocated vs disbursed vs remaining) +CREATE OR REPLACE VIEW treasury.v_financial_summary AS +SELECT + tc.id as treasury_id, + tc.contract_instance, + tc.name as treasury_name, + -- Allocation totals + COALESCE(SUM(vc.initial_amount_lovelace), 0)::BIGINT as total_allocated_lovelace, + -- Disbursement totals + COALESCE(SUM(m_totals.total_disbursed), 0)::BIGINT as total_disbursed_lovelace, + -- Remaining (allocated - disbursed) + (COALESCE(SUM(vc.initial_amount_lovelace), 0) - COALESCE(SUM(m_totals.total_disbursed), 0))::BIGINT as total_remaining_lovelace, + -- Treasury balance (actual UTXOs) + COALESCE(SUM(u.lovelace_amount) FILTER (WHERE NOT u.spent AND u.address = tc.contract_address), 0)::BIGINT as treasury_balance_lovelace, + -- Project-level balance (sum of project UTXOs) + COALESCE(( + SELECT SUM(u2.lovelace_amount) + FROM treasury.utxos u2 + JOIN treasury.vendor_contracts vc2 ON vc2.id = u2.vendor_contract_id + WHERE vc2.treasury_id = tc.id AND NOT u2.spent + ), 0)::BIGINT as project_balance_lovelace, + -- Counts + COUNT(DISTINCT vc.id) as project_count, + COUNT(DISTINCT CASE WHEN vc.status = 'active' THEN vc.id END) as active_project_count +FROM treasury.treasury_contracts tc +LEFT JOIN treasury.vendor_contracts vc ON vc.treasury_id = tc.id +LEFT JOIN ( + SELECT + m.vendor_contract_id, + SUM(COALESCE(m.disburse_amount, 0)) as total_disbursed + FROM treasury.milestones m + GROUP BY m.vendor_contract_id +) m_totals ON m_totals.vendor_contract_id = vc.id +LEFT JOIN treasury.utxos u ON u.address = tc.contract_address +GROUP BY tc.id; diff --git a/database/init/02-yaci-store-tables.sql b/database/init/02-yaci-store-tables.sql deleted file mode 100644 index b7c0418..0000000 --- a/database/init/02-yaci-store-tables.sql +++ /dev/null @@ -1,292 +0,0 @@ --- YACI Store Schema for PostgreSQL --- Combined from YACI Store 2.0.0 migration files --- This schema is run before the application starts to create all required tables --- All statements are idempotent (safe to re-run) - --- ===================================================== --- Create yaci_store schema --- ===================================================== -CREATE SCHEMA IF NOT EXISTS yaci_store; - --- Set search path so all tables are created in yaci_store schema -SET search_path TO yaci_store; - --- ===================================================== --- Core Tables (from components/core) --- ===================================================== - -CREATE TABLE IF NOT EXISTS cursor_ -( - id integer not null, - block_hash varchar(64), - slot bigint, - block_number bigint, - era int, - prev_block_hash varchar(64), - create_datetime timestamp, - update_datetime timestamp, - primary key (id, block_hash) -); - -CREATE INDEX IF NOT EXISTS idx_cursor_id ON cursor_(id); -CREATE INDEX IF NOT EXISTS idx_cursor_slot ON cursor_(slot); -CREATE INDEX IF NOT EXISTS idx_cursor_block_number ON cursor_(block_number); -CREATE INDEX IF NOT EXISTS idx_cursor_block_hash ON cursor_(block_hash); - -CREATE TABLE IF NOT EXISTS era -( - era int not null primary key, - start_slot bigint not null, - block bigint not null, - block_hash varchar(64) not null -); - --- ===================================================== --- Block Store Tables (from stores/blocks) --- ===================================================== - -CREATE TABLE IF NOT EXISTS block -( - hash varchar(64) not null - primary key, - number bigint, - body_hash varchar(64), - body_size integer, - epoch integer, - total_output numeric(38) null, - total_fees bigint null, - block_time bigint null, - era smallint, - issuer_vkey varchar(64), - leader_vrf jsonb, - nonce_vrf jsonb, - prev_hash varchar(64), - protocol_version varchar(64), - slot bigint, - vrf_result jsonb, - vrf_vkey varchar(64), - no_of_txs integer, - slot_leader varchar(56), - epoch_slot integer, - op_cert_hot_vkey varchar(64) null, - op_cert_seq_number bigint null, - op_cert_kes_period bigint null, - op_cert_sigma varchar(256) null, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_block_number ON block(number); -CREATE INDEX IF NOT EXISTS idx_block_epoch ON block(epoch); -CREATE INDEX IF NOT EXISTS idx_block_slot_leader ON block(slot_leader); -CREATE INDEX IF NOT EXISTS idx_block_slot ON block(slot); - -CREATE TABLE IF NOT EXISTS rollback -( - id bigint not null primary key generated always as identity, - rollback_to_block_hash varchar(64), - rollback_to_slot bigint, - current_block_hash varchar(64), - current_slot bigint, - current_block bigint, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE TABLE IF NOT EXISTS block_cbor -( - block_hash varchar(64) not null primary key, - cbor_data bytea not null, - cbor_size integer, - slot bigint not null, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_block_cbor_slot ON block_cbor(slot); - --- ===================================================== --- UTXO Store Tables (from stores/utxo) --- ===================================================== - -CREATE TABLE IF NOT EXISTS address_utxo -( - tx_hash varchar(64) not null, - output_index smallint not null, - slot bigint, - block_hash varchar(64), - epoch integer, - lovelace_amount bigint null, - amounts jsonb, - data_hash varchar(64), - inline_datum text, - owner_addr varchar(500), - owner_addr_full text, - owner_stake_addr varchar(255), - owner_payment_credential varchar(56), - owner_stake_credential varchar(56), - script_ref text, - reference_script_hash varchar(56) null, - is_collateral_return boolean, - block bigint, - block_time bigint, - update_datetime timestamp, - primary key (output_index, tx_hash) -); - -CREATE INDEX IF NOT EXISTS idx_address_utxo_slot ON address_utxo(slot); -CREATE INDEX IF NOT EXISTS idx_reference_script_hash ON address_utxo(reference_script_hash); - -CREATE TABLE IF NOT EXISTS tx_input -( - output_index smallint not null, - tx_hash varchar(64) not null, - spent_at_slot bigint, - spent_at_block bigint, - spent_at_block_hash varchar(64), - spent_block_time bigint, - spent_epoch integer, - spent_tx_hash varchar(64) null, - primary key (output_index, tx_hash) -); - -CREATE INDEX IF NOT EXISTS idx_tx_input_slot ON tx_input(spent_at_slot); -CREATE INDEX IF NOT EXISTS idx_tx_input_block ON tx_input(spent_at_block); - -CREATE TABLE IF NOT EXISTS address -( - id bigserial, - address varchar(500) unique not null, - addr_full text, - payment_credential varchar(56), - stake_address varchar(255), - stake_credential varchar(56), - slot bigint, - update_datetime timestamp, - primary key (id) -); - -CREATE INDEX IF NOT EXISTS idx_address_stake_address ON address (stake_address); -CREATE INDEX IF NOT EXISTS idx_address_slot ON address (slot); - -CREATE TABLE IF NOT EXISTS ptr_address -( - address varchar(500), - stake_address varchar(255), - primary key (address) -); - --- ===================================================== --- Transaction Store Tables (from stores/transaction) --- ===================================================== - -CREATE TABLE IF NOT EXISTS transaction -( - tx_hash varchar(64) not null - primary key, - auxiliary_datahash varchar(64), - block_hash varchar(64), - collateral_inputs jsonb, - collateral_return jsonb, - fee bigint, - inputs jsonb, - invalid boolean, - network_id smallint, - outputs jsonb, - reference_inputs jsonb, - required_signers jsonb, - script_datahash varchar(64), - slot bigint, - total_collateral bigint, - ttl bigint, - validity_interval_start bigint, - collateral_return_json jsonb, - tx_index integer, - treasury_donation bigint, - epoch integer, - block bigint, - block_time bigint, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_transaction_slot ON transaction(slot); - -CREATE TABLE IF NOT EXISTS transaction_witness -( - tx_hash varchar(64) not null, - idx integer not null, - pub_key varchar(128), - signature varchar(128), - pub_keyhash varchar(56), - type varchar(40), - additional_data jsonb, - slot bigint, - primary key (tx_hash, idx) -); - -CREATE INDEX IF NOT EXISTS idx_transaction_witness_slot ON transaction_witness(slot); - -CREATE TABLE IF NOT EXISTS withdrawal -( - tx_hash varchar(64), - address varchar(255), - amount numeric(38), - epoch integer, - slot bigint, - block bigint, - block_time bigint, - update_datetime timestamp, - primary key (address, tx_hash) -); - -CREATE INDEX IF NOT EXISTS idx_withdrawal_slot ON withdrawal(slot); - -CREATE TABLE IF NOT EXISTS invalid_transaction -( - tx_hash varchar(64) not null - primary key, - slot bigint not null, - block_hash varchar(64), - transaction jsonb, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_invalid_transaction_slot ON invalid_transaction(slot); - -CREATE TABLE IF NOT EXISTS transaction_cbor -( - tx_hash varchar(64) not null primary key, - cbor_data bytea not null, - cbor_size integer, - slot bigint not null, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_transaction_cbor_slot ON transaction_cbor(slot); - --- ===================================================== --- Metadata Store Tables (from stores/metadata) --- ===================================================== - -CREATE TABLE IF NOT EXISTS transaction_metadata -( - id uuid not null primary key, - slot bigint, - tx_hash varchar(64) not null, - label varchar(255), - body text, - cbor text, - block bigint, - block_time bigint, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_txn_metadata_slot ON transaction_metadata(slot); -CREATE INDEX IF NOT EXISTS idx_txn_metadata_label ON transaction_metadata(label); -CREATE INDEX IF NOT EXISTS idx_txn_metadata_tx_hash ON transaction_metadata(tx_hash); - --- ===================================================== --- Schema initialization complete --- ===================================================== diff --git a/database/migrations/V1__create_treasury_tables.sql b/database/migrations/V1__create_treasury_tables.sql deleted file mode 100644 index bb56f0b..0000000 --- a/database/migrations/V1__create_treasury_tables.sql +++ /dev/null @@ -1,106 +0,0 @@ --- Custom tables for administration data tracking --- These tables extend YACI Store's schema with treasury-specific data - --- Treasury transactions with parsed metadata -CREATE TABLE IF NOT EXISTS treasury_transactions ( - id BIGSERIAL PRIMARY KEY, - tx_hash VARCHAR(64) NOT NULL UNIQUE, - slot BIGINT NOT NULL, - block_number BIGINT NOT NULL, - block_time TIMESTAMP NOT NULL, - action_type VARCHAR(50), -- Fund, Disburse, Reorganize, SweepTreasury, Pause, Resume, Modify, Withdraw - amount_lovelace BIGINT, - amount_ada NUMERIC(20, 6) GENERATED ALWAYS AS (amount_lovelace / 1000000.0) STORED, - metadata JSONB, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - --- Indexes for treasury_transactions -CREATE INDEX IF NOT EXISTS idx_treasury_tx_hash ON treasury_transactions(tx_hash); -CREATE INDEX IF NOT EXISTS idx_treasury_tx_slot ON treasury_transactions(slot); -CREATE INDEX IF NOT EXISTS idx_treasury_tx_action_type ON treasury_transactions(action_type); -CREATE INDEX IF NOT EXISTS idx_treasury_tx_block_time ON treasury_transactions(block_time); - --- Treasury UTXOs tracking -CREATE TABLE IF NOT EXISTS treasury_utxos ( - id BIGSERIAL PRIMARY KEY, - tx_hash VARCHAR(64) NOT NULL, - output_index INTEGER NOT NULL, - owner_addr VARCHAR(255) NOT NULL, - lovelace_amount BIGINT NOT NULL, - slot BIGINT NOT NULL, - spent_at_slot BIGINT, - is_spent BOOLEAN DEFAULT FALSE, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(tx_hash, output_index) -); - --- Indexes for treasury_utxos -CREATE INDEX IF NOT EXISTS idx_treasury_utxo_owner ON treasury_utxos(owner_addr); -CREATE INDEX IF NOT EXISTS idx_treasury_utxo_spent ON treasury_utxos(is_spent); -CREATE INDEX IF NOT EXISTS idx_treasury_utxo_slot ON treasury_utxos(slot); - --- Vendor contracts discovered from transactions -CREATE TABLE IF NOT EXISTS vendor_contracts ( - id BIGSERIAL PRIMARY KEY, - contract_address VARCHAR(255) NOT NULL UNIQUE, - vendor_name VARCHAR(255), - project_name VARCHAR(255), - project_code VARCHAR(100), - treasury_contract_address VARCHAR(255), -- Parent treasury contract - created_at_tx_hash VARCHAR(64), - created_at_slot BIGINT, - current_balance_lovelace BIGINT DEFAULT 0, - status VARCHAR(50) DEFAULT 'active', -- active, paused, completed, cancelled - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - --- Indexes for vendor_contracts -CREATE INDEX IF NOT EXISTS idx_vendor_contract_address ON vendor_contracts(contract_address); -CREATE INDEX IF NOT EXISTS idx_vendor_contract_treasury ON vendor_contracts(treasury_contract_address); -CREATE INDEX IF NOT EXISTS idx_vendor_contract_status ON vendor_contracts(status); - --- Fund flows tracking movements between contracts -CREATE TABLE IF NOT EXISTS fund_flows ( - id BIGSERIAL PRIMARY KEY, - tx_hash VARCHAR(64) NOT NULL, - slot BIGINT NOT NULL, - block_time TIMESTAMP NOT NULL, - source_address VARCHAR(255) NOT NULL, - destination_address VARCHAR(255) NOT NULL, - amount_lovelace BIGINT NOT NULL, - amount_ada NUMERIC(20, 6) GENERATED ALWAYS AS (amount_lovelace / 1000000.0) STORED, - flow_type VARCHAR(50), -- Fund, Disburse, Withdraw, Sweep, etc. - metadata JSONB, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - --- Indexes for fund_flows -CREATE INDEX IF NOT EXISTS idx_fund_flows_tx_hash ON fund_flows(tx_hash); -CREATE INDEX IF NOT EXISTS idx_fund_flows_source ON fund_flows(source_address); -CREATE INDEX IF NOT EXISTS idx_fund_flows_destination ON fund_flows(destination_address); -CREATE INDEX IF NOT EXISTS idx_fund_flows_type ON fund_flows(flow_type); -CREATE INDEX IF NOT EXISTS idx_fund_flows_slot ON fund_flows(slot); -CREATE INDEX IF NOT EXISTS idx_fund_flows_block_time ON fund_flows(block_time); - --- Function to update updated_at timestamp -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ language 'plpgsql'; - --- Triggers for updated_at -CREATE TRIGGER update_treasury_transactions_updated_at BEFORE UPDATE ON treasury_transactions - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER update_treasury_utxos_updated_at BEFORE UPDATE ON treasury_utxos - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER update_vendor_contracts_updated_at BEFORE UPDATE ON vendor_contracts - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); diff --git a/dev.sh b/dev.sh index 2617815..4ff24d1 100755 --- a/dev.sh +++ b/dev.sh @@ -157,21 +157,11 @@ case "$COMMAND" in print_success "PostgreSQL is ready" - # Ensure yaci_store schema and tables exist - print_info "Ensuring yaci_store schema is applied..." - TABLES_COUNT=$(docker-compose exec -T postgres psql -U postgres -d administration_data -tAc \ - "SELECT count(*) FROM information_schema.tables WHERE table_schema='yaci_store'" 2>/dev/null \ - | tr -d '[:space:]' || echo "0") - - if [ "$TABLES_COUNT" = "0" ] || [ -z "$TABLES_COUNT" ]; then - print_info "Applying yaci_store schema and tables..." - docker-compose exec -T postgres psql -U postgres -d administration_data \ - -f /docker-entrypoint-initdb.d/01-create-schema.sql \ - -f /docker-entrypoint-initdb.d/02-yaci-store-tables.sql - print_success "Schema applied" - else - print_success "yaci_store schema exists ($TABLES_COUNT tables)" - fi + # Ensure treasury schema exists (YACI Store creates its own schema/tables via Flyway) + print_info "Ensuring treasury schema exists..." + docker-compose exec -T postgres psql -U postgres -d administration_data \ + -f /docker-entrypoint-initdb.d/02-treasury-schema.sql 2>/dev/null || true + print_success "Database schemas ready" # Start indexer if JAR is available if [ "$INDEXER_AVAILABLE" = true ]; then diff --git a/indexer/Dockerfile.render b/indexer/Dockerfile.render deleted file mode 100644 index 987a3da..0000000 --- a/indexer/Dockerfile.render +++ /dev/null @@ -1,20 +0,0 @@ -# Render-compatible Dockerfile for YACI Store Indexer -FROM bloxbean/yaci-store:2.0.0 - -USER root - -# Install PostgreSQL client for potential schema initialization -# Note: bloxbean/yaci-store is based on Red Hat UBI (uses microdnf) -RUN microdnf install -y postgresql && microdnf clean all - -# Copy configuration, schema, and startup script -COPY application.render.properties /app/config/application.properties.template -COPY config/application-plugins.yml /app/config/application-plugins.yml -COPY plugins/scripts/ /app/plugins/scripts/ -COPY init-schema.sql /app/init-schema.sql -COPY render-entrypoint.sh /app/render-entrypoint.sh -RUN chmod +x /app/render-entrypoint.sh - -EXPOSE 8080 - -ENTRYPOINT ["/app/render-entrypoint.sh"] diff --git a/indexer/README.md b/indexer/README.md index c9b566e..dc8cd3c 100644 --- a/indexer/README.md +++ b/indexer/README.md @@ -142,7 +142,7 @@ curl http://localhost:8081/actuator/prometheus ## Database Schema -YACI Store creates tables in the `yaci_store` schema: +YACI Store automatically creates and manages its tables via Flyway migrations on startup. Tables are created in the `yaci_store` schema: | Table | Description | |-------|-------------| diff --git a/indexer/SETUP.md b/indexer/SETUP.md index b3d2dde..40dba84 100644 --- a/indexer/SETUP.md +++ b/indexer/SETUP.md @@ -37,9 +37,12 @@ This is used by the `treasury-filter.mvel` plugin script to filter metadata by i Edit `application.properties` to configure: - Database connection details - Cardano network settings -- Sync start point (slot/block) - Which stores to enable/disable +### Database Schema + +YACI Store manages its own database schema and tables automatically via Flyway migrations on startup. No manual table creation is needed. + ## Build and Test 1. Build the Docker image: diff --git a/indexer/add-missing-tables.sql b/indexer/add-missing-tables.sql deleted file mode 100644 index f0a5463..0000000 --- a/indexer/add-missing-tables.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Add missing tables to yaci_store schema -SET search_path TO yaci_store; - -drop table if exists rollback cascade; -create table rollback -( - id bigint not null primary key generated always as identity, - rollback_to_block_hash varchar(64), - rollback_to_slot bigint, - current_block_hash varchar(64), - current_slot bigint, - current_block bigint, - create_datetime timestamp, - update_datetime timestamp -); - -drop table if exists block_cbor cascade; -create table block_cbor -( - block_hash varchar(64) not null primary key, - cbor_data bytea not null, - cbor_size integer, - slot bigint not null, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_block_cbor_slot ON block_cbor(slot); - --- Verify tables were created -SELECT table_name FROM information_schema.tables WHERE table_schema = 'yaci_store' ORDER BY table_name; diff --git a/indexer/application.properties b/indexer/application.properties index 0b741e3..8b17598 100644 --- a/indexer/application.properties +++ b/indexer/application.properties @@ -11,13 +11,8 @@ spring.datasource.password=postgres # HikariCP Connection Pool spring.datasource.hikari.maximum-pool-size=30 -# JPA/Hibernate - tables pre-created by init SQL, ddl-auto=none avoids type mapping errors -spring.jpa.hibernate.ddl-auto=none +# JPA/Hibernate spring.jpa.properties.hibernate.default_schema=yaci_store - -# Flyway - baseline existing schema so migrations can proceed -spring.flyway.baseline-on-migrate=true - spring.jpa.properties.hibernate.jdbc.batch_size=100 spring.jpa.properties.hibernate.order_inserts=true diff --git a/indexer/application.render.properties b/indexer/application.render.properties deleted file mode 100644 index d43c03a..0000000 --- a/indexer/application.render.properties +++ /dev/null @@ -1,91 +0,0 @@ -# Cardano Network Configuration (Mainnet) -store.cardano.host=backbone.cardano.iog.io -store.cardano.port=3001 -store.cardano.protocol-magic=764824073 - -# Database Configuration - Uses environment variables from Render -# URL is set via SPRING_DATASOURCE_URL env var (converted to JDBC format by entrypoint) -spring.datasource.username=${SPRING_DATASOURCE_USERNAME:postgres} -spring.datasource.password=${SPRING_DATASOURCE_PASSWORD:postgres} - -# HikariCP Connection Pool (conservative for shared DB) -spring.datasource.hikari.maximum-pool-size=10 -spring.datasource.hikari.minimum-idle=2 -spring.datasource.hikari.connection-timeout=30000 - -# JPA/Hibernate Configuration - tables created by init-schema.sql -# Using 'none' mode since we pre-create tables via SQL script in entrypoint -# This avoids Hibernate type mapping errors with YACI Store's custom types -spring.jpa.hibernate.ddl-auto=none -spring.jpa.properties.hibernate.default_schema=yaci_store - -# Flyway - baseline existing schema so migrations can proceed -spring.flyway.baseline-on-migrate=true - -spring.jpa.properties.hibernate.jdbc.batch_size=50 -spring.jpa.properties.hibernate.order_inserts=true - -# Logging -logging.level.com.bloxbean.cardano.yaci.core.protocol.keepalive=INFO -logging.level.com.bloxbean.cardano.yaci.store.core.service=INFO - -# Store Configuration - Enable what we need for administration data tracking -store.blocks.enabled=true -store.transaction.enabled=true -store.utxo.enabled=true -store.metadata.enabled=true - -# Disable unnecessary stores to reduce resource usage -store.assets.enabled=false -store.epoch.enabled=false -store.mir.enabled=false -store.script.enabled=false -store.staking.enabled=false -store.governance.enabled=false - -# Disable aggregation modules -store.epoch-aggr.enabled=false -store.account.enabled=false - -# Storage Optimization - Disable unnecessary data storage -store.blocks.save-cbor=false -store.transaction.save-cbor=false -store.transaction.save-witness=false - -# UTXO Pruning - Remove spent UTXOs to save space -store.utxo.pruning-enabled=true -store.utxo.pruning.interval=600 -store.utxo.pruning-safe-blocks=2160 -store.utxo.pruning-batch-size=3000 - -# Parallel Processing (reduced for Render's resource constraints) -store.executor.enable-parallel-processing=true -store.executor.blocks-batch-size=50 -store.executor.blocks-partition-size=5 -store.executor.use-virtual-thread-for-batch-processing=true -store.executor.use-virtual-thread-for-event-processing=true - -# DB batch settings (reduced for shared DB) -store.db.batch-size=500 -store.db.parallel-insert=true - -# Auto index management -store.auto-index-management=true - -# API Configuration -server.port=8080 -apiPrefix=/api/v1 -management.endpoints.web.exposure.include=health,info,prometheus -management.endpoint.health.show-details=always - -# Virtual threads -spring.threads.virtual.enabled=true - -# Auto-start sync -store.sync-auto-start=true - -# Enable plugins profile -spring.profiles.active=plugins - -# Plugin logging -logging.level.com.bloxbean.cardano.yaci.store.plugin=INFO diff --git a/indexer/create-yaci-tables.sql b/indexer/create-yaci-tables.sql deleted file mode 100644 index 0f8ad9a..0000000 --- a/indexer/create-yaci-tables.sql +++ /dev/null @@ -1,92 +0,0 @@ --- YACI Store Required Tables --- This script creates the essential tables needed for YACI Store to run --- Run this if migrations don't work automatically - --- Cursor table (fixed: id is BIGINT) -CREATE TABLE IF NOT EXISTS yaci_store.cursor_ ( - id BIGINT PRIMARY KEY, - slot BIGINT NOT NULL, - block_number BIGINT NOT NULL, - block_hash VARCHAR(64), - prev_block_hash VARCHAR(64), - era VARCHAR(50), - create_datetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_datetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - --- Block table (block_time is BIGINT, not TIMESTAMP) -CREATE TABLE IF NOT EXISTS yaci_store.block ( - hash VARCHAR(64) PRIMARY KEY, - body_hash VARCHAR(64), - body_size BIGINT, - block_time BIGINT, -- Unix timestamp, not TIMESTAMP - create_datetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - epoch BIGINT, - epoch_slot BIGINT, - era VARCHAR(50), - issuer_vkey VARCHAR(255), - leader_vrf VARCHAR(255), - no_of_txs INTEGER, - nonce_vrf VARCHAR(255), - number BIGINT, - op_cert_hot_vkey VARCHAR(255), - op_cert_seq_number BIGINT, - op_cert_sigma VARCHAR(255), - op_cert_kes_period INTEGER, - prev_hash VARCHAR(64), - protocol_version VARCHAR(50), - slot BIGINT, - slot_leader VARCHAR(255), - total_fees BIGINT, - total_output BIGINT, - update_datetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - vrf_result VARCHAR(255), - vrf_vkey VARCHAR(255) -); - -CREATE INDEX IF NOT EXISTS idx_block_slot ON yaci_store.block(slot); -CREATE INDEX IF NOT EXISTS idx_block_number ON yaci_store.block(number); -CREATE INDEX IF NOT EXISTS idx_block_epoch ON yaci_store.block(epoch); - --- Transaction table -CREATE TABLE IF NOT EXISTS yaci_store.transaction ( - tx_hash VARCHAR(64) PRIMARY KEY, - auxiliary_datahash VARCHAR(64), - block_hash VARCHAR(64), - block BIGINT, - block_time BIGINT, - collateral_inputs TEXT, - collateral_return TEXT, - collateral_return_json JSONB, - epoch BIGINT, - fee BIGINT, - inputs TEXT, - invalid BOOLEAN, - network_id INTEGER, - outputs TEXT, - reference_inputs TEXT, - required_signers TEXT, - script_datahash VARCHAR(64), - slot BIGINT, - total_collateral BIGINT, - treasury_donation BIGINT, - ttl BIGINT, - tx_index INTEGER, - validity_interval_start BIGINT, - update_datetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - -CREATE INDEX IF NOT EXISTS idx_transaction_slot ON yaci_store.transaction(slot); -CREATE INDEX IF NOT EXISTS idx_transaction_block ON yaci_store.transaction(block); -CREATE INDEX IF NOT EXISTS idx_transaction_block_hash ON yaci_store.transaction(block_hash); -CREATE INDEX IF NOT EXISTS idx_transaction_epoch ON yaci_store.transaction(epoch); - --- Era table -CREATE TABLE IF NOT EXISTS yaci_store.era ( - era INTEGER PRIMARY KEY, - block BIGINT, - block_hash VARCHAR(64), - start_slot BIGINT -); - -CREATE INDEX IF NOT EXISTS idx_era_start_slot ON yaci_store.era(start_slot); diff --git a/indexer/init-schema.sql b/indexer/init-schema.sql deleted file mode 100644 index b7c0418..0000000 --- a/indexer/init-schema.sql +++ /dev/null @@ -1,292 +0,0 @@ --- YACI Store Schema for PostgreSQL --- Combined from YACI Store 2.0.0 migration files --- This schema is run before the application starts to create all required tables --- All statements are idempotent (safe to re-run) - --- ===================================================== --- Create yaci_store schema --- ===================================================== -CREATE SCHEMA IF NOT EXISTS yaci_store; - --- Set search path so all tables are created in yaci_store schema -SET search_path TO yaci_store; - --- ===================================================== --- Core Tables (from components/core) --- ===================================================== - -CREATE TABLE IF NOT EXISTS cursor_ -( - id integer not null, - block_hash varchar(64), - slot bigint, - block_number bigint, - era int, - prev_block_hash varchar(64), - create_datetime timestamp, - update_datetime timestamp, - primary key (id, block_hash) -); - -CREATE INDEX IF NOT EXISTS idx_cursor_id ON cursor_(id); -CREATE INDEX IF NOT EXISTS idx_cursor_slot ON cursor_(slot); -CREATE INDEX IF NOT EXISTS idx_cursor_block_number ON cursor_(block_number); -CREATE INDEX IF NOT EXISTS idx_cursor_block_hash ON cursor_(block_hash); - -CREATE TABLE IF NOT EXISTS era -( - era int not null primary key, - start_slot bigint not null, - block bigint not null, - block_hash varchar(64) not null -); - --- ===================================================== --- Block Store Tables (from stores/blocks) --- ===================================================== - -CREATE TABLE IF NOT EXISTS block -( - hash varchar(64) not null - primary key, - number bigint, - body_hash varchar(64), - body_size integer, - epoch integer, - total_output numeric(38) null, - total_fees bigint null, - block_time bigint null, - era smallint, - issuer_vkey varchar(64), - leader_vrf jsonb, - nonce_vrf jsonb, - prev_hash varchar(64), - protocol_version varchar(64), - slot bigint, - vrf_result jsonb, - vrf_vkey varchar(64), - no_of_txs integer, - slot_leader varchar(56), - epoch_slot integer, - op_cert_hot_vkey varchar(64) null, - op_cert_seq_number bigint null, - op_cert_kes_period bigint null, - op_cert_sigma varchar(256) null, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_block_number ON block(number); -CREATE INDEX IF NOT EXISTS idx_block_epoch ON block(epoch); -CREATE INDEX IF NOT EXISTS idx_block_slot_leader ON block(slot_leader); -CREATE INDEX IF NOT EXISTS idx_block_slot ON block(slot); - -CREATE TABLE IF NOT EXISTS rollback -( - id bigint not null primary key generated always as identity, - rollback_to_block_hash varchar(64), - rollback_to_slot bigint, - current_block_hash varchar(64), - current_slot bigint, - current_block bigint, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE TABLE IF NOT EXISTS block_cbor -( - block_hash varchar(64) not null primary key, - cbor_data bytea not null, - cbor_size integer, - slot bigint not null, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_block_cbor_slot ON block_cbor(slot); - --- ===================================================== --- UTXO Store Tables (from stores/utxo) --- ===================================================== - -CREATE TABLE IF NOT EXISTS address_utxo -( - tx_hash varchar(64) not null, - output_index smallint not null, - slot bigint, - block_hash varchar(64), - epoch integer, - lovelace_amount bigint null, - amounts jsonb, - data_hash varchar(64), - inline_datum text, - owner_addr varchar(500), - owner_addr_full text, - owner_stake_addr varchar(255), - owner_payment_credential varchar(56), - owner_stake_credential varchar(56), - script_ref text, - reference_script_hash varchar(56) null, - is_collateral_return boolean, - block bigint, - block_time bigint, - update_datetime timestamp, - primary key (output_index, tx_hash) -); - -CREATE INDEX IF NOT EXISTS idx_address_utxo_slot ON address_utxo(slot); -CREATE INDEX IF NOT EXISTS idx_reference_script_hash ON address_utxo(reference_script_hash); - -CREATE TABLE IF NOT EXISTS tx_input -( - output_index smallint not null, - tx_hash varchar(64) not null, - spent_at_slot bigint, - spent_at_block bigint, - spent_at_block_hash varchar(64), - spent_block_time bigint, - spent_epoch integer, - spent_tx_hash varchar(64) null, - primary key (output_index, tx_hash) -); - -CREATE INDEX IF NOT EXISTS idx_tx_input_slot ON tx_input(spent_at_slot); -CREATE INDEX IF NOT EXISTS idx_tx_input_block ON tx_input(spent_at_block); - -CREATE TABLE IF NOT EXISTS address -( - id bigserial, - address varchar(500) unique not null, - addr_full text, - payment_credential varchar(56), - stake_address varchar(255), - stake_credential varchar(56), - slot bigint, - update_datetime timestamp, - primary key (id) -); - -CREATE INDEX IF NOT EXISTS idx_address_stake_address ON address (stake_address); -CREATE INDEX IF NOT EXISTS idx_address_slot ON address (slot); - -CREATE TABLE IF NOT EXISTS ptr_address -( - address varchar(500), - stake_address varchar(255), - primary key (address) -); - --- ===================================================== --- Transaction Store Tables (from stores/transaction) --- ===================================================== - -CREATE TABLE IF NOT EXISTS transaction -( - tx_hash varchar(64) not null - primary key, - auxiliary_datahash varchar(64), - block_hash varchar(64), - collateral_inputs jsonb, - collateral_return jsonb, - fee bigint, - inputs jsonb, - invalid boolean, - network_id smallint, - outputs jsonb, - reference_inputs jsonb, - required_signers jsonb, - script_datahash varchar(64), - slot bigint, - total_collateral bigint, - ttl bigint, - validity_interval_start bigint, - collateral_return_json jsonb, - tx_index integer, - treasury_donation bigint, - epoch integer, - block bigint, - block_time bigint, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_transaction_slot ON transaction(slot); - -CREATE TABLE IF NOT EXISTS transaction_witness -( - tx_hash varchar(64) not null, - idx integer not null, - pub_key varchar(128), - signature varchar(128), - pub_keyhash varchar(56), - type varchar(40), - additional_data jsonb, - slot bigint, - primary key (tx_hash, idx) -); - -CREATE INDEX IF NOT EXISTS idx_transaction_witness_slot ON transaction_witness(slot); - -CREATE TABLE IF NOT EXISTS withdrawal -( - tx_hash varchar(64), - address varchar(255), - amount numeric(38), - epoch integer, - slot bigint, - block bigint, - block_time bigint, - update_datetime timestamp, - primary key (address, tx_hash) -); - -CREATE INDEX IF NOT EXISTS idx_withdrawal_slot ON withdrawal(slot); - -CREATE TABLE IF NOT EXISTS invalid_transaction -( - tx_hash varchar(64) not null - primary key, - slot bigint not null, - block_hash varchar(64), - transaction jsonb, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_invalid_transaction_slot ON invalid_transaction(slot); - -CREATE TABLE IF NOT EXISTS transaction_cbor -( - tx_hash varchar(64) not null primary key, - cbor_data bytea not null, - cbor_size integer, - slot bigint not null, - create_datetime timestamp, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_transaction_cbor_slot ON transaction_cbor(slot); - --- ===================================================== --- Metadata Store Tables (from stores/metadata) --- ===================================================== - -CREATE TABLE IF NOT EXISTS transaction_metadata -( - id uuid not null primary key, - slot bigint, - tx_hash varchar(64) not null, - label varchar(255), - body text, - cbor text, - block bigint, - block_time bigint, - update_datetime timestamp -); - -CREATE INDEX IF NOT EXISTS idx_txn_metadata_slot ON transaction_metadata(slot); -CREATE INDEX IF NOT EXISTS idx_txn_metadata_label ON transaction_metadata(label); -CREATE INDEX IF NOT EXISTS idx_txn_metadata_tx_hash ON transaction_metadata(tx_hash); - --- ===================================================== --- Schema initialization complete --- ===================================================== diff --git a/indexer/render-entrypoint.sh b/indexer/render-entrypoint.sh deleted file mode 100644 index 958a586..0000000 --- a/indexer/render-entrypoint.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -e - -echo "=== YACI Store Render Entrypoint ===" - -# Convert Render's postgres:// URL to JDBC format -# Render provides: postgres://user:password@host:port/database -# Spring needs: jdbc:postgresql://host:port/database (with user/pass separate) - -if [ -n "$SPRING_DATASOURCE_URL" ]; then - # Remove the protocol prefix - URL_WITHOUT_PROTOCOL=$(echo "$SPRING_DATASOURCE_URL" | sed 's|^postgres://||' | sed 's|^postgresql://||') - - # Extract user:password (everything before @) - USER_PASS=$(echo "$URL_WITHOUT_PROTOCOL" | sed 's|@.*||') - - # Extract host:port/database (everything after @) - HOST_PORT_DB=$(echo "$URL_WITHOUT_PROTOCOL" | sed 's|.*@||') - - # Build JDBC URL with yaci_store schema as default - JDBC_URL="jdbc:postgresql://${HOST_PORT_DB}?currentSchema=yaci_store" - - # Extract username and password if not already set - if [ -z "$SPRING_DATASOURCE_USERNAME" ]; then - SPRING_DATASOURCE_USERNAME=$(echo "$USER_PASS" | sed 's|:.*||') - export SPRING_DATASOURCE_USERNAME - fi - - if [ -z "$SPRING_DATASOURCE_PASSWORD" ]; then - SPRING_DATASOURCE_PASSWORD=$(echo "$USER_PASS" | sed 's|[^:]*:||') - export SPRING_DATASOURCE_PASSWORD - fi - - export SPRING_DATASOURCE_URL="$JDBC_URL" - echo "Configured database: ${HOST_PORT_DB}" - echo "Database user: ${SPRING_DATASOURCE_USERNAME}" - - # Parse host, port, and database name for psql - # Handle both host:port/db and host/db formats - if echo "$HOST_PORT_DB" | grep -q ':[0-9]'; then - # Has explicit port (host:port/database) - DB_HOST=$(echo "$HOST_PORT_DB" | sed 's|:.*||') - DB_PORT_AND_NAME=$(echo "$HOST_PORT_DB" | sed 's|[^:]*:||') - DB_PORT=$(echo "$DB_PORT_AND_NAME" | sed 's|/.*||') - DB_NAME=$(echo "$DB_PORT_AND_NAME" | sed 's|[^/]*/||') - else - # No explicit port (host/database) - default to 5432 - DB_HOST=$(echo "$HOST_PORT_DB" | sed 's|/.*||') - DB_PORT="5432" - DB_NAME=$(echo "$HOST_PORT_DB" | sed 's|[^/]*/||') - fi - - echo "Parsed connection: host=$DB_HOST port=$DB_PORT database=$DB_NAME" - - # Initialize database schema before starting app - echo "Initializing database schema..." - export PGPASSWORD="$SPRING_DATASOURCE_PASSWORD" - - if psql -h "$DB_HOST" -p "$DB_PORT" -U "$SPRING_DATASOURCE_USERNAME" -d "$DB_NAME" -f /app/init-schema.sql; then - echo "Database schema initialized successfully" - else - echo "Warning: Schema initialization had errors (tables may already exist)" - fi - - unset PGPASSWORD -fi - -# Copy template to config -cp /app/config/application.properties.template /app/config/application.properties - -# Start YACI Store -echo "Starting YACI Store..." -exec java $JAVA_OPTS -jar /app/yaci-store.jar --spring.config.location=file:/app/config/application.properties diff --git a/render.yaml b/render.yaml deleted file mode 100644 index 93e31fc..0000000 --- a/render.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Render Blueprint for Cardano Administration Fund Tracking System -# https://render.com/docs/blueprint-spec -# -# Deploy: Connect your GitHub repo to Render and select this blueprint -# Or use: render blueprint apply -# -# After deployment: -# 1. The indexer will start syncing from the configured slot -# 2. Initial sync takes time - monitor via /actuator/health endpoint -# 3. API will be available immediately at /health - -services: - # Administration API - Rust backend for querying administration data - - type: web - name: administration-api - runtime: docker - dockerfilePath: ./api/Dockerfile - dockerContext: ./api - region: oregon - plan: starter - healthCheckPath: /health - envVars: - - key: DATABASE_URL - fromDatabase: - name: administration-db - property: connectionString - - key: RUST_LOG - value: info - autoDeploy: true - - # YACI Store Indexer - Syncs Cardano blockchain data - # Note: This is a long-running service that syncs blockchain data - # It needs more resources than the API - - type: web - name: administration-indexer - runtime: docker - dockerfilePath: ./indexer/Dockerfile.render - dockerContext: ./indexer - region: oregon - plan: standard # Needs more memory for blockchain sync - healthCheckPath: /actuator/health - envVars: - - key: JAVA_OPTS - value: -Xmx1536m -Xms512m -XX:+UseG1GC - - key: SPRING_DATASOURCE_URL - fromDatabase: - name: administration-db - property: connectionString - - key: SPRING_DATASOURCE_USERNAME - fromDatabase: - name: administration-db - property: user - - key: SPRING_DATASOURCE_PASSWORD - fromDatabase: - name: administration-db - property: password - autoDeploy: true - -databases: - # PostgreSQL database for storing blockchain and administration data - - name: administration-db - plan: basic-256mb # 1GB storage, upgrade to standard for production - region: oregon - postgresMajorVersion: "15"