diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 837cd50..17d3f56 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -12,6 +12,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: configure run: ./configure @@ -34,3 +36,8 @@ jobs: run: | export PATH="/home/runner/.config/.foundry/bin:$PATH"; make test; + + - name: Check storage layout + run: | + export PATH="/home/runner/.config/.foundry/bin:$PATH"; + make check-layout; diff --git a/Makefile b/Makefile index 6b41476..5da2657 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,10 @@ RPC_URL ?= KEYSTORE ?= PASSWORD ?= +# Generated files +LAYOUT=src/PDPVerifierLayout.sol +LAYOUT_JSON=src/PDPVerifierLayout.json + # Default target .PHONY: default default: build test @@ -54,4 +58,43 @@ extract-abis: .PHONY: contract-size-check contract-size-check: @echo "Checking contract sizes..." - bash tools/check-contract-size.sh \ No newline at end of file + bash tools/check-contract-size.sh + +# Storage layout generation +$(LAYOUT): tools/generate_storage_layout.sh src/PDPVerifier.sol + bash tools/generate_storage_layout.sh src/PDPVerifier.sol:PDPVerifier | forge fmt -r - > $@ + +# Storage layout JSON (full metadata for upgrade safety checks) +$(LAYOUT_JSON): src/PDPVerifier.sol + forge inspect --json src/PDPVerifier.sol:PDPVerifier storageLayout | jq -S '. as $$root | def normalize_type($$id): ($$root.types[$$id]) as $$type | {label: $$type.label, encoding: $$type.encoding, numberOfBytes: $$type.numberOfBytes} + (if $$type.key then {key: normalize_type($$type.key)} else {} end) + (if $$type.value then {value: normalize_type($$type.value)} else {} end) + (if $$type.base then {base: normalize_type($$type.base)} else {} end) + (if $$type.members then {members: [$$type.members[] | {label, slot, offset, type: normalize_type(.type)}]} else {} end); [.storage[] | {label, slot, offset, type: normalize_type(.type)}]' > $@ + +# Main code generation target +.PHONY: gen +gen: check-tools $(LAYOUT) $(LAYOUT_JSON) + @echo "Code generation complete" + +# Force regeneration - useful when things are broken +.PHONY: force-gen +force-gen: clean-gen gen + @echo "Force regeneration complete" + +# Clean generated files only +.PHONY: clean-gen +clean-gen: + @echo "Removing generated files..." + @rm -f $(LAYOUT) $(LAYOUT_JSON) + @echo "Generated files removed" + +# Check required tools +.PHONY: check-tools +check-tools: + @which jq >/dev/null 2>&1 || (echo "Error: jq is required but not installed" && exit 1) + @which forge >/dev/null 2>&1 || (echo "Error: forge is required but not installed" && exit 1) + +# Storage layout validation +.PHONY: check-layout +check-layout: force-gen + @echo "Checking if layout files are up to date..." + @git diff --exit-code $(LAYOUT) $(LAYOUT_JSON) || (echo "Error: Layout files are stale. Please commit the generated changes." && exit 1) + @echo "Checking storage layout for destructive changes..." + @bash tools/check_storage_layout.sh diff --git a/src/PDPVerifierLayout.json b/src/PDPVerifierLayout.json new file mode 100644 index 0000000..a6ec5e6 --- /dev/null +++ b/src/PDPVerifierLayout.json @@ -0,0 +1,413 @@ +[ + { + "label": "challengeFinality", + "offset": 0, + "slot": "0", + "type": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + }, + { + "label": "nextDataSetId", + "offset": 0, + "slot": "1", + "type": { + "encoding": "inplace", + "label": "uint64", + "numberOfBytes": "8" + } + }, + { + "label": "pieceCids", + "offset": 0, + "slot": "2", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => mapping(uint256 => struct Cids.Cid))", + "numberOfBytes": "32", + "value": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => struct Cids.Cid)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "struct Cids.Cid", + "members": [ + { + "label": "data", + "offset": 0, + "slot": "0", + "type": { + "encoding": "bytes", + "label": "bytes", + "numberOfBytes": "32" + } + } + ], + "numberOfBytes": "32" + } + } + } + }, + { + "label": "pieceLeafCounts", + "offset": 0, + "slot": "3", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => mapping(uint256 => uint256))", + "numberOfBytes": "32", + "value": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + } + }, + { + "label": "sumTreeCounts", + "offset": 0, + "slot": "4", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => mapping(uint256 => uint256))", + "numberOfBytes": "32", + "value": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + } + }, + { + "label": "nextPieceId", + "offset": 0, + "slot": "5", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + }, + { + "label": "dataSetLeafCount", + "offset": 0, + "slot": "6", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + }, + { + "label": "nextChallengeEpoch", + "offset": 0, + "slot": "7", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + }, + { + "label": "dataSetListener", + "offset": 0, + "slot": "8", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => address)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "address", + "numberOfBytes": "20" + } + } + }, + { + "label": "challengeRange", + "offset": 0, + "slot": "9", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + }, + { + "label": "scheduledRemovals", + "offset": 0, + "slot": "10", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256[])", + "numberOfBytes": "32", + "value": { + "base": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "encoding": "dynamic_array", + "label": "uint256[]", + "numberOfBytes": "32" + } + } + }, + { + "label": "scheduledRemovalsBitmap", + "offset": 0, + "slot": "11", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => mapping(uint256 => uint256))", + "numberOfBytes": "32", + "value": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + } + }, + { + "label": "storageProvider", + "offset": 0, + "slot": "12", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => address)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "address", + "numberOfBytes": "20" + } + } + }, + { + "label": "dataSetProposedStorageProvider", + "offset": 0, + "slot": "13", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => address)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "address", + "numberOfBytes": "20" + } + } + }, + { + "label": "dataSetLastProvenEpoch", + "offset": 0, + "slot": "14", + "type": { + "encoding": "mapping", + "key": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + }, + "label": "mapping(uint256 => uint256)", + "numberOfBytes": "32", + "value": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + }, + { + "label": "feeStatus", + "offset": 0, + "slot": "15", + "type": { + "encoding": "inplace", + "label": "struct PDPVerifier.FeeStatus", + "members": [ + { + "label": "currentFeePerTiB", + "offset": 0, + "slot": "0", + "type": { + "encoding": "inplace", + "label": "uint96", + "numberOfBytes": "12" + } + }, + { + "label": "nextFeePerTiB", + "offset": 12, + "slot": "0", + "type": { + "encoding": "inplace", + "label": "uint96", + "numberOfBytes": "12" + } + }, + { + "label": "transitionTime", + "offset": 24, + "slot": "0", + "type": { + "encoding": "inplace", + "label": "uint64", + "numberOfBytes": "8" + } + } + ], + "numberOfBytes": "32" + } + }, + { + "label": "nextUpgrade", + "offset": 0, + "slot": "16", + "type": { + "encoding": "inplace", + "label": "struct PDPVerifier.PlannedUpgrade", + "members": [ + { + "label": "nextImplementation", + "offset": 0, + "slot": "0", + "type": { + "encoding": "inplace", + "label": "address", + "numberOfBytes": "20" + } + }, + { + "label": "afterEpoch", + "offset": 20, + "slot": "0", + "type": { + "encoding": "inplace", + "label": "uint96", + "numberOfBytes": "12" + } + } + ], + "numberOfBytes": "32" + } + } +] diff --git a/src/PDPVerifierLayout.sol b/src/PDPVerifierLayout.sol new file mode 100644 index 0000000..fc20391 --- /dev/null +++ b/src/PDPVerifierLayout.sol @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT +pragma solidity ^0.8.20; + +// Code generated - DO NOT EDIT. +// This file is a generated binding and any changes will be lost. +// Generated with tools/generate_storage_layout.sh + +bytes32 constant CHALLENGE_FINALITY_SLOT = bytes32(uint256(0)); +bytes32 constant NEXT_DATA_SET_ID_SLOT = bytes32(uint256(1)); +bytes32 constant PIECE_CIDS_SLOT = bytes32(uint256(2)); +bytes32 constant PIECE_LEAF_COUNTS_SLOT = bytes32(uint256(3)); +bytes32 constant SUM_TREE_COUNTS_SLOT = bytes32(uint256(4)); +bytes32 constant NEXT_PIECE_ID_SLOT = bytes32(uint256(5)); +bytes32 constant DATA_SET_LEAF_COUNT_SLOT = bytes32(uint256(6)); +bytes32 constant NEXT_CHALLENGE_EPOCH_SLOT = bytes32(uint256(7)); +bytes32 constant DATA_SET_LISTENER_SLOT = bytes32(uint256(8)); +bytes32 constant CHALLENGE_RANGE_SLOT = bytes32(uint256(9)); +bytes32 constant SCHEDULED_REMOVALS_SLOT = bytes32(uint256(10)); +bytes32 constant SCHEDULED_REMOVALS_BITMAP_SLOT = bytes32(uint256(11)); +bytes32 constant STORAGE_PROVIDER_SLOT = bytes32(uint256(12)); +bytes32 constant DATA_SET_PROPOSED_STORAGE_PROVIDER_SLOT = bytes32(uint256(13)); +bytes32 constant DATA_SET_LAST_PROVEN_EPOCH_SLOT = bytes32(uint256(14)); +bytes32 constant FEE_STATUS_SLOT = bytes32(uint256(15)); +bytes32 constant NEXT_UPGRADE_SLOT = bytes32(uint256(16)); diff --git a/tools/check_storage_layout.sh b/tools/check_storage_layout.sh new file mode 100755 index 0000000..8def622 --- /dev/null +++ b/tools/check_storage_layout.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# Check that storage layout changes are additive only. +# Prevents destructive changes to upgradeable contract storage: +# - Removing existing storage slots/variables +# - Changing the type of an existing variable +# - Changing the offset of an existing variable +# - Inserting new slots in the middle (shifting existing slots) +# Allowed: Appending new slots at the end (highest slot numbers) +# +# Usage: check_storage_layout.sh [ ] +# No args: compares base branch/history to working tree +# Two args: compares base_layout.json to new_layout.json + +set -euo pipefail + +# Clean up temp files on exit +TEMP_FILES=() +cleanup() { rm -f "${TEMP_FILES[@]:-}" 2>/dev/null || true; } +trap cleanup EXIT + +LAYOUT_JSON="src/PDPVerifierLayout.json" + +# Function to validate a single layout JSON file +validate_layout_json() { + local file="$1" + if [ ! -f "$file" ]; then + echo "Error: Layout file not found: $file" >&2 + return 1 + fi + + # Check if it's a valid JSON array + if ! jq -e 'type == "array"' "$file" >/dev/null 2>&1; then + echo "Error: Invalid JSON layout file (must be an array): $file" >&2 + return 1 + fi + + local entry_count=$(jq 'length' "$file") + if [ "$entry_count" -eq 0 ]; then + echo "Error: No storage entries found in: $file" >&2 + return 1 + fi + + # Check that all entries have required fields + local missing=$(jq '[.[] | select(.label == null or .slot == null or .offset == null or .type == null)] | length' "$file") + if [ "$missing" -gt 0 ]; then + echo "Error: Some entries in $file are missing required fields (label, slot, offset, type)" >&2 + return 1 + fi + + # Check for duplicate slot+offset combinations + local dupes=$(jq '[group_by(.slot + ":" + (.offset | tostring)) | .[] | select(length > 1)] | length' "$file") + if [ "$dupes" -gt 0 ]; then + echo "Error: Duplicate slot/offset combinations detected in $file" >&2 + return 1 + fi + + return 0 +} + +# Function to compare two layouts and detect destructive changes +compare_layouts() { + local base_file="$1" + local new_file="$2" + local errors=0 + + # Find the highest slot number currently in use in the base branch + local max_base_slot=$(jq '[.[].slot | tonumber] | max // -1' "$base_file") + + local base_count=$(jq 'length' "$base_file") + local new_count=$(jq 'length' "$new_file") + + echo "Comparing storage layouts..." + echo " Base: $base_file ($base_count entries, max slot: $max_base_slot)" + echo " New: $new_file ($new_count entries)" + + # Check 1: No existing entries removed or modified (type/slot/offset) + while IFS= read -r entry; do + # Extract fields from the base entry + local label=$(echo "$entry" | jq -r '.label') + local slot=$(echo "$entry" | jq -r '.slot') + local offset=$(echo "$entry" | jq -r '.offset') + local type=$(echo "$entry" | jq -cS '.type') + + # Try to find an entry with the exact same name (label) in the new file + local new_entry=$(jq -c --arg l "$label" '.[] | select(.label == $l)' "$new_file") + + if [ -z "$new_entry" ]; then + echo " DESTRUCTIVE: Variable '$label' (slot $slot, offset $offset) was removed" >&2 + errors=$((errors + 1)) + continue + fi + + # Extract fields from the new entry + local new_slot=$(echo "$new_entry" | jq -r '.slot') + local new_offset=$(echo "$new_entry" | jq -r '.offset') + local new_type=$(echo "$new_entry" | jq -cS '.type') + + # Compare fields + if [ "$slot" != "$new_slot" ]; then + echo " DESTRUCTIVE: Variable '$label' slot changed from $slot to $new_slot" >&2 + errors=$((errors + 1)) + fi + + if [ "$offset" != "$new_offset" ]; then + echo " DESTRUCTIVE: Variable '$label' offset changed from $offset to $new_offset (slot $slot)" >&2 + errors=$((errors + 1)) + fi + + if [ "$type" != "$new_type" ]; then + echo " DESTRUCTIVE: Variable '$label' type changed from '$type' to '$new_type' (slot $slot)" >&2 + errors=$((errors + 1)) + fi + done < <(jq -c '.[]' "$base_file") + + # Check 2: New entries must be appended (slot numbers > max_base_slot) + while IFS= read -r entry; do + local label=$(echo "$entry" | jq -r '.label') + local slot=$(echo "$entry" | jq -r '.slot') + local offset=$(echo "$entry" | jq -r '.offset') + local type_label=$(echo "$entry" | jq -r '.type.label // .type') + + # Check if this is a newly added variable + local base_match=$(jq -c --arg l "$label" '.[] | select(.label == $l)' "$base_file") + + if [ -z "$base_match" ]; then + if [ "$slot" -le "$max_base_slot" ]; then + echo " DESTRUCTIVE: New variable '$label' inserted at slot $slot (must be > $max_base_slot)" >&2 + errors=$((errors + 1)) + else + echo " Added: '$label' at slot $slot (offset $offset, type $type_label)" + fi + fi + done < <(jq -c '.[]' "$new_file") + + # Report results + local added=$((new_count - base_count)) + echo "" + if [ "$errors" -eq 0 ]; then + echo "Storage layout check passed" + echo " Entries: ${base_count} → ${new_count} (+${added} added)" + return 0 + else + echo "Storage layout check FAILED (${errors} destructive change(s) detected)" >&2 + return 1 + fi +} + +case $# in + 0) + # No arguments: compare base history to working tree JSON + if [ ! -f "$LAYOUT_JSON" ]; then + echo "Error: Layout API JSON not found locally: $LAYOUT_JSON" >&2 + exit 1 + fi + + IS_CI=${GITHUB_ACTIONS:-false} + + # Get the base commit (HEAD for regular check, or base branch for PRs) + if [ -n "${GITHUB_BASE_REF:-}" ]; then + BASE_REF="origin/$GITHUB_BASE_REF" + elif git rev-parse --quiet --verify HEAD~1 >/dev/null 2>&1; then + BASE_REF="HEAD~1" + else + if [ "$IS_CI" = "true" ]; then + echo "Error: Running in CI but neither GITHUB_BASE_REF nor HEAD~1 could be resolved." >&2 + exit 1 + fi + echo "Warning: No base commit found, assuming initial repository commit" + BASE_REF="" + fi + + if [ -z "$BASE_REF" ]; then + # Genuine initial commit without base ref + echo "Initial commit detected, validating format only..." + if validate_layout_json "$LAYOUT_JSON"; then + echo "Storage layout format validated" + exit 0 + else + exit 1 + fi + fi + + # Ensure base ref actually exists in our git tree + if ! git rev-parse --quiet --verify "$BASE_REF" >/dev/null 2>&1; then + if [ "$IS_CI" = "true" ]; then + echo "Error: CI base ref '$BASE_REF' could not be resolved! Please ensure fetch-depth: 0 is set in the workflow." >&2 + exit 1 + else + echo "Error: Base ref '$BASE_REF' could not be resolved." >&2 + exit 1 + fi + fi + + # Get base version (must use repository-root relative path for git show) + GIT_PREFIX=$(git rev-parse --show-prefix) + FULL_LAYOUT_JSON="${GIT_PREFIX}${LAYOUT_JSON}" + + # Check if the file ACTUALLY exists in the base branch tree + if git cat-file -e "$BASE_REF:$FULL_LAYOUT_JSON" 2>/dev/null; then + TEMP_BASE_LAYOUT=$(mktemp) + TEMP_FILES+=("$TEMP_BASE_LAYOUT") + + if ! git show "$BASE_REF:$FULL_LAYOUT_JSON" > "$TEMP_BASE_LAYOUT" 2>/dev/null; then + echo "Error: Layout file exists in base branch ($BASE_REF) but could not be retrieved via git show." >&2 + exit 1 + fi + else + # The file truly doesn't exist in the base ref + echo "Initial layout detected (file does not exist in $BASE_REF), validating format only..." + if validate_layout_json "$LAYOUT_JSON"; then + echo "Storage layout format validated" + exit 0 + else + echo "Error: New layout validation failed" >&2 + exit 1 + fi + fi + + # Validate both layouts before comparison + if ! validate_layout_json "$TEMP_BASE_LAYOUT"; then + echo "Error: Base layout validation failed on file $TEMP_BASE_LAYOUT" >&2 + exit 1 + fi + if ! validate_layout_json "$LAYOUT_JSON"; then + echo "Error: New layout validation failed" >&2 + exit 1 + fi + + compare_layouts "$TEMP_BASE_LAYOUT" "$LAYOUT_JSON" + ;; + + 2) + # Two arguments: compare base JSON to new JSON explicitly + if ! validate_layout_json "$1"; then exit 1; fi + if ! validate_layout_json "$2"; then exit 1; fi + compare_layouts "$1" "$2" + ;; + + *) + echo "Usage: $0 [ ]" >&2 + echo "" + echo " With no args: Compares base branch to working tree" >&2 + echo " With two args: Compares base_layout.json to new_layout.json" >&2 + exit 1 + ;; +esac diff --git a/tools/generate_storage_layout.sh b/tools/generate_storage_layout.sh new file mode 100755 index 0000000..fb9f5c7 --- /dev/null +++ b/tools/generate_storage_layout.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo // SPDX-License-Identifier: Apache-2.0 OR MIT +echo pragma solidity ^0.8.20\; +echo +echo // Code generated - DO NOT EDIT. +echo // This file is a generated binding and any changes will be lost. +echo // Generated with tools/generate_storage_layout.sh +echo + +forge inspect --json $1 storageLayout \ + | jq -rM '.storage | map("bytes32 constant " + ( + .label + | [scan("[A-Z]+(?=[A-Z][a-z]|$)|[A-Z]?[a-z0-9]+")] + | map(ascii_upcase) + | join("_") + ) + "_SLOT = bytes32(uint256(" + .slot + "));") | join("\n")' \ No newline at end of file