feat: add validate command to validate configuration files

cmd/validate.go

@@ -0,0 +1,60 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/ethpandaops/assertoor/pkg/coordinator"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var validateCmd = &cobra.Command{
+	Use:   "validate",
+	Short: "Validate assertoor configuration",
+	Long:  `Validates the assertoor configuration file for syntax and semantic correctness`,
+	Args:  cobra.NoArgs,
+	Run: func(_ *cobra.Command, _ []string) {
+		// Set up minimal logging for error output
+		logrus.SetLevel(logrus.ErrorLevel)
+		if verbose {
+			logrus.SetLevel(logrus.DebugLevel)
+		}
+
+		// Check if config file is specified
+		if cfgFile == "" {
+			logrus.Error("no configuration file specified")
+			os.Exit(1)
+		}
+
+		// Load configuration
+		config, err := coordinator.NewConfig(cfgFile)
+		if err != nil {
+			logrus.WithError(err).Error("failed to load configuration")
+			os.Exit(1)
+		}
+
+		// Validate configuration
+		if err := config.Validate(); err != nil {
+			logrus.WithError(err).Error("configuration validation failed")
+			os.Exit(1)
+		}
+
+		// Validate external test files exist and can be parsed
+		for _, extTest := range config.ExternalTests {
+			if extTest.File != "" {
+				// Check if external test file exists
+				if _, err := os.Stat(extTest.File); os.IsNotExist(err) {
+					logrus.WithField("test", extTest.File).Error("external test file does not exist")
+					os.Exit(1)
+				}
+			}
+		}
+
+		// Success - exit cleanly with no output
+		os.Exit(0)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(validateCmd)
+}

pkg/coordinator/config.go

@@ -1,12 +1,17 @@
 package coordinator
 
 import (
+	"fmt"
+	"net/url"
 	"os"
+	"strconv"
+	"strings"
 
 	"github.com/ethpandaops/assertoor/pkg/coordinator/clients"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/db"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/helper"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/names"
+	"github.com/ethpandaops/assertoor/pkg/coordinator/test"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/types"
 	web_types "github.com/ethpandaops/assertoor/pkg/coordinator/web/types"
 	"gopkg.in/yaml.v3"
@@ -26,7 +31,7 @@ type Config struct {
 	ValidatorNames *names.Config `yaml:"validatorNames" json:"validatorNames"`
 
 	// Global variables
-	GlobalVars map[string]interface{} `yaml:"globalVars" json:"globalVars"`
+	GlobalVars map[string]any `yaml:"globalVars" json:"globalVars"`
 
 	// Coordinator config
 	Coordinator *CoordinatorConfig `yaml:"coordinator" json:"coordinator"`
@@ -57,7 +62,7 @@ func DefaultConfig() *Config {
 				ConsensusURL: "http://localhost:5052",
 			},
 		},
-		GlobalVars:    make(map[string]interface{}),
+		GlobalVars:    make(map[string]any),
 		Coordinator:   &CoordinatorConfig{},
 		Tests:         []*types.TestConfig{},
 		ExternalTests: []*types.ExternalTestConfig{},
@@ -82,3 +87,113 @@ func NewConfig(path string) (*Config, error) {
 
 	return config, nil
 }
+
+func (c *Config) Validate() error {
+	var errs []error
+
+	// Validate database config
+	if c.Database != nil {
+		if c.Database.Engine != "" && c.Database.Engine != "sqlite" && c.Database.Engine != "postgres" {
+			errs = append(errs, fmt.Errorf("invalid database engine: %s", c.Database.Engine))
+		}
+	}
+
+	// Validate endpoints
+	for i, endpoint := range c.Endpoints {
+		if endpoint.Name == "" {
+			errs = append(errs, fmt.Errorf("endpoint[%d]: name cannot be empty", i))
+		}
+
+		if endpoint.ConsensusURL == "" && endpoint.ExecutionURL == "" {
+			errs = append(errs, fmt.Errorf("endpoint[%d] '%s': must have at least one URL", i, endpoint.Name))
+		}
+		// Validate URLs are parseable
+		if endpoint.ConsensusURL != "" {
+			if _, err := url.Parse(endpoint.ConsensusURL); err != nil {
+				errs = append(errs, fmt.Errorf("endpoint[%d] '%s': invalid consensus URL: %v", i, endpoint.Name, err))
+			}
+		}
+
+		if endpoint.ExecutionURL != "" {
+			if _, err := url.Parse(endpoint.ExecutionURL); err != nil {
+				errs = append(errs, fmt.Errorf("endpoint[%d] '%s': invalid execution URL: %v", i, endpoint.Name, err))
+			}
+		}
+	}
+
+	// Validate web config
+	if c.Web != nil {
+		if c.Web.Frontend != nil && c.Web.Frontend.Enabled {
+			// Validate port is in valid range
+			if c.Web.Server.Port != "" {
+				if port, err := strconv.Atoi(c.Web.Server.Port); err != nil {
+					errs = append(errs, fmt.Errorf("invalid web server port: %s (must be a number)", c.Web.Server.Port))
+				} else if port < 1 || port > 65535 {
+					errs = append(errs, fmt.Errorf("invalid web server port: %d (must be between 1 and 65535)", port))
+				}
+			}
+		}
+	}
+
+	// Validate coordinator config
+	if c.Coordinator != nil {
+		if err := c.Coordinator.Validate(); err != nil {
+			errs = append(errs, fmt.Errorf("coordinator config: %v", err))
+		}
+	}
+
+	// Validate tests
+	for i, testCfg := range c.Tests {
+		if testCfg.ID == "" {
+			errs = append(errs, fmt.Errorf("test[%d]: ID cannot be empty", i))
+		}
+
+		if testCfg.Name == "" {
+			errs = append(errs, fmt.Errorf("test[%d] '%s': name cannot be empty", i, testCfg.ID))
+		}
+
+		// Validate task configurations
+		if err := test.ValidateTestConfig(testCfg); err != nil {
+			errs = append(errs, fmt.Errorf("test[%d] '%s': %v", i, testCfg.ID, err))
+		}
+	}
+
+	// Validate external tests
+	for i, extTest := range c.ExternalTests {
+		if extTest.File == "" {
+			errs = append(errs, fmt.Errorf("external test[%d]: file cannot be empty", i))
+		}
+
+		if extTest.ID == "" {
+			errs = append(errs, fmt.Errorf("external test[%d]: ID cannot be empty", i))
+		}
+	}
+
+	if len(errs) > 0 {
+		return fmt.Errorf("configuration validation failed:\n%s", formatErrors(errs))
+	}
+
+	return nil
+}
+
+func formatErrors(errs []error) string {
+	var buf strings.Builder
+	for _, err := range errs {
+		buf.WriteString("  - ")
+		buf.WriteString(err.Error())
+		buf.WriteString("\n")
+	}
+
+	return strings.TrimSuffix(buf.String(), "\n")
+}
+
+func (c *CoordinatorConfig) Validate() error {
+	if c.TestRetentionTime.Duration != 0 {
+		// Duration is valid if it parsed successfully
+		if c.TestRetentionTime.Duration < 0 {
+			return fmt.Errorf("testRetentionTime cannot be negative")
+		}
+	}
+
+	return nil
+}

pkg/coordinator/test/test.go

@@ -6,7 +6,9 @@ import (
 	"time"
 
 	"github.com/ethpandaops/assertoor/pkg/coordinator/db"
+	"github.com/ethpandaops/assertoor/pkg/coordinator/logger"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/scheduler"
+	"github.com/ethpandaops/assertoor/pkg/coordinator/tasks"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/types"
 	"github.com/ethpandaops/assertoor/pkg/coordinator/vars"
 	"github.com/jmoiron/sqlx"
@@ -31,11 +33,11 @@ type Test struct {
 	timeout   time.Duration
 }
 
-func CreateTest(runID uint64, descriptor types.TestDescriptor, logger logrus.FieldLogger, services types.TaskServices, configOverrides map[string]any) (types.TestRunner, error) {
+func CreateTest(runID uint64, descriptor types.TestDescriptor, log logrus.FieldLogger, services types.TaskServices, configOverrides map[string]any) (types.TestRunner, error) {
 	test := &Test{
 		runID:      runID,
 		services:   services,
-		logger:     logger.WithField("RunID", runID).WithField("TestID", descriptor.ID()),
+		logger:     log.WithField("RunID", runID).WithField("TestID", descriptor.ID()),
 		descriptor: descriptor,
 		config:     descriptor.Config(),
 		status:     types.TestStatusPending,
@@ -252,3 +254,59 @@ func (t *Test) GetTaskScheduler() types.TaskScheduler {
 func (t *Test) GetTestVariables() types.Variables {
 	return t.variables
 }
+
+// ValidateTestConfig validates a test configuration including its task configurations
+func ValidateTestConfig(config *types.TestConfig) error {
+	if len(config.Tasks) == 0 {
+		return fmt.Errorf("test must have at least one task")
+	}
+
+	// Import the tasks package to get access to GetTaskDescriptor
+	tasksRegistry := tasks.GetTaskDescriptor
+
+	// Validate each task configuration
+	for i, rawTask := range config.Tasks {
+		// Parse the raw task message into TaskOptions
+		var taskOptions types.TaskOptions
+		if err := rawTask.Unmarshal(&taskOptions); err != nil {
+			return fmt.Errorf("task[%d]: failed to parse task configuration: %v", i, err)
+		}
+
+		// Check if task type exists
+		taskDescriptor := tasksRegistry(taskOptions.Name)
+		if taskDescriptor == nil {
+			return fmt.Errorf("task[%d]: unknown task type '%s'", i, taskOptions.Name)
+		}
+
+		// For validation purposes, we only need to check if the config can be loaded
+		// We don't need to create the actual task instance since that requires runtime context
+		// Instead, we'll create a minimal task instance just to validate the config structure
+
+		// Check if task has a config
+		if taskOptions.Config == nil {
+			// Some tasks don't require config, that's ok
+			continue
+		}
+
+		// Create a temporary task context for validation
+		tmpCtx := &types.TaskContext{
+			Logger: logger.NewLogger(&logger.ScopeOptions{
+				Parent: logrus.StandardLogger(),
+			}),
+			Vars: vars.NewVariables(nil), // Empty variables for validation
+		}
+
+		// Try to create the task with minimal context
+		taskInst, err := taskDescriptor.NewTask(tmpCtx, &taskOptions)
+		if err != nil {
+			return fmt.Errorf("task[%d] '%s': failed to create task instance: %v", i, taskOptions.Name, err)
+		}
+
+		// Load and validate the task configuration
+		if err := taskInst.LoadConfig(); err != nil {
+			return fmt.Errorf("task[%d] '%s': %v", i, taskOptions.Name, err)
+		}
+	}
+
+	return nil
+}