From 7d3effdc65b0b73b61f879173351147c31c0733b Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Tue, 5 Aug 2025 15:14:34 +1000 Subject: [PATCH 1/3] feat: add `validate` command to validate configuration files refactor(config): add `Validate` method to `Config` and `CoordinatorConfig` structs refactor(test): add `ValidateTestConfig` function to validate test configurations --- cmd/validate.go | 65 ++++++++++++++++++++ pkg/coordinator/config.go | 115 +++++++++++++++++++++++++++++++++++ pkg/coordinator/test/test.go | 58 ++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 cmd/validate.go diff --git a/cmd/validate.go b/cmd/validate.go new file mode 100644 index 00000000..0a0b4bd2 --- /dev/null +++ b/cmd/validate.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "context" + "os" + + "github.com/ethpandaops/assertoor/pkg/coordinator" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate assertoor configuration", + Long: `Validates the assertoor configuration file for syntax and semantic correctness`, + Args: cobra.NoArgs, + Run: func(_ *cobra.Command, _ []string) { + // Set up minimal logging for error output + logrus.SetLevel(logrus.ErrorLevel) + if verbose { + logrus.SetLevel(logrus.DebugLevel) + } + + // Check if config file is specified + if cfgFile == "" { + logrus.Error("no configuration file specified") + os.Exit(1) + } + + // Load configuration + config, err := coordinator.NewConfig(cfgFile) + if err != nil { + logrus.WithError(err).Error("failed to load configuration") + os.Exit(1) + } + + // Validate configuration + if err := config.Validate(); err != nil { + logrus.WithError(err).Error("configuration validation failed") + os.Exit(1) + } + + // Create a minimal coordinator instance just for test loading validation + // We don't need to run the full coordinator, just validate external tests can be loaded + coord := coordinator.NewCoordinator(config, logrus.StandardLogger(), metricsPort) + testRegistry := coordinator.NewTestRegistry(coord) + + // Validate external tests can be loaded + ctx := context.Background() + for _, extTest := range config.ExternalTests { + _, err := testRegistry.AddExternalTest(ctx, extTest) + if err != nil { + logrus.WithError(err).WithField("test", extTest.File).Error("failed to load external test") + os.Exit(1) + } + } + + // Success - exit cleanly with no output + os.Exit(0) + }, +} + +func init() { + rootCmd.AddCommand(validateCmd) +} diff --git a/pkg/coordinator/config.go b/pkg/coordinator/config.go index a216f85c..ad3b3e4b 100644 --- a/pkg/coordinator/config.go +++ b/pkg/coordinator/config.go @@ -1,12 +1,17 @@ package coordinator import ( + "fmt" + "net/url" "os" + "strconv" + "strings" "github.com/ethpandaops/assertoor/pkg/coordinator/clients" "github.com/ethpandaops/assertoor/pkg/coordinator/db" "github.com/ethpandaops/assertoor/pkg/coordinator/helper" "github.com/ethpandaops/assertoor/pkg/coordinator/names" + "github.com/ethpandaops/assertoor/pkg/coordinator/test" "github.com/ethpandaops/assertoor/pkg/coordinator/types" web_types "github.com/ethpandaops/assertoor/pkg/coordinator/web/types" "gopkg.in/yaml.v3" @@ -82,3 +87,113 @@ func NewConfig(path string) (*Config, error) { return config, nil } + +func (c *Config) Validate() error { + var errs []error + + // Validate database config + if c.Database != nil { + if c.Database.Engine != "" && c.Database.Engine != "sqlite" && c.Database.Engine != "postgres" { + errs = append(errs, fmt.Errorf("invalid database engine: %s", c.Database.Engine)) + } + } + + // Validate endpoints + for i, endpoint := range c.Endpoints { + if endpoint.Name == "" { + errs = append(errs, fmt.Errorf("endpoint[%d]: name cannot be empty", i)) + } + + if endpoint.ConsensusURL == "" && endpoint.ExecutionURL == "" { + errs = append(errs, fmt.Errorf("endpoint[%d] '%s': must have at least one URL", i, endpoint.Name)) + } + // Validate URLs are parseable + if endpoint.ConsensusURL != "" { + if _, err := url.Parse(endpoint.ConsensusURL); err != nil { + errs = append(errs, fmt.Errorf("endpoint[%d] '%s': invalid consensus URL: %v", i, endpoint.Name, err)) + } + } + + if endpoint.ExecutionURL != "" { + if _, err := url.Parse(endpoint.ExecutionURL); err != nil { + errs = append(errs, fmt.Errorf("endpoint[%d] '%s': invalid execution URL: %v", i, endpoint.Name, err)) + } + } + } + + // Validate web config + if c.Web != nil { + if c.Web.Frontend != nil && c.Web.Frontend.Enabled { + // Validate port is in valid range + if c.Web.Server.Port != "" { + if port, err := strconv.Atoi(c.Web.Server.Port); err != nil { + errs = append(errs, fmt.Errorf("invalid web server port: %s (must be a number)", c.Web.Server.Port)) + } else if port < 1 || port > 65535 { + errs = append(errs, fmt.Errorf("invalid web server port: %d (must be between 1 and 65535)", port)) + } + } + } + } + + // Validate coordinator config + if c.Coordinator != nil { + if err := c.Coordinator.Validate(); err != nil { + errs = append(errs, fmt.Errorf("coordinator config: %v", err)) + } + } + + // Validate tests + for i, testCfg := range c.Tests { + if testCfg.ID == "" { + errs = append(errs, fmt.Errorf("test[%d]: ID cannot be empty", i)) + } + + if testCfg.Name == "" { + errs = append(errs, fmt.Errorf("test[%d] '%s': name cannot be empty", i, testCfg.ID)) + } + + // Validate task configurations + if err := test.ValidateTestConfig(testCfg); err != nil { + errs = append(errs, fmt.Errorf("test[%d] '%s': %v", i, testCfg.ID, err)) + } + } + + // Validate external tests + for i, extTest := range c.ExternalTests { + if extTest.File == "" { + errs = append(errs, fmt.Errorf("external test[%d]: file cannot be empty", i)) + } + + if extTest.ID == "" { + errs = append(errs, fmt.Errorf("external test[%d]: ID cannot be empty", i)) + } + } + + if len(errs) > 0 { + return fmt.Errorf("configuration validation failed:\n%s", formatErrors(errs)) + } + + return nil +} + +func formatErrors(errs []error) string { + var buf strings.Builder + for _, err := range errs { + buf.WriteString(" - ") + buf.WriteString(err.Error()) + buf.WriteString("\n") + } + + return strings.TrimSuffix(buf.String(), "\n") +} + +func (c *CoordinatorConfig) Validate() error { + if c.TestRetentionTime.Duration != 0 { + // Duration is valid if it parsed successfully + if c.TestRetentionTime.Duration < 0 { + return fmt.Errorf("testRetentionTime cannot be negative") + } + } + + return nil +} diff --git a/pkg/coordinator/test/test.go b/pkg/coordinator/test/test.go index 41d59b86..5be1acbd 100644 --- a/pkg/coordinator/test/test.go +++ b/pkg/coordinator/test/test.go @@ -6,7 +6,9 @@ import ( "time" "github.com/ethpandaops/assertoor/pkg/coordinator/db" + "github.com/ethpandaops/assertoor/pkg/coordinator/logger" "github.com/ethpandaops/assertoor/pkg/coordinator/scheduler" + "github.com/ethpandaops/assertoor/pkg/coordinator/tasks" "github.com/ethpandaops/assertoor/pkg/coordinator/types" "github.com/ethpandaops/assertoor/pkg/coordinator/vars" "github.com/jmoiron/sqlx" @@ -252,3 +254,59 @@ func (t *Test) GetTaskScheduler() types.TaskScheduler { func (t *Test) GetTestVariables() types.Variables { return t.variables } + +// ValidateTestConfig validates a test configuration including its task configurations +func ValidateTestConfig(config *types.TestConfig) error { + if len(config.Tasks) == 0 { + return fmt.Errorf("test must have at least one task") + } + + // Import the tasks package to get access to GetTaskDescriptor + tasksRegistry := tasks.GetTaskDescriptor + + // Validate each task configuration + for i, rawTask := range config.Tasks { + // Parse the raw task message into TaskOptions + var taskOptions types.TaskOptions + if err := rawTask.Unmarshal(&taskOptions); err != nil { + return fmt.Errorf("task[%d]: failed to parse task configuration: %v", i, err) + } + + // Check if task type exists + taskDescriptor := tasksRegistry(taskOptions.Name) + if taskDescriptor == nil { + return fmt.Errorf("task[%d]: unknown task type '%s'", i, taskOptions.Name) + } + + // For validation purposes, we only need to check if the config can be loaded + // We don't need to create the actual task instance since that requires runtime context + // Instead, we'll create a minimal task instance just to validate the config structure + + // Check if task has a config + if taskOptions.Config == nil { + // Some tasks don't require config, that's ok + continue + } + + // Create a temporary task context for validation + tmpCtx := &types.TaskContext{ + Logger: logger.NewLogger(&logger.ScopeOptions{ + Parent: logrus.StandardLogger(), + }), + Vars: vars.NewVariables(nil), // Empty variables for validation + } + + // Try to create the task with minimal context + taskInst, err := taskDescriptor.NewTask(tmpCtx, &taskOptions) + if err != nil { + return fmt.Errorf("task[%d] '%s': failed to create task instance: %v", i, taskOptions.Name, err) + } + + // Load and validate the task configuration + if err := taskInst.LoadConfig(); err != nil { + return fmt.Errorf("task[%d] '%s': %v", i, taskOptions.Name, err) + } + } + + return nil +} From 584d74235c048aeb6f00cb1fbde2f9550cdf0d98 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Tue, 5 Aug 2025 15:20:27 +1000 Subject: [PATCH 2/3] refactor(cmd/validate): simplify external test validation by checking file existence refactor(pkg/coordinator): update GlobalVars map value type from interface{} to any --- cmd/validate.go | 19 +++++++------------ pkg/coordinator/config.go | 4 ++-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/cmd/validate.go b/cmd/validate.go index 0a0b4bd2..10859c50 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "os" "github.com/ethpandaops/assertoor/pkg/coordinator" @@ -40,18 +39,14 @@ var validateCmd = &cobra.Command{ os.Exit(1) } - // Create a minimal coordinator instance just for test loading validation - // We don't need to run the full coordinator, just validate external tests can be loaded - coord := coordinator.NewCoordinator(config, logrus.StandardLogger(), metricsPort) - testRegistry := coordinator.NewTestRegistry(coord) - - // Validate external tests can be loaded - ctx := context.Background() + // Validate external test files exist and can be parsed for _, extTest := range config.ExternalTests { - _, err := testRegistry.AddExternalTest(ctx, extTest) - if err != nil { - logrus.WithError(err).WithField("test", extTest.File).Error("failed to load external test") - os.Exit(1) + if extTest.File != "" { + // Check if external test file exists + if _, err := os.Stat(extTest.File); os.IsNotExist(err) { + logrus.WithField("test", extTest.File).Error("external test file does not exist") + os.Exit(1) + } } } diff --git a/pkg/coordinator/config.go b/pkg/coordinator/config.go index ad3b3e4b..27d7d1a0 100644 --- a/pkg/coordinator/config.go +++ b/pkg/coordinator/config.go @@ -31,7 +31,7 @@ type Config struct { ValidatorNames *names.Config `yaml:"validatorNames" json:"validatorNames"` // Global variables - GlobalVars map[string]interface{} `yaml:"globalVars" json:"globalVars"` + GlobalVars map[string]any `yaml:"globalVars" json:"globalVars"` // Coordinator config Coordinator *CoordinatorConfig `yaml:"coordinator" json:"coordinator"` @@ -62,7 +62,7 @@ func DefaultConfig() *Config { ConsensusURL: "http://localhost:5052", }, }, - GlobalVars: make(map[string]interface{}), + GlobalVars: make(map[string]any), Coordinator: &CoordinatorConfig{}, Tests: []*types.TestConfig{}, ExternalTests: []*types.ExternalTestConfig{}, From 896e71b9b84e32381b91f097a7425247b9373a4e Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Tue, 5 Aug 2025 15:25:40 +1000 Subject: [PATCH 3/3] refactor(test): rename logger parameter to log for clarity and consistency --- pkg/coordinator/test/test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/coordinator/test/test.go b/pkg/coordinator/test/test.go index 5be1acbd..d5956142 100644 --- a/pkg/coordinator/test/test.go +++ b/pkg/coordinator/test/test.go @@ -33,11 +33,11 @@ type Test struct { timeout time.Duration } -func CreateTest(runID uint64, descriptor types.TestDescriptor, logger logrus.FieldLogger, services types.TaskServices, configOverrides map[string]any) (types.TestRunner, error) { +func CreateTest(runID uint64, descriptor types.TestDescriptor, log logrus.FieldLogger, services types.TaskServices, configOverrides map[string]any) (types.TestRunner, error) { test := &Test{ runID: runID, services: services, - logger: logger.WithField("RunID", runID).WithField("TestID", descriptor.ID()), + logger: log.WithField("RunID", runID).WithField("TestID", descriptor.ID()), descriptor: descriptor, config: descriptor.Config(), status: types.TestStatusPending,