allow relative paths in run_external_tasks, resolving against parent base path/URL

pkg/coordinator/tasks/run_external_tasks/task.go

@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -75,15 +77,22 @@ func (t *Task) LoadConfig() error {
 }
 
 func (t *Task) Execute(ctx context.Context) error {
+	// get task base path
+	taskBasePath, ok := t.ctx.Vars.GetVar("taskBasePath").(string)
+	if !ok {
+		t.logger.Warn("could not read taskBasePath from variables")
+	}
+
 	// load test yaml file
-	testConfig, err := t.loadTestConfig(ctx, t.config.TestFile)
+	testConfig, testBasePath, err := t.loadTestConfig(ctx, taskBasePath, t.config.TestFile)
 	if err != nil {
 		return err
 	}
 
 	// create new variable scope for test configuration
 	testVars := t.ctx.Vars.NewScope()
 	testVars.SetVar("scopeOwner", uint64(t.ctx.Index))
+	testVars.SetVar("taskBasePath", testBasePath)
 	t.ctx.Outputs.SetSubScope("childScope", vars.NewScopeFilter(testVars))
 
 	// add default config from external test to variable scope
@@ -187,20 +196,34 @@ taskLoop:
 	return resError
 }
 
-func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.TestConfig, error) {
+func (t *Task) loadTestConfig(ctx context.Context, basePath, testFile string) (*types.TestConfig, string, error) {
 	var reader io.Reader
 
+	var testBasePath string
+
 	if strings.HasPrefix(testFile, "http://") || strings.HasPrefix(testFile, "https://") {
+		parsedURL, err := url.Parse(testFile)
+		if err != nil {
+			return nil, "", err
+		}
+
+		// Remove the filename from the path
+		parsedURL.Path = path.Dir(parsedURL.Path)
+		parsedURL.RawQuery = ""
+		parsedURL.Fragment = ""
+
+		testBasePath = parsedURL.String()
+
 		client := &http.Client{Timeout: time.Second * 120}
 
 		req, err := http.NewRequestWithContext(ctx, "GET", testFile, http.NoBody)
 		if err != nil {
-			return nil, err
+			return nil, testBasePath, err
 		}
 
 		resp, err := client.Do(req)
 		if err != nil {
-			return nil, err
+			return nil, testBasePath, err
 		}
 
 		defer func() {
@@ -210,14 +233,20 @@ func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.Test
 		}()
 
 		if resp.StatusCode != http.StatusOK {
-			return nil, fmt.Errorf("error loading test config from url: %v, result: %v %v", testFile, resp.StatusCode, resp.Status)
+			return nil, testBasePath, fmt.Errorf("error loading test config from url: %v, result: %v %v", testFile, resp.StatusCode, resp.Status)
 		}
 
 		reader = resp.Body
 	} else {
+		if !path.IsAbs(testFile) && basePath != "" {
+			testFile = path.Join(basePath, testFile)
+		}
+
+		testBasePath = path.Dir(testFile)
+
 		f, err := os.Open(testFile)
 		if err != nil {
-			return nil, fmt.Errorf("error loading test config from file %v: %w", testFile, err)
+			return nil, testBasePath, fmt.Errorf("error loading test config from file %v: %w", testFile, err)
 		}
 
 		defer func() {
@@ -234,7 +263,7 @@ func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.Test
 
 	err := decoder.Decode(testConfig)
 	if err != nil {
-		return nil, fmt.Errorf("error decoding external test config %v: %v", testFile, err)
+		return nil, testBasePath, fmt.Errorf("error decoding external test config %v: %v", testFile, err)
 	}
 
 	if testConfig.Config == nil {
@@ -245,5 +274,5 @@ func (t *Task) loadTestConfig(ctx context.Context, testFile string) (*types.Test
 		testConfig.ConfigVars = map[string]string{}
 	}
 
-	return testConfig, nil
+	return testConfig, testBasePath, nil
 }

pkg/coordinator/test/descriptor.go

@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -15,25 +17,32 @@ import (
 )
 
 type Descriptor struct {
-	id     string
-	source string
-	config *types.TestConfig
-	vars   types.Variables
-	err    error
+	id       string
+	source   string
+	basePath string
+	config   *types.TestConfig
+	vars     types.Variables
+	err      error
 }
 
-func NewDescriptor(testID, testSrc string, config *types.TestConfig, variables types.Variables) *Descriptor {
+func NewDescriptor(testID, testSrc, basePath string, config *types.TestConfig, variables types.Variables) *Descriptor {
 	return &Descriptor{
-		id:     testID,
-		source: testSrc,
-		config: config,
-		vars:   variables,
+		id:       testID,
+		source:   testSrc,
+		basePath: basePath,
+		config:   config,
+		vars:     variables,
 	}
 }
 
 func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localTests []*types.TestConfig, externalTests []*types.ExternalTestConfig) []types.TestDescriptor {
 	descriptors := []types.TestDescriptor{}
 
+	workingDir, err := os.Getwd()
+	if err != nil {
+		logrus.WithError(err).Warn("failed to get working directory")
+	}
+
 	// load local tests
 	for testIdx, testCfg := range localTests {
 		testID := testCfg.ID
@@ -52,11 +61,12 @@ func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localT
 		err := testVars.CopyVars(globalVars, testCfg.ConfigVars)
 
 		descriptors = append(descriptors, &Descriptor{
-			id:     testID,
-			source: testSrc,
-			vars:   testVars,
-			config: localTests[testIdx],
-			err:    err,
+			id:       testID,
+			source:   testSrc,
+			basePath: workingDir,
+			vars:     testVars,
+			config:   localTests[testIdx],
+			err:      err,
 		})
 	}
 
@@ -65,7 +75,7 @@ func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localT
 		testSrc := fmt.Sprintf("external:%v", extTestCfg.File)
 		testID := ""
 
-		testConfig, testVars, err := LoadExternalTestConfig(ctx, globalVars, extTestCfg)
+		testConfig, testVars, basePath, err := LoadExternalTestConfig(ctx, globalVars, extTestCfg)
 
 		if testConfig != nil && testConfig.ID != "" {
 			testID = testConfig.ID
@@ -76,31 +86,46 @@ func LoadTestDescriptors(ctx context.Context, globalVars types.Variables, localT
 		}
 
 		descriptors = append(descriptors, &Descriptor{
-			id:     testID,
-			source: testSrc,
-			config: testConfig,
-			vars:   testVars,
-			err:    err,
+			id:       testID,
+			source:   testSrc,
+			basePath: basePath,
+			config:   testConfig,
+			vars:     testVars,
+			err:      err,
 		})
 	}
 
 	return descriptors
 }
 
-func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, extTestCfg *types.ExternalTestConfig) (*types.TestConfig, types.Variables, error) {
+func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, extTestCfg *types.ExternalTestConfig) (*types.TestConfig, types.Variables, string, error) {
 	var reader io.Reader
 
+	var basePath string
+
 	if strings.HasPrefix(extTestCfg.File, "http://") || strings.HasPrefix(extTestCfg.File, "https://") {
+		parsedURL, err := url.Parse(extTestCfg.File)
+		if err != nil {
+			return nil, nil, "", err
+		}
+
+		// Remove the filename from the path
+		parsedURL.Path = path.Dir(parsedURL.Path)
+		parsedURL.RawQuery = ""
+		parsedURL.Fragment = ""
+
+		basePath = parsedURL.String()
+
 		client := &http.Client{Timeout: time.Second * 120}
 
 		req, err := http.NewRequestWithContext(ctx, "GET", extTestCfg.File, http.NoBody)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, basePath, err
 		}
 
 		resp, err := client.Do(req)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, basePath, err
 		}
 
 		defer func() {
@@ -110,14 +135,16 @@ func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, ext
 		}()
 
 		if resp.StatusCode != http.StatusOK {
-			return nil, nil, fmt.Errorf("error loading test config from url: %v, result: %v %v", extTestCfg.File, resp.StatusCode, resp.Status)
+			return nil, nil, basePath, fmt.Errorf("error loading test config from url: %v, result: %v %v", extTestCfg.File, resp.StatusCode, resp.Status)
 		}
 
 		reader = resp.Body
 	} else {
+		basePath = path.Dir(extTestCfg.File)
+
 		f, err := os.Open(extTestCfg.File)
 		if err != nil {
-			return nil, nil, fmt.Errorf("error loading test config from file %v: %w", extTestCfg.File, err)
+			return nil, nil, basePath, fmt.Errorf("error loading test config from file %v: %w", extTestCfg.File, err)
 		}
 
 		defer func() {
@@ -135,7 +162,7 @@ func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, ext
 
 	err := decoder.Decode(testConfig)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error decoding external test config %v: %v", extTestCfg.File, err)
+		return nil, nil, basePath, fmt.Errorf("error decoding external test config %v: %v", extTestCfg.File, err)
 	}
 
 	if testConfig.Config == nil {
@@ -177,10 +204,10 @@ func LoadExternalTestConfig(ctx context.Context, globalVars types.Variables, ext
 
 	err = testVars.CopyVars(testVars, testConfig.ConfigVars)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error decoding external test configVars %v: %v", extTestCfg.File, err)
+		return nil, nil, basePath, fmt.Errorf("error decoding external test configVars %v: %v", extTestCfg.File, err)
 	}
 
-	return testConfig, testVars, nil
+	return testConfig, testVars, basePath, nil
 }
 
 func (d *Descriptor) ID() string {
@@ -191,6 +218,10 @@ func (d *Descriptor) Source() string {
 	return d.source
 }
 
+func (d *Descriptor) BasePath() string {
+	return d.basePath
+}
+
 func (d *Descriptor) Config() *types.TestConfig {
 	return d.config
 }

pkg/coordinator/test/test.go

@@ -52,6 +52,10 @@ func CreateTest(runID uint64, descriptor types.TestDescriptor, log logrus.FieldL
 		test.variables.SetVar(cfgKey, cfgValue)
 	}
 
+	// set base path
+	test.variables.SetVar("testBasePath", descriptor.BasePath())
+	test.variables.SetVar("taskBasePath", descriptor.BasePath())
+
 	// add test run to database
 	configYaml, err := yaml.Marshal(test.variables.GetVarsMap(nil, false))
 	if err != nil {

pkg/coordinator/testregistry.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"sort"
 	"sync"
 	"time"
@@ -205,7 +206,12 @@ func (c *TestRegistry) AddLocalTest(testConfig *types.TestConfig) (types.TestDes
 		return nil, fmt.Errorf("failed decoding configVars: %v", err)
 	}
 
-	testDescriptor := test.NewDescriptor(testConfig.ID, "api-call", testConfig, testVars)
+	workingDir, err := os.Getwd()
+	if err != nil {
+		return nil, fmt.Errorf("failed getting working directory: %v", err)
+	}
+
+	testDescriptor := test.NewDescriptor(testConfig.ID, "api-call", workingDir, testConfig, testVars)
 
 	c.testDescriptorsMutex.Lock()
 	defer c.testDescriptorsMutex.Unlock()
@@ -225,7 +231,7 @@ func (c *TestRegistry) AddLocalTest(testConfig *types.TestConfig) (types.TestDes
 }
 
 func (c *TestRegistry) AddExternalTest(ctx context.Context, extTestCfg *types.ExternalTestConfig) (types.TestDescriptor, error) {
-	testConfig, testVars, err := test.LoadExternalTestConfig(ctx, c.coordinator.GlobalVariables(), extTestCfg)
+	testConfig, testVars, basePath, err := test.LoadExternalTestConfig(ctx, c.coordinator.GlobalVariables(), extTestCfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed loading test config from %v: %w", extTestCfg.File, err)
 	}
@@ -242,7 +248,7 @@ func (c *TestRegistry) AddExternalTest(ctx context.Context, extTestCfg *types.Ex
 		return nil, errors.New("test must have 1 or more tasks")
 	}
 
-	testDescriptor := test.NewDescriptor(testConfig.ID, fmt.Sprintf("external:%v", extTestCfg.File), testConfig, testVars)
+	testDescriptor := test.NewDescriptor(testConfig.ID, fmt.Sprintf("external:%v", extTestCfg.File), basePath, testConfig, testVars)
 	extTestCfg.ID = testDescriptor.ID()
 	extTestCfg.Name = testConfig.Name
 

pkg/coordinator/types/test.go

@@ -69,6 +69,7 @@ type TestSchedule struct {
 type TestDescriptor interface {
 	ID() string
 	Source() string
+	BasePath() string
 	Config() *TestConfig
 	Vars() Variables
 	Err() error

pkg/coordinator/web/api/get_test_api.go

@@ -10,6 +10,7 @@ import (
 type GetTestResponse struct {
 	ID         string              `json:"id"`
 	Source     string              `json:"source"`
+	BasePath   string              `json:"basePath"`
 	Name       string              `json:"name"`
 	Timeout    uint64              `json:"timeout"`
 	Config     map[string]any      `json:"config"`
@@ -60,6 +61,7 @@ func (ah *APIHandler) GetTest(w http.ResponseWriter, r *http.Request) {
 	ah.sendOKResponse(w, r.URL.String(), &GetTestResponse{
 		ID:         testDescriptor.ID(),
 		Source:     testDescriptor.Source(),
+		BasePath:   testDescriptor.BasePath(),
 		Name:       testConfig.Name,
 		Timeout:    uint64(testConfig.Timeout.Seconds()),
 		Config:     testConfig.Config,