wip

Author: patinthehat

lib/app/workflow.go

@@ -48,7 +48,14 @@ type WorkflowSettings struct {
 }
 
 type WorkflowSettingsDomains struct {
-	Allowed []string `yaml:"allowed"`
+	Allowed []string                       `yaml:"allowed"`
+	Hosts   []WorkflowSettingsDomainsHosts `yaml:"hosts"`
+}
+
+type WorkflowSettingsDomainsHosts struct {
+	Hostname string   `yaml:"hostname"`
+	Gateway  *string  `yaml:"gateway"`
+	Headers  []string `yaml:"headers"`
 }
 
 type WorkflowSettingsCache struct {
@@ -180,6 +187,17 @@ func (workflow *StackupWorkflow) configureDefaultSettings() {
 		workflow.Settings.Domains.Allowed = []string{"raw.githubusercontent.com", "api.github.com"}
 	}
 
+	if len(workflow.Settings.Domains.Hosts) > 0 {
+		for _, host := range workflow.Settings.Domains.Hosts {
+			if host.Gateway != nil && *host.Gateway == "allow" {
+				workflow.Settings.Domains.Allowed = append(workflow.Settings.Domains.Allowed, host.Hostname)
+			}
+			if len(host.Headers) > 0 {
+				App.Gateway.SetDomainHeaders(host.Hostname, host.Headers)
+			}
+		}
+	}
+
 	App.Gateway.SetAllowedDomains(workflow.Settings.Domains.Allowed)
 
 	if workflow.Settings.Cache.TtlMinutes <= 0 {
@@ -211,13 +229,18 @@ func (workflow *StackupWorkflow) createMissingSettingsSection() {
 	if workflow.Settings == nil {
 		verifyChecksums := true
 		enableStats := false
+		gatewayAllow := "allowed"
 		workflow.Settings = &WorkflowSettings{
 			AnonymousStatistics:  &enableStats,
 			DotEnvFiles:          []string{".env"},
 			Cache:                &WorkflowSettingsCache{TtlMinutes: 5},
 			ChecksumVerification: &verifyChecksums,
 			Domains: &WorkflowSettingsDomains{
 				Allowed: []string{"raw.githubusercontent.com", "api.github.com"},
+				Hosts: []WorkflowSettingsDomainsHosts{
+					{Hostname: "raw.githubusercontent.com", Gateway: &gatewayAllow, Headers: nil},
+					{Hostname: "api.github.com", Gateway: &gatewayAllow, Headers: nil},
+				},
 			},
 			Defaults: &WorkflowSettingsDefaults{
 				Tasks: &WorkflowSettingsDefaultsTasks{
@@ -355,9 +378,9 @@ func (workflow *StackupWorkflow) handleDataNotCached(found bool, data *cache.Cac
 
 	if !found || data.IsExpired() {
 		if include.IsLocalFile() {
-			include.Contents, err = utils.GetFileContents(include.Filename())
+			include.Contents, err = App.Gateway.GetUrl(include.Filename())
 		} else if include.IsRemoteUrl() {
-			include.Contents, err = utils.GetUrlContentsEx(include.FullUrl(), include.Headers)
+			include.Contents, err = App.Gateway.GetUrl(include.FullUrl(), include.Headers...)
 		} else if include.IsS3Url() {
 			include.AccessKey = os.ExpandEnv(include.AccessKey)
 			include.SecretKey = os.ExpandEnv(include.SecretKey)

lib/downloader/s3downloader.go

@@ -57,28 +57,6 @@ func ParseS3Url(urlstr string) (*S3Url, error) {
 }
 
 func ReadS3FileContents(s3url string, accessKey string, secretKey string, secure bool) string {
-	// sess, _ := session.NewSession(&aws.Config{
-	// 	Region:      aws.String("us-west-2"),
-	// 	Credentials: credentials.NewStaticCredentials("AKID", "SECRET_KEY", "TOKEN"),
-	// })
-
-	// writer, err := os.Open(targetFile)
-	// if err != nil {
-	// 	return
-	// }
-	// defer writer.Close()
-
-	// downloader := s3manager.NewDownloader(sess)
-	// downloader.Download(writer,
-	// 	&s3.GetObjectInput{
-	// 		Bucket: &bucket,
-	// 		Key:    &key,
-	// 	})
-
-	// fmt.Printf("s3url: %s\n", s3url)
-	// fmt.Printf("accessKey: %s\n", accessKey)
-	// fmt.Printf("secretKey: %s\n", secretKey)
-
 	data, err := ParseS3Url(s3url)
 
 	s3Client, err := minio.New(data.Endpoint, &minio.Options{

lib/gateway/gateway.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"sync"
 
 	glob "github.com/ryanuber/go-glob"
 )
@@ -20,6 +21,7 @@ type Gateway struct {
 	AllowedDomains []string
 	DeniedDomains  []string
 	Middleware     []*GatewayUrlRequestMiddleware
+	DomainHeaders  *sync.Map
 }
 
 // New initializes the gateway with deny/allow lists
@@ -29,6 +31,7 @@ func New(deniedDomains, allowedDomains []string) *Gateway {
 		DeniedDomains:  deniedDomains,
 		AllowedDomains: allowedDomains,
 		Middleware:     []*GatewayUrlRequestMiddleware{},
+		DomainHeaders:  &sync.Map{},
 	}
 
 	result.Initialize()
@@ -56,6 +59,12 @@ func (g *Gateway) SetDeniedDomains(domains []string) {
 	g.normalizeDataArray(&g.DeniedDomains)
 }
 
+func (g *Gateway) SetDomainHeaders(domain string, headers []string) {
+	for _, header := range headers {
+		g.DomainHeaders.Store(domain, header)
+	}
+}
+
 func (g *Gateway) AddMiddleware(mw *GatewayUrlRequestMiddleware) {
 	g.Middleware = append(g.Middleware, mw)
 }
@@ -128,11 +137,21 @@ func (g *Gateway) checkArrayForMatch(arr *[]string, s string) bool {
 func (g *Gateway) GetUrl(urlStr string, headers ...string) (string, error) {
 	err := g.runUrlRequestPipeline(urlStr)
 	if err != nil {
+		fmt.Printf("error: %v\n", err)
 		return "", err
 	}
 
 	// remove the header items that are empty strings:
-	var tempHeaders []string
+	var tempHeaders []string = []string{"User-Agent: stackup/1.0"}
+
+	g.DomainHeaders.Range(func(key, value any) bool {
+		parsed, _ := url.Parse(urlStr)
+		if glob.Glob(key.(string), parsed.Hostname()) {
+			tempHeaders = append(tempHeaders, value.(string))
+		}
+		return true
+	})
+
 	for _, header := range headers {
 		if strings.TrimSpace(header) != "" {
 			tempHeaders = append(tempHeaders, header)