From 644a8e87bc0400829ebdb442225c3d37b18eb2df Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:52:38 +0000 Subject: [PATCH 01/10] feat: harden deploy migration flow Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/a286b823-c9cc-41e5-9805-7ad02eb0248b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/deploy_azure.go | 875 ++++++++++++------------- cmd/deploy_local.go | 16 +- cmd/helpers.go | 47 +- cmd/helpers_migration_test.go | 84 +++ internal/devlake/client.go | 1093 ++++++++++++++++--------------- internal/devlake/client_test.go | 47 ++ 6 files changed, 1164 insertions(+), 998 deletions(-) create mode 100644 cmd/helpers_migration_test.go diff --git a/cmd/deploy_azure.go b/cmd/deploy_azure.go index 14e9a12..142a141 100644 --- a/cmd/deploy_azure.go +++ b/cmd/deploy_azure.go @@ -1,440 +1,435 @@ -package cmd - -import ( - "encoding/json" - "fmt" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/DevExpGBB/gh-devlake/internal/azure" - dockerpkg "github.com/DevExpGBB/gh-devlake/internal/docker" - "github.com/DevExpGBB/gh-devlake/internal/gitclone" - "github.com/DevExpGBB/gh-devlake/internal/prompt" - "github.com/DevExpGBB/gh-devlake/internal/secrets" - "github.com/spf13/cobra" -) - -var ( - azureRG string - azureLocation string - azureBaseName string - azureSkipImageBuild bool - azureRepoURL string - azureOfficial bool - deployAzureDir string - deployAzureQuiet bool // suppress "Next Steps" when called from init wizard -) - -func newDeployAzureCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "azure", - Short: "Deploy DevLake to Azure Container Apps", - Long: `Provisions DevLake on Azure using Container Instances, Azure Database for MySQL, -and (optionally) Azure Container Registry. - -Example: - gh devlake deploy azure --resource-group devlake-rg --location eastus - gh devlake deploy azure --resource-group devlake-rg --location eastus --official`, - RunE: runDeployAzure, - } - - cmd.Flags().StringVar(&azureRG, "resource-group", "", "Azure Resource Group name") - cmd.Flags().StringVar(&azureLocation, "location", "", "Azure region") - cmd.Flags().StringVar(&azureBaseName, "base-name", "devlake", "Base name for Azure resources") - cmd.Flags().BoolVar(&azureSkipImageBuild, "skip-image-build", false, "Skip Docker image building") - cmd.Flags().StringVar(&azureRepoURL, "repo-url", "", "Clone a remote DevLake repository for building") - cmd.Flags().BoolVar(&azureOfficial, "official", false, "Use official Apache images from Docker Hub (no ACR)") - cmd.Flags().StringVar(&deployAzureDir, "dir", ".", "Directory to save deployment state (.devlake-azure.json)") - - return cmd -} - -// Common Azure regions for interactive selection. -var azureRegions = []string{ - "eastus", "eastus2", "westus2", "westus3", - "centralus", "northeurope", "westeurope", - "southeastasia", "australiaeast", "uksouth", -} - -func runDeployAzure(cmd *cobra.Command, args []string) error { - // Suggest a dedicated directory unless already in the right place or called from init - if !deployAzureQuiet { - if suggestDedicatedDir("azure", "gh devlake deploy azure") { - return nil - } - } - if deployAzureDir == "" { - deployAzureDir = "." - } - if err := os.MkdirAll(deployAzureDir, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %w", deployAzureDir, err) - } - - // ── Interactive image-source prompt (when no explicit flag set) ── - if !cmd.Flags().Changed("official") && !cmd.Flags().Changed("repo-url") { - imageChoices := []string{ - "official - Apache DevLake images from Docker Hub (recommended)", - "fork - Clone a DevLake repo and build from source", - "custom - Use a local repo or pre-built images", - } - fmt.Println() - imgChoice := prompt.Select("Which DevLake images to use?", imageChoices) - if imgChoice == "" { - return fmt.Errorf("image choice is required") - } - switch { - case strings.HasPrefix(imgChoice, "official"): - azureOfficial = true - case strings.HasPrefix(imgChoice, "fork"): - azureOfficial = false - if azureRepoURL == "" { - azureRepoURL = prompt.ReadLine(fmt.Sprintf("Repository URL [%s]", gitclone.DefaultForkURL)) - if azureRepoURL == "" { - azureRepoURL = gitclone.DefaultForkURL - } - } - default: // custom - azureOfficial = false - if azureRepoURL == "" { - azureRepoURL = prompt.ReadLine("Path or URL to DevLake repo (leave blank to auto-detect)") - } - } - } - - // ── Interactive prompts for missing required flags ── - if azureLocation == "" { - azureLocation = prompt.SelectWithOther("Select Azure region", azureRegions, true) - if azureLocation == "" { - return fmt.Errorf("--location is required") - } - } - if azureRG == "" { - azureRG = prompt.ReadLine("Resource group name (e.g. devlake-rg)") - if azureRG == "" { - return fmt.Errorf("--resource-group is required") - } - } - - suffix := azure.Suffix(azureRG) - acrName := "devlakeacr" + suffix - - fmt.Println() - if azureOfficial { - printBanner("DevLake Azure Deployment (Official)") - fmt.Println("\nUsing official Apache DevLake images from Docker Hub") - azureSkipImageBuild = true - } else { - printBanner("DevLake Azure Deployment") - } - - fmt.Printf("\n📋 Configuration:\n") - fmt.Printf(" Resource Group: %s\n", azureRG) - fmt.Printf(" Location: %s\n", azureLocation) - fmt.Printf(" Base Name: %s\n", azureBaseName) - if !azureOfficial { - fmt.Printf(" ACR Name: %s\n", acrName) - } else { - fmt.Println(" Images: Official (Docker Hub)") - } - - // ── Check Azure login ── - fmt.Println("\n🔑 Checking Azure CLI login...") - acct, err := azure.CheckLogin() - if err != nil { - fmt.Println(" Not logged in. Running az login...") - if loginErr := azure.Login(); loginErr != nil { - return fmt.Errorf("az login failed: %w", loginErr) - } - acct, err = azure.CheckLogin() - if err != nil { - return fmt.Errorf("still not logged in after az login: %w", err) - } - } - fmt.Printf(" Logged in as: %s\n", acct.User.Name) - - // ── Create Resource Group ── - fmt.Println("\n📦 Creating Resource Group...") - if err := azure.CreateResourceGroup(azureRG, azureLocation); err != nil { - return err - } - fmt.Println(" ✅ Resource Group created") - - // ── Write early checkpoint — ensures cleanup works even if deployment fails ── - savePartialAzureState(azureRG, azureLocation) - - // ── Generate secrets ── - fmt.Println("\n🔐 Generating secrets...") - mysqlPwd, err := secrets.MySQLPassword() - if err != nil { - return err - } - encSecret, err := secrets.EncryptionSecret(32) - if err != nil { - return err - } - fmt.Println(" ✅ Secrets generated") - - // ── Build and push images (if needed) ── - if !azureSkipImageBuild { - repoRoot, err := findRepoRoot() - if err != nil { - return err - } - if azureRepoURL != "" { - defer os.RemoveAll(repoRoot) - if err := applyPoetryPinWorkaround(repoRoot); err != nil { - fmt.Printf(" ⚠️ Could not apply temporary Poetry pin workaround: %v\n", err) - } else { - fmt.Printf(" ⚠️ Applied temporary Poetry pin workaround (poetry==%s) for fork builds\n", poetryWorkaroundVersion) - } - } - fmt.Printf("\n🏗️ Building Docker images from %s...\n", repoRoot) - - // Create ACR (idempotent — safe for re-runs) - fmt.Println(" Creating Container Registry...") - if err := azure.CreateACR(acrName, azureRG, azureLocation); err != nil { - return fmt.Errorf("failed to create ACR: %w", err) - } - fmt.Println(" ✅ Container Registry ready") - - acrServer := acrName + ".azurecr.io" - - fmt.Println("\n Logging into ACR...") - if err := azure.ACRLogin(acrName); err != nil { - return err - } - - images := []struct { - name string - dockerfile string - context string - }{ - {"devlake-backend", "backend/Dockerfile", filepath.Join(repoRoot, "backend")}, - {"devlake-config-ui", "config-ui/Dockerfile", filepath.Join(repoRoot, "config-ui")}, - {"devlake-grafana", "grafana/Dockerfile", filepath.Join(repoRoot, "grafana")}, - } - - for _, img := range images { - fmt.Printf("\n Building %s...\n", img.name) - localTag := img.name + ":latest" - if err := dockerpkg.Build(localTag, filepath.Join(repoRoot, img.dockerfile), img.context); err != nil { - fmt.Fprintf(os.Stderr, "\n ❌ Docker build failed for %s.\n", img.name) - fmt.Fprintf(os.Stderr, " Tip: re-run with --official to skip building and use\n") - fmt.Fprintf(os.Stderr, " official Apache DevLake images from Docker Hub instead.\n") - return fmt.Errorf("docker build failed for %s: %w", img.name, err) - } - remoteTag := acrServer + "/" + localTag - fmt.Printf(" Pushing %s...\n", img.name) - if err := dockerpkg.TagAndPush(localTag, remoteTag); err != nil { - return err - } - } - fmt.Println("\n ✅ All images pushed") - } - - // ── Check MySQL state ── - mysqlName := fmt.Sprintf("%smysql%s", azureBaseName, suffix) - fmt.Println("\n🗄️ Checking MySQL state...") - state, err := azure.MySQLState(mysqlName, azureRG) - if err == nil && state == "Stopped" { - fmt.Println(" MySQL is stopped. Starting...") - if err := azure.MySQLStart(mysqlName, azureRG); err != nil { - fmt.Printf(" ⚠️ Could not start MySQL: %v\n", err) - } else { - fmt.Println(" Waiting 30s for MySQL...") - time.Sleep(30 * time.Second) - fmt.Println(" ✅ MySQL started") - } - } else if state != "" { - fmt.Printf(" MySQL state: %s\n", state) - } else { - fmt.Println(" MySQL not yet created (will be created by Bicep)") - } - - // ── Check for soft-deleted Key Vault ── - kvName := fmt.Sprintf("%skv%s", azureBaseName, suffix) - found, _ := azure.CheckSoftDeletedKeyVault(kvName) - if found { - fmt.Printf("\n🔑 Key Vault %q found in soft-deleted state, purging...\n", kvName) - if err := azure.PurgeKeyVault(kvName, azureLocation); err != nil { - return fmt.Errorf("failed to purge soft-deleted Key Vault %q: %w\nManual fix: az keyvault purge --name %s --location %s", kvName, err, kvName, azureLocation) - } - fmt.Println(" ✅ Key Vault purged") - } - - // ── Deploy infrastructure ── - fmt.Println("\n🚀 Deploying infrastructure with Bicep...") - templateName := "main.bicep" - if azureOfficial { - templateName = "main-official.bicep" - } - templatePath, cleanup, err := azure.WriteTemplate(templateName) - if err != nil { - return err - } - defer cleanup() - - params := map[string]string{ - "baseName": azureBaseName, - "uniqueSuffix": suffix, - "mysqlAdminPassword": mysqlPwd, - "encryptionSecret": encSecret, - } - if !azureOfficial { - params["acrName"] = acrName - } - - deployment, err := azure.DeployBicep(azureRG, templatePath, params) - if err != nil { - return fmt.Errorf("Bicep deployment failed: %w", err) - } - - printBanner("✅ Deployment Complete!") - fmt.Printf("\nEndpoints:\n") - fmt.Printf(" Backend API: %s\n", deployment.BackendEndpoint) - fmt.Printf(" Config UI: %s\n", deployment.ConfigUIEndpoint) - fmt.Printf(" Grafana: %s\n", deployment.GrafanaEndpoint) - - // ── Wait for backend and trigger migration ── - fmt.Println("\n⏳ Waiting for backend to start...") - backendReady := waitForReady(deployment.BackendEndpoint, 30, 10*time.Second) == nil - - if backendReady { - fmt.Println(" ✅ Backend is responding!") - fmt.Println("\n🔄 Triggering database migration...") - httpClient := &http.Client{Timeout: 5 * time.Second} - resp, err := httpClient.Get(deployment.BackendEndpoint + "/proceed-db-migration") - if err == nil { - resp.Body.Close() - fmt.Println(" ✅ Migration triggered") - } else { - fmt.Printf(" ⚠️ Migration may need manual trigger: %v\n", err) - } - } else { - fmt.Println(" Backend not ready after 30 attempts.") - fmt.Printf(" Trigger migration manually: GET %s/proceed-db-migration\n", deployment.BackendEndpoint) - } - - // ── Save state file ── - stateFile := filepath.Join(deployAzureDir, ".devlake-azure.json") - containers := []string{ - fmt.Sprintf("%s-backend-%s", azureBaseName, suffix), - fmt.Sprintf("%s-grafana-%s", azureBaseName, suffix), - fmt.Sprintf("%s-ui-%s", azureBaseName, suffix), - } - - kvName = deployment.KeyVaultName - if kvName == "" { - kvName = fmt.Sprintf("%skv%s", azureBaseName, suffix) - } - - // Write a combined state file: Azure-specific metadata + DevLake discovery fields - combinedState := map[string]any{ - "deployedAt": time.Now().Format(time.RFC3339), - "method": methodName(), - "subscription": acct.Name, - "subscriptionId": acct.ID, - "resourceGroup": azureRG, - "region": azureLocation, - "suffix": suffix, - "useOfficialImages": azureOfficial, - "resources": map[string]any{ - "acr": conditionalACR(), - "keyVault": kvName, - "mysql": mysqlName, - "database": "lake", - "containers": containers, - }, - "endpoints": map[string]string{ - "backend": deployment.BackendEndpoint, - "grafana": deployment.GrafanaEndpoint, - "configUi": deployment.ConfigUIEndpoint, - }, - } - - data, _ := json.MarshalIndent(combinedState, "", " ") - if err := os.WriteFile(stateFile, data, 0644); err != nil { - fmt.Fprintf(os.Stderr, "⚠️ Could not save state file: %v\n", err) - } else { - fmt.Printf("\n💾 State saved to %s\n", stateFile) - if deployAzureDir != "." { - fmt.Println(" Next commands should be run from this directory:") - fmt.Println(" PowerShell:") - fmt.Printf(" Set-Location \"%s\"\n", deployAzureDir) - fmt.Println(" Bash/Zsh:") - fmt.Printf(" cd \"%s\"\n", deployAzureDir) - } - } - - if !deployAzureQuiet { - fmt.Println("\nNext steps:") - fmt.Println(" 1. Wait 2-3 minutes for containers to start") - fmt.Printf(" 2. Open Config UI: %s\n", deployment.ConfigUIEndpoint) - fmt.Println(" 3. Configure your data sources") - fmt.Printf("\nTo cleanup: gh devlake cleanup --azure\n") - } - - return nil -} - -func findRepoRoot() (string, error) { - if azureRepoURL != "" { - tmpDir, err := os.MkdirTemp("", "devlake-clone-*") - if err != nil { - return "", err - } - fmt.Printf(" Cloning %s...\n", azureRepoURL) - if err := gitclone.Clone(azureRepoURL, tmpDir); err != nil { - return "", err - } - return tmpDir, nil - } - - // Walk up looking for backend/Dockerfile - dir, _ := os.Getwd() - for dir != "" && dir != filepath.Dir(dir) { - if _, err := os.Stat(filepath.Join(dir, "backend", "Dockerfile")); err == nil { - return dir, nil - } - dir = filepath.Dir(dir) - } - return "", fmt.Errorf("could not find DevLake repo root.\n" + - "Options:\n" + - " --repo-url Clone a fork with the custom Dockerfile\n" + - " --official Use official Apache images (no build needed)") -} - -func methodName() string { - if azureOfficial { - return "bicep-official" - } - return "bicep" -} - -func conditionalACR() any { - if azureOfficial { - return nil - } - return "devlakeacr" + azure.Suffix(azureRG) -} - -// savePartialAzureState writes a minimal state file immediately after the -// Resource Group is created so that cleanup --azure always has a breadcrumb, -// even when the deployment fails mid-flight (e.g. Docker build errors). -// The full state write at the end of a successful deployment overwrites this. -func savePartialAzureState(rg, region string) { - stateFile := ".devlake-azure.json" - partial := map[string]any{ - "deployedAt": time.Now().Format(time.RFC3339), - "resourceGroup": rg, - "region": region, - "partial": true, - } - data, _ := json.MarshalIndent(partial, "", " ") - if err := os.WriteFile(stateFile, data, 0644); err != nil { - fmt.Fprintf(os.Stderr, "⚠️ Could not save early state checkpoint: %v\n", err) - } -} +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/DevExpGBB/gh-devlake/internal/azure" + dockerpkg "github.com/DevExpGBB/gh-devlake/internal/docker" + "github.com/DevExpGBB/gh-devlake/internal/gitclone" + "github.com/DevExpGBB/gh-devlake/internal/prompt" + "github.com/DevExpGBB/gh-devlake/internal/secrets" + "github.com/spf13/cobra" +) + +var ( + azureRG string + azureLocation string + azureBaseName string + azureSkipImageBuild bool + azureRepoURL string + azureOfficial bool + deployAzureDir string + deployAzureQuiet bool // suppress "Next Steps" when called from init wizard +) + +func newDeployAzureCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "azure", + Short: "Deploy DevLake to Azure Container Apps", + Long: `Provisions DevLake on Azure using Container Instances, Azure Database for MySQL, +and (optionally) Azure Container Registry. + +Example: + gh devlake deploy azure --resource-group devlake-rg --location eastus + gh devlake deploy azure --resource-group devlake-rg --location eastus --official`, + RunE: runDeployAzure, + } + + cmd.Flags().StringVar(&azureRG, "resource-group", "", "Azure Resource Group name") + cmd.Flags().StringVar(&azureLocation, "location", "", "Azure region") + cmd.Flags().StringVar(&azureBaseName, "base-name", "devlake", "Base name for Azure resources") + cmd.Flags().BoolVar(&azureSkipImageBuild, "skip-image-build", false, "Skip Docker image building") + cmd.Flags().StringVar(&azureRepoURL, "repo-url", "", "Clone a remote DevLake repository for building") + cmd.Flags().BoolVar(&azureOfficial, "official", false, "Use official Apache images from Docker Hub (no ACR)") + cmd.Flags().StringVar(&deployAzureDir, "dir", ".", "Directory to save deployment state (.devlake-azure.json)") + + return cmd +} + +// Common Azure regions for interactive selection. +var azureRegions = []string{ + "eastus", "eastus2", "westus2", "westus3", + "centralus", "northeurope", "westeurope", + "southeastasia", "australiaeast", "uksouth", +} + +func runDeployAzure(cmd *cobra.Command, args []string) error { + // Suggest a dedicated directory unless already in the right place or called from init + if !deployAzureQuiet { + if suggestDedicatedDir("azure", "gh devlake deploy azure") { + return nil + } + } + if deployAzureDir == "" { + deployAzureDir = "." + } + if err := os.MkdirAll(deployAzureDir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", deployAzureDir, err) + } + + // ── Interactive image-source prompt (when no explicit flag set) ── + if !cmd.Flags().Changed("official") && !cmd.Flags().Changed("repo-url") { + imageChoices := []string{ + "official - Apache DevLake images from Docker Hub (recommended)", + "fork - Clone a DevLake repo and build from source", + "custom - Use a local repo or pre-built images", + } + fmt.Println() + imgChoice := prompt.Select("Which DevLake images to use?", imageChoices) + if imgChoice == "" { + return fmt.Errorf("image choice is required") + } + switch { + case strings.HasPrefix(imgChoice, "official"): + azureOfficial = true + case strings.HasPrefix(imgChoice, "fork"): + azureOfficial = false + if azureRepoURL == "" { + azureRepoURL = prompt.ReadLine(fmt.Sprintf("Repository URL [%s]", gitclone.DefaultForkURL)) + if azureRepoURL == "" { + azureRepoURL = gitclone.DefaultForkURL + } + } + default: // custom + azureOfficial = false + if azureRepoURL == "" { + azureRepoURL = prompt.ReadLine("Path or URL to DevLake repo (leave blank to auto-detect)") + } + } + } + + // ── Interactive prompts for missing required flags ── + if azureLocation == "" { + azureLocation = prompt.SelectWithOther("Select Azure region", azureRegions, true) + if azureLocation == "" { + return fmt.Errorf("--location is required") + } + } + if azureRG == "" { + azureRG = prompt.ReadLine("Resource group name (e.g. devlake-rg)") + if azureRG == "" { + return fmt.Errorf("--resource-group is required") + } + } + + suffix := azure.Suffix(azureRG) + acrName := "devlakeacr" + suffix + + fmt.Println() + if azureOfficial { + printBanner("DevLake Azure Deployment (Official)") + fmt.Println("\nUsing official Apache DevLake images from Docker Hub") + azureSkipImageBuild = true + } else { + printBanner("DevLake Azure Deployment") + } + + fmt.Printf("\n📋 Configuration:\n") + fmt.Printf(" Resource Group: %s\n", azureRG) + fmt.Printf(" Location: %s\n", azureLocation) + fmt.Printf(" Base Name: %s\n", azureBaseName) + if !azureOfficial { + fmt.Printf(" ACR Name: %s\n", acrName) + } else { + fmt.Println(" Images: Official (Docker Hub)") + } + + // ── Check Azure login ── + fmt.Println("\n🔑 Checking Azure CLI login...") + acct, err := azure.CheckLogin() + if err != nil { + fmt.Println(" Not logged in. Running az login...") + if loginErr := azure.Login(); loginErr != nil { + return fmt.Errorf("az login failed: %w", loginErr) + } + acct, err = azure.CheckLogin() + if err != nil { + return fmt.Errorf("still not logged in after az login: %w", err) + } + } + fmt.Printf(" Logged in as: %s\n", acct.User.Name) + + // ── Create Resource Group ── + fmt.Println("\n📦 Creating Resource Group...") + if err := azure.CreateResourceGroup(azureRG, azureLocation); err != nil { + return err + } + fmt.Println(" ✅ Resource Group created") + + // ── Write early checkpoint — ensures cleanup works even if deployment fails ── + savePartialAzureState(azureRG, azureLocation) + + // ── Generate secrets ── + fmt.Println("\n🔐 Generating secrets...") + mysqlPwd, err := secrets.MySQLPassword() + if err != nil { + return err + } + encSecret, err := secrets.EncryptionSecret(32) + if err != nil { + return err + } + fmt.Println(" ✅ Secrets generated") + + // ── Build and push images (if needed) ── + if !azureSkipImageBuild { + repoRoot, err := findRepoRoot() + if err != nil { + return err + } + if azureRepoURL != "" { + defer os.RemoveAll(repoRoot) + if err := applyPoetryPinWorkaround(repoRoot); err != nil { + fmt.Printf(" ⚠️ Could not apply temporary Poetry pin workaround: %v\n", err) + } else { + fmt.Printf(" ⚠️ Applied temporary Poetry pin workaround (poetry==%s) for fork builds\n", poetryWorkaroundVersion) + } + } + fmt.Printf("\n🏗️ Building Docker images from %s...\n", repoRoot) + + // Create ACR (idempotent — safe for re-runs) + fmt.Println(" Creating Container Registry...") + if err := azure.CreateACR(acrName, azureRG, azureLocation); err != nil { + return fmt.Errorf("failed to create ACR: %w", err) + } + fmt.Println(" ✅ Container Registry ready") + + acrServer := acrName + ".azurecr.io" + + fmt.Println("\n Logging into ACR...") + if err := azure.ACRLogin(acrName); err != nil { + return err + } + + images := []struct { + name string + dockerfile string + context string + }{ + {"devlake-backend", "backend/Dockerfile", filepath.Join(repoRoot, "backend")}, + {"devlake-config-ui", "config-ui/Dockerfile", filepath.Join(repoRoot, "config-ui")}, + {"devlake-grafana", "grafana/Dockerfile", filepath.Join(repoRoot, "grafana")}, + } + + for _, img := range images { + fmt.Printf("\n Building %s...\n", img.name) + localTag := img.name + ":latest" + if err := dockerpkg.Build(localTag, filepath.Join(repoRoot, img.dockerfile), img.context); err != nil { + fmt.Fprintf(os.Stderr, "\n ❌ Docker build failed for %s.\n", img.name) + fmt.Fprintf(os.Stderr, " Tip: re-run with --official to skip building and use\n") + fmt.Fprintf(os.Stderr, " official Apache DevLake images from Docker Hub instead.\n") + return fmt.Errorf("docker build failed for %s: %w", img.name, err) + } + remoteTag := acrServer + "/" + localTag + fmt.Printf(" Pushing %s...\n", img.name) + if err := dockerpkg.TagAndPush(localTag, remoteTag); err != nil { + return err + } + } + fmt.Println("\n ✅ All images pushed") + } + + // ── Check MySQL state ── + mysqlName := fmt.Sprintf("%smysql%s", azureBaseName, suffix) + fmt.Println("\n🗄️ Checking MySQL state...") + state, err := azure.MySQLState(mysqlName, azureRG) + if err == nil && state == "Stopped" { + fmt.Println(" MySQL is stopped. Starting...") + if err := azure.MySQLStart(mysqlName, azureRG); err != nil { + fmt.Printf(" ⚠️ Could not start MySQL: %v\n", err) + } else { + fmt.Println(" Waiting 30s for MySQL...") + time.Sleep(30 * time.Second) + fmt.Println(" ✅ MySQL started") + } + } else if state != "" { + fmt.Printf(" MySQL state: %s\n", state) + } else { + fmt.Println(" MySQL not yet created (will be created by Bicep)") + } + + // ── Check for soft-deleted Key Vault ── + kvName := fmt.Sprintf("%skv%s", azureBaseName, suffix) + found, _ := azure.CheckSoftDeletedKeyVault(kvName) + if found { + fmt.Printf("\n🔑 Key Vault %q found in soft-deleted state, purging...\n", kvName) + if err := azure.PurgeKeyVault(kvName, azureLocation); err != nil { + return fmt.Errorf("failed to purge soft-deleted Key Vault %q: %w\nManual fix: az keyvault purge --name %s --location %s", kvName, err, kvName, azureLocation) + } + fmt.Println(" ✅ Key Vault purged") + } + + // ── Deploy infrastructure ── + fmt.Println("\n🚀 Deploying infrastructure with Bicep...") + templateName := "main.bicep" + if azureOfficial { + templateName = "main-official.bicep" + } + templatePath, cleanup, err := azure.WriteTemplate(templateName) + if err != nil { + return err + } + defer cleanup() + + params := map[string]string{ + "baseName": azureBaseName, + "uniqueSuffix": suffix, + "mysqlAdminPassword": mysqlPwd, + "encryptionSecret": encSecret, + } + if !azureOfficial { + params["acrName"] = acrName + } + + deployment, err := azure.DeployBicep(azureRG, templatePath, params) + if err != nil { + return fmt.Errorf("Bicep deployment failed: %w", err) + } + + printBanner("✅ Deployment Complete!") + fmt.Printf("\nEndpoints:\n") + fmt.Printf(" Backend API: %s\n", deployment.BackendEndpoint) + fmt.Printf(" Config UI: %s\n", deployment.ConfigUIEndpoint) + fmt.Printf(" Grafana: %s\n", deployment.GrafanaEndpoint) + + // ── Wait for backend and trigger migration ── + fmt.Println("\n⏳ Waiting for backend to start...") + backendReady := waitForReady(deployment.BackendEndpoint, 30, 10*time.Second) == nil + + if backendReady { + fmt.Println(" ✅ Backend is responding!") + if err := triggerAndWaitForMigration(deployment.BackendEndpoint); err != nil { + fmt.Printf(" ⚠️ %v\n", err) + fmt.Printf(" Trigger migration manually if needed: GET %s/proceed-db-migration\n", deployment.BackendEndpoint) + fmt.Println(" Migration may still be running — proceeding anyway") + } + } else { + fmt.Println(" Backend not ready after 30 attempts.") + fmt.Printf(" Trigger migration manually: GET %s/proceed-db-migration\n", deployment.BackendEndpoint) + } + + // ── Save state file ── + stateFile := filepath.Join(deployAzureDir, ".devlake-azure.json") + containers := []string{ + fmt.Sprintf("%s-backend-%s", azureBaseName, suffix), + fmt.Sprintf("%s-grafana-%s", azureBaseName, suffix), + fmt.Sprintf("%s-ui-%s", azureBaseName, suffix), + } + + kvName = deployment.KeyVaultName + if kvName == "" { + kvName = fmt.Sprintf("%skv%s", azureBaseName, suffix) + } + + // Write a combined state file: Azure-specific metadata + DevLake discovery fields + combinedState := map[string]any{ + "deployedAt": time.Now().Format(time.RFC3339), + "method": methodName(), + "subscription": acct.Name, + "subscriptionId": acct.ID, + "resourceGroup": azureRG, + "region": azureLocation, + "suffix": suffix, + "useOfficialImages": azureOfficial, + "resources": map[string]any{ + "acr": conditionalACR(), + "keyVault": kvName, + "mysql": mysqlName, + "database": "lake", + "containers": containers, + }, + "endpoints": map[string]string{ + "backend": deployment.BackendEndpoint, + "grafana": deployment.GrafanaEndpoint, + "configUi": deployment.ConfigUIEndpoint, + }, + } + + data, _ := json.MarshalIndent(combinedState, "", " ") + if err := os.WriteFile(stateFile, data, 0644); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Could not save state file: %v\n", err) + } else { + fmt.Printf("\n💾 State saved to %s\n", stateFile) + if deployAzureDir != "." { + fmt.Println(" Next commands should be run from this directory:") + fmt.Println(" PowerShell:") + fmt.Printf(" Set-Location \"%s\"\n", deployAzureDir) + fmt.Println(" Bash/Zsh:") + fmt.Printf(" cd \"%s\"\n", deployAzureDir) + } + } + + if !deployAzureQuiet { + fmt.Println("\nNext steps:") + fmt.Println(" 1. Wait 2-3 minutes for containers to start") + fmt.Printf(" 2. Open Config UI: %s\n", deployment.ConfigUIEndpoint) + fmt.Println(" 3. Configure your data sources") + fmt.Printf("\nTo cleanup: gh devlake cleanup --azure\n") + } + + return nil +} + +func findRepoRoot() (string, error) { + if azureRepoURL != "" { + tmpDir, err := os.MkdirTemp("", "devlake-clone-*") + if err != nil { + return "", err + } + fmt.Printf(" Cloning %s...\n", azureRepoURL) + if err := gitclone.Clone(azureRepoURL, tmpDir); err != nil { + return "", err + } + return tmpDir, nil + } + + // Walk up looking for backend/Dockerfile + dir, _ := os.Getwd() + for dir != "" && dir != filepath.Dir(dir) { + if _, err := os.Stat(filepath.Join(dir, "backend", "Dockerfile")); err == nil { + return dir, nil + } + dir = filepath.Dir(dir) + } + return "", fmt.Errorf("could not find DevLake repo root.\n" + + "Options:\n" + + " --repo-url Clone a fork with the custom Dockerfile\n" + + " --official Use official Apache images (no build needed)") +} + +func methodName() string { + if azureOfficial { + return "bicep-official" + } + return "bicep" +} + +func conditionalACR() any { + if azureOfficial { + return nil + } + return "devlakeacr" + azure.Suffix(azureRG) +} + +// savePartialAzureState writes a minimal state file immediately after the +// Resource Group is created so that cleanup --azure always has a breadcrumb, +// even when the deployment fails mid-flight (e.g. Docker build errors). +// The full state write at the end of a successful deployment overwrites this. +func savePartialAzureState(rg, region string) { + stateFile := ".devlake-azure.json" + partial := map[string]any{ + "deployedAt": time.Now().Format(time.RFC3339), + "resourceGroup": rg, + "region": region, + "partial": true, + } + data, _ := json.MarshalIndent(partial, "", " ") + if err := os.WriteFile(stateFile, data, 0644); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Could not save early state checkpoint: %v\n", err) + } +} diff --git a/cmd/deploy_local.go b/cmd/deploy_local.go index 917282c..61104ac 100644 --- a/cmd/deploy_local.go +++ b/cmd/deploy_local.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/DevExpGBB/gh-devlake/internal/devlake" dockerpkg "github.com/DevExpGBB/gh-devlake/internal/docker" "github.com/DevExpGBB/gh-devlake/internal/download" "github.com/DevExpGBB/gh-devlake/internal/gitclone" @@ -196,17 +195,10 @@ func runDeployLocal(cmd *cobra.Command, args []string) error { } cfgURL = backendURL - fmt.Println("\n🔄 Triggering database migration...") - migClient := devlake.NewClient(backendURL) - if err := migClient.TriggerMigration(); err != nil { - fmt.Printf(" ⚠️ Migration may need manual trigger: %v\n", err) - } else { - fmt.Println(" ✅ Migration triggered") - fmt.Println("\n⏳ Waiting for migration to complete...") - if err := waitForMigration(backendURL, 60, 5*time.Second); err != nil { - fmt.Printf(" ⚠️ %v\n", err) - fmt.Println(" Migration may still be running — proceeding anyway") - } + if err := triggerAndWaitForMigration(backendURL); err != nil { + fmt.Printf(" ⚠️ %v\n", err) + fmt.Printf(" Trigger migration manually if needed: GET %s/proceed-db-migration\n", backendURL) + fmt.Println(" Migration may still be running — proceeding anyway") } if !deployLocalQuiet { diff --git a/cmd/helpers.go b/cmd/helpers.go index edfe91a..1466804 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -221,21 +221,66 @@ func waitForReadyAny(baseURLs []string, maxAttempts int, interval time.Duration) // During migration the API returns 428 (Precondition Required). func waitForMigration(baseURL string, maxAttempts int, interval time.Duration) error { httpClient := &http.Client{Timeout: 5 * time.Second} + lastStatus := 0 for attempt := 1; attempt <= maxAttempts; attempt++ { resp, err := httpClient.Get(baseURL + "/ping") if err == nil { + lastStatus = resp.StatusCode resp.Body.Close() if resp.StatusCode == http.StatusOK { fmt.Println(" ✅ Migration complete!") return nil } } - fmt.Printf(" Migrating... (%d/%d)\n", attempt, maxAttempts) + if lastStatus != 0 { + fmt.Printf(" Migrating... (%d/%d, status=%d)\n", attempt, maxAttempts, lastStatus) + } else { + fmt.Printf(" Migrating... (%d/%d)\n", attempt, maxAttempts) + } time.Sleep(interval) } + if lastStatus != 0 { + return fmt.Errorf("migration did not complete after %d attempts (last status %d)", maxAttempts, lastStatus) + } return fmt.Errorf("migration did not complete after %d attempts", maxAttempts) } +func triggerAndWaitForMigration(baseURL string) error { + return triggerAndWaitForMigrationWithClient(baseURL, devlake.NewClient(baseURL), 3, 10*time.Second, 60, 5*time.Second) +} + +func triggerAndWaitForMigrationWithClient(baseURL string, migClient *devlake.Client, triggerAttempts int, triggerInterval time.Duration, waitAttempts int, waitInterval time.Duration) error { + fmt.Println("\n🔄 Triggering database migration...") + + var lastErr error + for attempt := 1; attempt <= triggerAttempts; attempt++ { + if err := migClient.TriggerMigration(); err == nil { + fmt.Println(" ✅ Migration triggered") + lastErr = nil + break + } else { + lastErr = err + fmt.Printf(" ⚠️ Trigger attempt %d/%d did not complete cleanly: %v\n", attempt, triggerAttempts, err) + if attempt < triggerAttempts { + fmt.Println(" DevLake may still be starting or migration may already be running — retrying...") + time.Sleep(triggerInterval) + } + } + } + + fmt.Println("\n⏳ Waiting for migration to complete...") + if lastErr != nil { + fmt.Println(" Continuing to monitor migration status anyway...") + } + if err := waitForMigration(baseURL, waitAttempts, waitInterval); err != nil { + if lastErr != nil { + return fmt.Errorf("migration trigger did not complete cleanly: %w; %w", lastErr, err) + } + return err + } + return nil +} + // ── Scope orchestration ───────────────────────────────────────── // scopeAllConnections iterates connection results and configures scopes diff --git a/cmd/helpers_migration_test.go b/cmd/helpers_migration_test.go new file mode 100644 index 0000000..03c4434 --- /dev/null +++ b/cmd/helpers_migration_test.go @@ -0,0 +1,84 @@ +package cmd + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/DevExpGBB/gh-devlake/internal/devlake" +) + +func TestTriggerAndWaitForMigrationWithClient_CompletesAfterTriggerTimeout(t *testing.T) { + triggerCalls := 0 + pingCalls := 0 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/proceed-db-migration": + triggerCalls++ + time.Sleep(25 * time.Millisecond) + w.WriteHeader(http.StatusOK) + case "/ping": + pingCalls++ + if pingCalls == 1 { + w.WriteHeader(http.StatusPreconditionRequired) + return + } + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + client := devlake.NewClient(srv.URL) + client.HTTPClient.Timeout = 5 * time.Millisecond + + err := triggerAndWaitForMigrationWithClient(srv.URL, client, 1, time.Millisecond, 3, time.Millisecond) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if triggerCalls != 1 { + t.Fatalf("trigger calls = %d, want 1", triggerCalls) + } + if pingCalls != 2 { + t.Fatalf("ping calls = %d, want 2", pingCalls) + } +} + +func TestTriggerAndWaitForMigrationWithClient_RetriesBeforeWaiting(t *testing.T) { + triggerCalls := 0 + pingCalls := 0 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/proceed-db-migration": + triggerCalls++ + if triggerCalls == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + case "/ping": + pingCalls++ + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + client := devlake.NewClient(srv.URL) + + err := triggerAndWaitForMigrationWithClient(srv.URL, client, 2, time.Millisecond, 2, time.Millisecond) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if triggerCalls != 2 { + t.Fatalf("trigger calls = %d, want 2", triggerCalls) + } + if pingCalls != 1 { + t.Fatalf("ping calls = %d, want 1", pingCalls) + } +} diff --git a/internal/devlake/client.go b/internal/devlake/client.go index 9831e3a..c9a3cd7 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -1,545 +1,548 @@ -// Package devlake provides an HTTP client for the DevLake REST API. -package devlake - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "time" -) - -// Client wraps HTTP calls to the DevLake backend API. -type Client struct { - BaseURL string - HTTPClient *http.Client -} - -// NewClient creates a Client for the given base URL. -func NewClient(baseURL string) *Client { - return &Client{ - BaseURL: baseURL, - HTTPClient: &http.Client{ - Timeout: 90 * time.Second, - }, - } -} - -// Ping checks if the DevLake backend is reachable. -func (c *Client) Ping() error { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) - } - return nil -} - -// Connection represents a DevLake plugin connection. -type Connection struct { - ID int `json:"id"` - Name string `json:"name"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. -// Fields with omitempty are only included in the request when non-empty, -// enabling sparse updates (only changed fields are sent). -type ConnectionUpdateRequest struct { - Name string `json:"name,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionCreateRequest is the payload for creating a plugin connection. -type ConnectionCreateRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` - TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` - RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` -} - -// ConnectionTestRequest is the payload for testing a connection before creating. -type ConnectionTestRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Proxy string `json:"proxy"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionTestResult is the response from testing a connection. -type ConnectionTestResult struct { - Success bool `json:"success"` - Message string `json:"message"` -} - -// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). -func (c *Client) ListConnections(plugin string) ([]Connection, error) { - resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) - } - - var conns []Connection - if err := json.Unmarshal(body, &conns); err != nil { - return nil, err - } - return conns, nil -} - -// FindConnectionByName returns the first connection matching the given name, or nil. -func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { - conns, err := c.ListConnections(plugin) - if err != nil { - return nil, err - } - for _, conn := range conns { - if conn.Name == name { - return &conn, nil - } - } - return nil, nil -} - -// TestConnection tests connection parameters before creating. -func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { - return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) -} - -// CreateConnection creates a new connection for the given plugin. -func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { - return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) -} - -// DeleteConnection deletes a plugin connection by ID. -func (c *Client) DeleteConnection(plugin string, connID int) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) - } - return nil -} - -// TestSavedConnection tests an already-created connection by ID. -func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) - - reqBody := bytes.NewBufferString("{}") - resp, err := c.HTTPClient.Post(url, "application/json", reqBody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var result ConnectionTestResult - if err := json.Unmarshal(body, &result); err != nil { - // Non-JSON response is ok — treat as success if status 200 - if resp.StatusCode == http.StatusOK { - return &ConnectionTestResult{Success: true}, nil - } - return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) - } - return &result, nil -} - -// GetConnection retrieves a single connection by plugin and ID. -func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { - return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) -} - -// UpdateConnection patches an existing connection for the given plugin. -func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { - return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) -} - -// HealthStatus represents the response from /health or /ping. -type HealthStatus struct { - Status string `json:"status"` -} - -// Health returns the DevLake health status. -func (c *Client) Health() (*HealthStatus, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var hs HealthStatus - _ = json.Unmarshal(body, &hs) - if resp.StatusCode == http.StatusOK { - if hs.Status == "" { - hs.Status = "ok" - } - return &hs, nil - } - return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) -} - -// doPost is a generic helper for POST requests that return JSON. -func doPost[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - url := c.BaseURL + path - resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doGet is a generic helper for GET requests that return JSON. -func doGet[T any](c *Client, path string) (*T, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + path) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPut is a generic helper for PUT requests that return JSON. -func doPut[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPatch is a generic helper for PATCH requests that return JSON. -func doPatch[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// CreateScopeConfig creates a scope config for a plugin connection. -func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { - return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) -} - -// ListScopeConfigs returns all scope configs for a plugin connection. -func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { - result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) - if err != nil { - return nil, err - } - return *result, nil -} - -// PutScopes batch-upserts scopes for a plugin connection. -func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { - _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) - return err -} - -// ListScopes returns the scopes configured on a plugin connection. -func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { - return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) -} - -// ListProjects returns all DevLake projects. -func (c *Client) ListProjects() ([]Project, error) { - result, err := doGet[ProjectListResponse](c, "/projects") - if err != nil { - return nil, err - } - return result.Projects, nil -} - -// DeleteProject deletes a project by name. -func (c *Client) DeleteProject(name string) error { - url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("project not found: %s", name) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) - } - return nil -} - -// DeleteScope removes a scope from a plugin connection. -func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) - } - return nil -} - -// CreateProject creates a new DevLake project. -func (c *Client) CreateProject(project *Project) (*Project, error) { - return doPost[Project](c, "/projects", project) -} - -// GetProject retrieves a project by name. -func (c *Client) GetProject(name string) (*Project, error) { - return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) -} - -// PatchBlueprint updates a blueprint by ID. -func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { - return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) -} - -// TriggerBlueprint triggers a blueprint to run and returns the pipeline. -func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { - return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) -} - -// GetPipeline retrieves a pipeline by ID. -func (c *Client) GetPipeline(id int) (*Pipeline, error) { - return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) -} - -// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. -// groupID and pageToken are optional (pass "" to omit). -func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) - q := url.Values{} - if groupID != "" { - q.Set("groupId", groupID) - } - if pageToken != "" { - q.Set("pageToken", pageToken) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. -// page and pageSize control pagination; pass 0 to use DevLake defaults. -func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) - q := url.Values{} - if search != "" { - q.Set("search", search) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pageSize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// TriggerMigration triggers the DevLake database migration endpoint. -func (c *Client) TriggerMigration() error { - resp, err := c.HTTPClient.Get(c.BaseURL + "/proceed-db-migration") - if err != nil { - return err - } - resp.Body.Close() - return nil -} - -// PipelineListResponse is the response from GET /pipelines. -type PipelineListResponse struct { - Pipelines []Pipeline `json:"pipelines"` - Count int64 `json:"count"` -} - -// ListPipelines returns pipelines with optional query parameters. -// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. -// blueprintID filters by blueprint (0 = no filter). -// page and pageSize control pagination (0 = use defaults). -func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { - path := "/pipelines" - q := url.Values{} - if status != "" { - q.Set("status", status) - } - if blueprintID > 0 { - q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pagesize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[PipelineListResponse](c, path) -} +// Package devlake provides an HTTP client for the DevLake REST API. +package devlake + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// Client wraps HTTP calls to the DevLake backend API. +type Client struct { + BaseURL string + HTTPClient *http.Client +} + +// NewClient creates a Client for the given base URL. +func NewClient(baseURL string) *Client { + return &Client{ + BaseURL: baseURL, + HTTPClient: &http.Client{ + Timeout: 90 * time.Second, + }, + } +} + +// Ping checks if the DevLake backend is reachable. +func (c *Client) Ping() error { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) + } + return nil +} + +// Connection represents a DevLake plugin connection. +type Connection struct { + ID int `json:"id"` + Name string `json:"name"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. +// Fields with omitempty are only included in the request when non-empty, +// enabling sparse updates (only changed fields are sent). +type ConnectionUpdateRequest struct { + Name string `json:"name,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionCreateRequest is the payload for creating a plugin connection. +type ConnectionCreateRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` + TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` + RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` +} + +// ConnectionTestRequest is the payload for testing a connection before creating. +type ConnectionTestRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Proxy string `json:"proxy"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionTestResult is the response from testing a connection. +type ConnectionTestResult struct { + Success bool `json:"success"` + Message string `json:"message"` +} + +// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). +func (c *Client) ListConnections(plugin string) ([]Connection, error) { + resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) + } + + var conns []Connection + if err := json.Unmarshal(body, &conns); err != nil { + return nil, err + } + return conns, nil +} + +// FindConnectionByName returns the first connection matching the given name, or nil. +func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { + conns, err := c.ListConnections(plugin) + if err != nil { + return nil, err + } + for _, conn := range conns { + if conn.Name == name { + return &conn, nil + } + } + return nil, nil +} + +// TestConnection tests connection parameters before creating. +func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { + return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) +} + +// CreateConnection creates a new connection for the given plugin. +func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { + return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) +} + +// DeleteConnection deletes a plugin connection by ID. +func (c *Client) DeleteConnection(plugin string, connID int) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) + } + return nil +} + +// TestSavedConnection tests an already-created connection by ID. +func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) + + reqBody := bytes.NewBufferString("{}") + resp, err := c.HTTPClient.Post(url, "application/json", reqBody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var result ConnectionTestResult + if err := json.Unmarshal(body, &result); err != nil { + // Non-JSON response is ok — treat as success if status 200 + if resp.StatusCode == http.StatusOK { + return &ConnectionTestResult{Success: true}, nil + } + return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) + } + return &result, nil +} + +// GetConnection retrieves a single connection by plugin and ID. +func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { + return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) +} + +// UpdateConnection patches an existing connection for the given plugin. +func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { + return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) +} + +// HealthStatus represents the response from /health or /ping. +type HealthStatus struct { + Status string `json:"status"` +} + +// Health returns the DevLake health status. +func (c *Client) Health() (*HealthStatus, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var hs HealthStatus + _ = json.Unmarshal(body, &hs) + if resp.StatusCode == http.StatusOK { + if hs.Status == "" { + hs.Status = "ok" + } + return &hs, nil + } + return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) +} + +// doPost is a generic helper for POST requests that return JSON. +func doPost[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + url := c.BaseURL + path + resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doGet is a generic helper for GET requests that return JSON. +func doGet[T any](c *Client, path string) (*T, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + path) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPut is a generic helper for PUT requests that return JSON. +func doPut[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPatch is a generic helper for PATCH requests that return JSON. +func doPatch[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// CreateScopeConfig creates a scope config for a plugin connection. +func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { + return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) +} + +// ListScopeConfigs returns all scope configs for a plugin connection. +func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { + result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) + if err != nil { + return nil, err + } + return *result, nil +} + +// PutScopes batch-upserts scopes for a plugin connection. +func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { + _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) + return err +} + +// ListScopes returns the scopes configured on a plugin connection. +func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { + return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) +} + +// ListProjects returns all DevLake projects. +func (c *Client) ListProjects() ([]Project, error) { + result, err := doGet[ProjectListResponse](c, "/projects") + if err != nil { + return nil, err + } + return result.Projects, nil +} + +// DeleteProject deletes a project by name. +func (c *Client) DeleteProject(name string) error { + url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("project not found: %s", name) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) + } + return nil +} + +// DeleteScope removes a scope from a plugin connection. +func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) + } + return nil +} + +// CreateProject creates a new DevLake project. +func (c *Client) CreateProject(project *Project) (*Project, error) { + return doPost[Project](c, "/projects", project) +} + +// GetProject retrieves a project by name. +func (c *Client) GetProject(name string) (*Project, error) { + return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) +} + +// PatchBlueprint updates a blueprint by ID. +func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { + return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) +} + +// TriggerBlueprint triggers a blueprint to run and returns the pipeline. +func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { + return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) +} + +// GetPipeline retrieves a pipeline by ID. +func (c *Client) GetPipeline(id int) (*Pipeline, error) { + return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) +} + +// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. +// groupID and pageToken are optional (pass "" to omit). +func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) + q := url.Values{} + if groupID != "" { + q.Set("groupId", groupID) + } + if pageToken != "" { + q.Set("pageToken", pageToken) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. +// page and pageSize control pagination; pass 0 to use DevLake defaults. +func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) + q := url.Values{} + if search != "" { + q.Set("search", search) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pageSize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// TriggerMigration triggers the DevLake database migration endpoint. +func (c *Client) TriggerMigration() error { + resp, err := c.HTTPClient.Get(c.BaseURL + "/proceed-db-migration") + if err != nil { + return fmt.Errorf("triggering migration: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return fmt.Errorf("triggering migration: DevLake returned status %d", resp.StatusCode) + } + return nil +} + +// PipelineListResponse is the response from GET /pipelines. +type PipelineListResponse struct { + Pipelines []Pipeline `json:"pipelines"` + Count int64 `json:"count"` +} + +// ListPipelines returns pipelines with optional query parameters. +// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. +// blueprintID filters by blueprint (0 = no filter). +// page and pageSize control pagination (0 = use defaults). +func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { + path := "/pipelines" + q := url.Values{} + if status != "" { + q.Set("status", status) + } + if blueprintID > 0 { + q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pagesize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[PipelineListResponse](c, path) +} diff --git a/internal/devlake/client_test.go b/internal/devlake/client_test.go index ea5efec..950c128 100644 --- a/internal/devlake/client_test.go +++ b/internal/devlake/client_test.go @@ -909,6 +909,53 @@ func TestHealth(t *testing.T) { } } +func TestTriggerMigration(t *testing.T) { + tests := []struct { + name string + statusCode int + wantErr bool + }{ + { + name: "success", + statusCode: http.StatusOK, + }, + { + name: "no content", + statusCode: http.StatusNoContent, + }, + { + name: "server error", + statusCode: http.StatusServiceUnavailable, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/proceed-db-migration" { + t.Errorf("path = %s, want /proceed-db-migration", r.URL.Path) + } + w.WriteHeader(tt.statusCode) + })) + defer srv.Close() + + client := NewClient(srv.URL) + err := client.TriggerMigration() + + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + // TestTestSavedConnection tests the TestSavedConnection method. func TestTestSavedConnection(t *testing.T) { tests := []struct { From 0735ac2441ab1a09d6ae1a15717fc3c86979faeb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:54:39 +0000 Subject: [PATCH 02/10] fix: preserve line endings in migration changes Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/a286b823-c9cc-41e5-9805-7ad02eb0248b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/deploy_azure.go | 870 ++++++++++++++-------------- internal/devlake/client.go | 1096 ++++++++++++++++++------------------ 2 files changed, 983 insertions(+), 983 deletions(-) diff --git a/cmd/deploy_azure.go b/cmd/deploy_azure.go index 142a141..ff4a7ba 100644 --- a/cmd/deploy_azure.go +++ b/cmd/deploy_azure.go @@ -1,435 +1,435 @@ -package cmd - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/DevExpGBB/gh-devlake/internal/azure" - dockerpkg "github.com/DevExpGBB/gh-devlake/internal/docker" - "github.com/DevExpGBB/gh-devlake/internal/gitclone" - "github.com/DevExpGBB/gh-devlake/internal/prompt" - "github.com/DevExpGBB/gh-devlake/internal/secrets" - "github.com/spf13/cobra" -) - -var ( - azureRG string - azureLocation string - azureBaseName string - azureSkipImageBuild bool - azureRepoURL string - azureOfficial bool - deployAzureDir string - deployAzureQuiet bool // suppress "Next Steps" when called from init wizard -) - -func newDeployAzureCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "azure", - Short: "Deploy DevLake to Azure Container Apps", - Long: `Provisions DevLake on Azure using Container Instances, Azure Database for MySQL, -and (optionally) Azure Container Registry. - -Example: - gh devlake deploy azure --resource-group devlake-rg --location eastus - gh devlake deploy azure --resource-group devlake-rg --location eastus --official`, - RunE: runDeployAzure, - } - - cmd.Flags().StringVar(&azureRG, "resource-group", "", "Azure Resource Group name") - cmd.Flags().StringVar(&azureLocation, "location", "", "Azure region") - cmd.Flags().StringVar(&azureBaseName, "base-name", "devlake", "Base name for Azure resources") - cmd.Flags().BoolVar(&azureSkipImageBuild, "skip-image-build", false, "Skip Docker image building") - cmd.Flags().StringVar(&azureRepoURL, "repo-url", "", "Clone a remote DevLake repository for building") - cmd.Flags().BoolVar(&azureOfficial, "official", false, "Use official Apache images from Docker Hub (no ACR)") - cmd.Flags().StringVar(&deployAzureDir, "dir", ".", "Directory to save deployment state (.devlake-azure.json)") - - return cmd -} - -// Common Azure regions for interactive selection. -var azureRegions = []string{ - "eastus", "eastus2", "westus2", "westus3", - "centralus", "northeurope", "westeurope", - "southeastasia", "australiaeast", "uksouth", -} - -func runDeployAzure(cmd *cobra.Command, args []string) error { - // Suggest a dedicated directory unless already in the right place or called from init - if !deployAzureQuiet { - if suggestDedicatedDir("azure", "gh devlake deploy azure") { - return nil - } - } - if deployAzureDir == "" { - deployAzureDir = "." - } - if err := os.MkdirAll(deployAzureDir, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %w", deployAzureDir, err) - } - - // ── Interactive image-source prompt (when no explicit flag set) ── - if !cmd.Flags().Changed("official") && !cmd.Flags().Changed("repo-url") { - imageChoices := []string{ - "official - Apache DevLake images from Docker Hub (recommended)", - "fork - Clone a DevLake repo and build from source", - "custom - Use a local repo or pre-built images", - } - fmt.Println() - imgChoice := prompt.Select("Which DevLake images to use?", imageChoices) - if imgChoice == "" { - return fmt.Errorf("image choice is required") - } - switch { - case strings.HasPrefix(imgChoice, "official"): - azureOfficial = true - case strings.HasPrefix(imgChoice, "fork"): - azureOfficial = false - if azureRepoURL == "" { - azureRepoURL = prompt.ReadLine(fmt.Sprintf("Repository URL [%s]", gitclone.DefaultForkURL)) - if azureRepoURL == "" { - azureRepoURL = gitclone.DefaultForkURL - } - } - default: // custom - azureOfficial = false - if azureRepoURL == "" { - azureRepoURL = prompt.ReadLine("Path or URL to DevLake repo (leave blank to auto-detect)") - } - } - } - - // ── Interactive prompts for missing required flags ── - if azureLocation == "" { - azureLocation = prompt.SelectWithOther("Select Azure region", azureRegions, true) - if azureLocation == "" { - return fmt.Errorf("--location is required") - } - } - if azureRG == "" { - azureRG = prompt.ReadLine("Resource group name (e.g. devlake-rg)") - if azureRG == "" { - return fmt.Errorf("--resource-group is required") - } - } - - suffix := azure.Suffix(azureRG) - acrName := "devlakeacr" + suffix - - fmt.Println() - if azureOfficial { - printBanner("DevLake Azure Deployment (Official)") - fmt.Println("\nUsing official Apache DevLake images from Docker Hub") - azureSkipImageBuild = true - } else { - printBanner("DevLake Azure Deployment") - } - - fmt.Printf("\n📋 Configuration:\n") - fmt.Printf(" Resource Group: %s\n", azureRG) - fmt.Printf(" Location: %s\n", azureLocation) - fmt.Printf(" Base Name: %s\n", azureBaseName) - if !azureOfficial { - fmt.Printf(" ACR Name: %s\n", acrName) - } else { - fmt.Println(" Images: Official (Docker Hub)") - } - - // ── Check Azure login ── - fmt.Println("\n🔑 Checking Azure CLI login...") - acct, err := azure.CheckLogin() - if err != nil { - fmt.Println(" Not logged in. Running az login...") - if loginErr := azure.Login(); loginErr != nil { - return fmt.Errorf("az login failed: %w", loginErr) - } - acct, err = azure.CheckLogin() - if err != nil { - return fmt.Errorf("still not logged in after az login: %w", err) - } - } - fmt.Printf(" Logged in as: %s\n", acct.User.Name) - - // ── Create Resource Group ── - fmt.Println("\n📦 Creating Resource Group...") - if err := azure.CreateResourceGroup(azureRG, azureLocation); err != nil { - return err - } - fmt.Println(" ✅ Resource Group created") - - // ── Write early checkpoint — ensures cleanup works even if deployment fails ── - savePartialAzureState(azureRG, azureLocation) - - // ── Generate secrets ── - fmt.Println("\n🔐 Generating secrets...") - mysqlPwd, err := secrets.MySQLPassword() - if err != nil { - return err - } - encSecret, err := secrets.EncryptionSecret(32) - if err != nil { - return err - } - fmt.Println(" ✅ Secrets generated") - - // ── Build and push images (if needed) ── - if !azureSkipImageBuild { - repoRoot, err := findRepoRoot() - if err != nil { - return err - } - if azureRepoURL != "" { - defer os.RemoveAll(repoRoot) - if err := applyPoetryPinWorkaround(repoRoot); err != nil { - fmt.Printf(" ⚠️ Could not apply temporary Poetry pin workaround: %v\n", err) - } else { - fmt.Printf(" ⚠️ Applied temporary Poetry pin workaround (poetry==%s) for fork builds\n", poetryWorkaroundVersion) - } - } - fmt.Printf("\n🏗️ Building Docker images from %s...\n", repoRoot) - - // Create ACR (idempotent — safe for re-runs) - fmt.Println(" Creating Container Registry...") - if err := azure.CreateACR(acrName, azureRG, azureLocation); err != nil { - return fmt.Errorf("failed to create ACR: %w", err) - } - fmt.Println(" ✅ Container Registry ready") - - acrServer := acrName + ".azurecr.io" - - fmt.Println("\n Logging into ACR...") - if err := azure.ACRLogin(acrName); err != nil { - return err - } - - images := []struct { - name string - dockerfile string - context string - }{ - {"devlake-backend", "backend/Dockerfile", filepath.Join(repoRoot, "backend")}, - {"devlake-config-ui", "config-ui/Dockerfile", filepath.Join(repoRoot, "config-ui")}, - {"devlake-grafana", "grafana/Dockerfile", filepath.Join(repoRoot, "grafana")}, - } - - for _, img := range images { - fmt.Printf("\n Building %s...\n", img.name) - localTag := img.name + ":latest" - if err := dockerpkg.Build(localTag, filepath.Join(repoRoot, img.dockerfile), img.context); err != nil { - fmt.Fprintf(os.Stderr, "\n ❌ Docker build failed for %s.\n", img.name) - fmt.Fprintf(os.Stderr, " Tip: re-run with --official to skip building and use\n") - fmt.Fprintf(os.Stderr, " official Apache DevLake images from Docker Hub instead.\n") - return fmt.Errorf("docker build failed for %s: %w", img.name, err) - } - remoteTag := acrServer + "/" + localTag - fmt.Printf(" Pushing %s...\n", img.name) - if err := dockerpkg.TagAndPush(localTag, remoteTag); err != nil { - return err - } - } - fmt.Println("\n ✅ All images pushed") - } - - // ── Check MySQL state ── - mysqlName := fmt.Sprintf("%smysql%s", azureBaseName, suffix) - fmt.Println("\n🗄️ Checking MySQL state...") - state, err := azure.MySQLState(mysqlName, azureRG) - if err == nil && state == "Stopped" { - fmt.Println(" MySQL is stopped. Starting...") - if err := azure.MySQLStart(mysqlName, azureRG); err != nil { - fmt.Printf(" ⚠️ Could not start MySQL: %v\n", err) - } else { - fmt.Println(" Waiting 30s for MySQL...") - time.Sleep(30 * time.Second) - fmt.Println(" ✅ MySQL started") - } - } else if state != "" { - fmt.Printf(" MySQL state: %s\n", state) - } else { - fmt.Println(" MySQL not yet created (will be created by Bicep)") - } - - // ── Check for soft-deleted Key Vault ── - kvName := fmt.Sprintf("%skv%s", azureBaseName, suffix) - found, _ := azure.CheckSoftDeletedKeyVault(kvName) - if found { - fmt.Printf("\n🔑 Key Vault %q found in soft-deleted state, purging...\n", kvName) - if err := azure.PurgeKeyVault(kvName, azureLocation); err != nil { - return fmt.Errorf("failed to purge soft-deleted Key Vault %q: %w\nManual fix: az keyvault purge --name %s --location %s", kvName, err, kvName, azureLocation) - } - fmt.Println(" ✅ Key Vault purged") - } - - // ── Deploy infrastructure ── - fmt.Println("\n🚀 Deploying infrastructure with Bicep...") - templateName := "main.bicep" - if azureOfficial { - templateName = "main-official.bicep" - } - templatePath, cleanup, err := azure.WriteTemplate(templateName) - if err != nil { - return err - } - defer cleanup() - - params := map[string]string{ - "baseName": azureBaseName, - "uniqueSuffix": suffix, - "mysqlAdminPassword": mysqlPwd, - "encryptionSecret": encSecret, - } - if !azureOfficial { - params["acrName"] = acrName - } - - deployment, err := azure.DeployBicep(azureRG, templatePath, params) - if err != nil { - return fmt.Errorf("Bicep deployment failed: %w", err) - } - - printBanner("✅ Deployment Complete!") - fmt.Printf("\nEndpoints:\n") - fmt.Printf(" Backend API: %s\n", deployment.BackendEndpoint) - fmt.Printf(" Config UI: %s\n", deployment.ConfigUIEndpoint) - fmt.Printf(" Grafana: %s\n", deployment.GrafanaEndpoint) - - // ── Wait for backend and trigger migration ── - fmt.Println("\n⏳ Waiting for backend to start...") - backendReady := waitForReady(deployment.BackendEndpoint, 30, 10*time.Second) == nil - - if backendReady { - fmt.Println(" ✅ Backend is responding!") - if err := triggerAndWaitForMigration(deployment.BackendEndpoint); err != nil { - fmt.Printf(" ⚠️ %v\n", err) - fmt.Printf(" Trigger migration manually if needed: GET %s/proceed-db-migration\n", deployment.BackendEndpoint) - fmt.Println(" Migration may still be running — proceeding anyway") - } - } else { - fmt.Println(" Backend not ready after 30 attempts.") - fmt.Printf(" Trigger migration manually: GET %s/proceed-db-migration\n", deployment.BackendEndpoint) - } - - // ── Save state file ── - stateFile := filepath.Join(deployAzureDir, ".devlake-azure.json") - containers := []string{ - fmt.Sprintf("%s-backend-%s", azureBaseName, suffix), - fmt.Sprintf("%s-grafana-%s", azureBaseName, suffix), - fmt.Sprintf("%s-ui-%s", azureBaseName, suffix), - } - - kvName = deployment.KeyVaultName - if kvName == "" { - kvName = fmt.Sprintf("%skv%s", azureBaseName, suffix) - } - - // Write a combined state file: Azure-specific metadata + DevLake discovery fields - combinedState := map[string]any{ - "deployedAt": time.Now().Format(time.RFC3339), - "method": methodName(), - "subscription": acct.Name, - "subscriptionId": acct.ID, - "resourceGroup": azureRG, - "region": azureLocation, - "suffix": suffix, - "useOfficialImages": azureOfficial, - "resources": map[string]any{ - "acr": conditionalACR(), - "keyVault": kvName, - "mysql": mysqlName, - "database": "lake", - "containers": containers, - }, - "endpoints": map[string]string{ - "backend": deployment.BackendEndpoint, - "grafana": deployment.GrafanaEndpoint, - "configUi": deployment.ConfigUIEndpoint, - }, - } - - data, _ := json.MarshalIndent(combinedState, "", " ") - if err := os.WriteFile(stateFile, data, 0644); err != nil { - fmt.Fprintf(os.Stderr, "⚠️ Could not save state file: %v\n", err) - } else { - fmt.Printf("\n💾 State saved to %s\n", stateFile) - if deployAzureDir != "." { - fmt.Println(" Next commands should be run from this directory:") - fmt.Println(" PowerShell:") - fmt.Printf(" Set-Location \"%s\"\n", deployAzureDir) - fmt.Println(" Bash/Zsh:") - fmt.Printf(" cd \"%s\"\n", deployAzureDir) - } - } - - if !deployAzureQuiet { - fmt.Println("\nNext steps:") - fmt.Println(" 1. Wait 2-3 minutes for containers to start") - fmt.Printf(" 2. Open Config UI: %s\n", deployment.ConfigUIEndpoint) - fmt.Println(" 3. Configure your data sources") - fmt.Printf("\nTo cleanup: gh devlake cleanup --azure\n") - } - - return nil -} - -func findRepoRoot() (string, error) { - if azureRepoURL != "" { - tmpDir, err := os.MkdirTemp("", "devlake-clone-*") - if err != nil { - return "", err - } - fmt.Printf(" Cloning %s...\n", azureRepoURL) - if err := gitclone.Clone(azureRepoURL, tmpDir); err != nil { - return "", err - } - return tmpDir, nil - } - - // Walk up looking for backend/Dockerfile - dir, _ := os.Getwd() - for dir != "" && dir != filepath.Dir(dir) { - if _, err := os.Stat(filepath.Join(dir, "backend", "Dockerfile")); err == nil { - return dir, nil - } - dir = filepath.Dir(dir) - } - return "", fmt.Errorf("could not find DevLake repo root.\n" + - "Options:\n" + - " --repo-url Clone a fork with the custom Dockerfile\n" + - " --official Use official Apache images (no build needed)") -} - -func methodName() string { - if azureOfficial { - return "bicep-official" - } - return "bicep" -} - -func conditionalACR() any { - if azureOfficial { - return nil - } - return "devlakeacr" + azure.Suffix(azureRG) -} - -// savePartialAzureState writes a minimal state file immediately after the -// Resource Group is created so that cleanup --azure always has a breadcrumb, -// even when the deployment fails mid-flight (e.g. Docker build errors). -// The full state write at the end of a successful deployment overwrites this. -func savePartialAzureState(rg, region string) { - stateFile := ".devlake-azure.json" - partial := map[string]any{ - "deployedAt": time.Now().Format(time.RFC3339), - "resourceGroup": rg, - "region": region, - "partial": true, - } - data, _ := json.MarshalIndent(partial, "", " ") - if err := os.WriteFile(stateFile, data, 0644); err != nil { - fmt.Fprintf(os.Stderr, "⚠️ Could not save early state checkpoint: %v\n", err) - } -} +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/DevExpGBB/gh-devlake/internal/azure" + dockerpkg "github.com/DevExpGBB/gh-devlake/internal/docker" + "github.com/DevExpGBB/gh-devlake/internal/gitclone" + "github.com/DevExpGBB/gh-devlake/internal/prompt" + "github.com/DevExpGBB/gh-devlake/internal/secrets" + "github.com/spf13/cobra" +) + +var ( + azureRG string + azureLocation string + azureBaseName string + azureSkipImageBuild bool + azureRepoURL string + azureOfficial bool + deployAzureDir string + deployAzureQuiet bool // suppress "Next Steps" when called from init wizard +) + +func newDeployAzureCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "azure", + Short: "Deploy DevLake to Azure Container Apps", + Long: `Provisions DevLake on Azure using Container Instances, Azure Database for MySQL, +and (optionally) Azure Container Registry. + +Example: + gh devlake deploy azure --resource-group devlake-rg --location eastus + gh devlake deploy azure --resource-group devlake-rg --location eastus --official`, + RunE: runDeployAzure, + } + + cmd.Flags().StringVar(&azureRG, "resource-group", "", "Azure Resource Group name") + cmd.Flags().StringVar(&azureLocation, "location", "", "Azure region") + cmd.Flags().StringVar(&azureBaseName, "base-name", "devlake", "Base name for Azure resources") + cmd.Flags().BoolVar(&azureSkipImageBuild, "skip-image-build", false, "Skip Docker image building") + cmd.Flags().StringVar(&azureRepoURL, "repo-url", "", "Clone a remote DevLake repository for building") + cmd.Flags().BoolVar(&azureOfficial, "official", false, "Use official Apache images from Docker Hub (no ACR)") + cmd.Flags().StringVar(&deployAzureDir, "dir", ".", "Directory to save deployment state (.devlake-azure.json)") + + return cmd +} + +// Common Azure regions for interactive selection. +var azureRegions = []string{ + "eastus", "eastus2", "westus2", "westus3", + "centralus", "northeurope", "westeurope", + "southeastasia", "australiaeast", "uksouth", +} + +func runDeployAzure(cmd *cobra.Command, args []string) error { + // Suggest a dedicated directory unless already in the right place or called from init + if !deployAzureQuiet { + if suggestDedicatedDir("azure", "gh devlake deploy azure") { + return nil + } + } + if deployAzureDir == "" { + deployAzureDir = "." + } + if err := os.MkdirAll(deployAzureDir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", deployAzureDir, err) + } + + // ── Interactive image-source prompt (when no explicit flag set) ── + if !cmd.Flags().Changed("official") && !cmd.Flags().Changed("repo-url") { + imageChoices := []string{ + "official - Apache DevLake images from Docker Hub (recommended)", + "fork - Clone a DevLake repo and build from source", + "custom - Use a local repo or pre-built images", + } + fmt.Println() + imgChoice := prompt.Select("Which DevLake images to use?", imageChoices) + if imgChoice == "" { + return fmt.Errorf("image choice is required") + } + switch { + case strings.HasPrefix(imgChoice, "official"): + azureOfficial = true + case strings.HasPrefix(imgChoice, "fork"): + azureOfficial = false + if azureRepoURL == "" { + azureRepoURL = prompt.ReadLine(fmt.Sprintf("Repository URL [%s]", gitclone.DefaultForkURL)) + if azureRepoURL == "" { + azureRepoURL = gitclone.DefaultForkURL + } + } + default: // custom + azureOfficial = false + if azureRepoURL == "" { + azureRepoURL = prompt.ReadLine("Path or URL to DevLake repo (leave blank to auto-detect)") + } + } + } + + // ── Interactive prompts for missing required flags ── + if azureLocation == "" { + azureLocation = prompt.SelectWithOther("Select Azure region", azureRegions, true) + if azureLocation == "" { + return fmt.Errorf("--location is required") + } + } + if azureRG == "" { + azureRG = prompt.ReadLine("Resource group name (e.g. devlake-rg)") + if azureRG == "" { + return fmt.Errorf("--resource-group is required") + } + } + + suffix := azure.Suffix(azureRG) + acrName := "devlakeacr" + suffix + + fmt.Println() + if azureOfficial { + printBanner("DevLake Azure Deployment (Official)") + fmt.Println("\nUsing official Apache DevLake images from Docker Hub") + azureSkipImageBuild = true + } else { + printBanner("DevLake Azure Deployment") + } + + fmt.Printf("\n📋 Configuration:\n") + fmt.Printf(" Resource Group: %s\n", azureRG) + fmt.Printf(" Location: %s\n", azureLocation) + fmt.Printf(" Base Name: %s\n", azureBaseName) + if !azureOfficial { + fmt.Printf(" ACR Name: %s\n", acrName) + } else { + fmt.Println(" Images: Official (Docker Hub)") + } + + // ── Check Azure login ── + fmt.Println("\n🔑 Checking Azure CLI login...") + acct, err := azure.CheckLogin() + if err != nil { + fmt.Println(" Not logged in. Running az login...") + if loginErr := azure.Login(); loginErr != nil { + return fmt.Errorf("az login failed: %w", loginErr) + } + acct, err = azure.CheckLogin() + if err != nil { + return fmt.Errorf("still not logged in after az login: %w", err) + } + } + fmt.Printf(" Logged in as: %s\n", acct.User.Name) + + // ── Create Resource Group ── + fmt.Println("\n📦 Creating Resource Group...") + if err := azure.CreateResourceGroup(azureRG, azureLocation); err != nil { + return err + } + fmt.Println(" ✅ Resource Group created") + + // ── Write early checkpoint — ensures cleanup works even if deployment fails ── + savePartialAzureState(azureRG, azureLocation) + + // ── Generate secrets ── + fmt.Println("\n🔐 Generating secrets...") + mysqlPwd, err := secrets.MySQLPassword() + if err != nil { + return err + } + encSecret, err := secrets.EncryptionSecret(32) + if err != nil { + return err + } + fmt.Println(" ✅ Secrets generated") + + // ── Build and push images (if needed) ── + if !azureSkipImageBuild { + repoRoot, err := findRepoRoot() + if err != nil { + return err + } + if azureRepoURL != "" { + defer os.RemoveAll(repoRoot) + if err := applyPoetryPinWorkaround(repoRoot); err != nil { + fmt.Printf(" ⚠️ Could not apply temporary Poetry pin workaround: %v\n", err) + } else { + fmt.Printf(" ⚠️ Applied temporary Poetry pin workaround (poetry==%s) for fork builds\n", poetryWorkaroundVersion) + } + } + fmt.Printf("\n🏗️ Building Docker images from %s...\n", repoRoot) + + // Create ACR (idempotent — safe for re-runs) + fmt.Println(" Creating Container Registry...") + if err := azure.CreateACR(acrName, azureRG, azureLocation); err != nil { + return fmt.Errorf("failed to create ACR: %w", err) + } + fmt.Println(" ✅ Container Registry ready") + + acrServer := acrName + ".azurecr.io" + + fmt.Println("\n Logging into ACR...") + if err := azure.ACRLogin(acrName); err != nil { + return err + } + + images := []struct { + name string + dockerfile string + context string + }{ + {"devlake-backend", "backend/Dockerfile", filepath.Join(repoRoot, "backend")}, + {"devlake-config-ui", "config-ui/Dockerfile", filepath.Join(repoRoot, "config-ui")}, + {"devlake-grafana", "grafana/Dockerfile", filepath.Join(repoRoot, "grafana")}, + } + + for _, img := range images { + fmt.Printf("\n Building %s...\n", img.name) + localTag := img.name + ":latest" + if err := dockerpkg.Build(localTag, filepath.Join(repoRoot, img.dockerfile), img.context); err != nil { + fmt.Fprintf(os.Stderr, "\n ❌ Docker build failed for %s.\n", img.name) + fmt.Fprintf(os.Stderr, " Tip: re-run with --official to skip building and use\n") + fmt.Fprintf(os.Stderr, " official Apache DevLake images from Docker Hub instead.\n") + return fmt.Errorf("docker build failed for %s: %w", img.name, err) + } + remoteTag := acrServer + "/" + localTag + fmt.Printf(" Pushing %s...\n", img.name) + if err := dockerpkg.TagAndPush(localTag, remoteTag); err != nil { + return err + } + } + fmt.Println("\n ✅ All images pushed") + } + + // ── Check MySQL state ── + mysqlName := fmt.Sprintf("%smysql%s", azureBaseName, suffix) + fmt.Println("\n🗄️ Checking MySQL state...") + state, err := azure.MySQLState(mysqlName, azureRG) + if err == nil && state == "Stopped" { + fmt.Println(" MySQL is stopped. Starting...") + if err := azure.MySQLStart(mysqlName, azureRG); err != nil { + fmt.Printf(" ⚠️ Could not start MySQL: %v\n", err) + } else { + fmt.Println(" Waiting 30s for MySQL...") + time.Sleep(30 * time.Second) + fmt.Println(" ✅ MySQL started") + } + } else if state != "" { + fmt.Printf(" MySQL state: %s\n", state) + } else { + fmt.Println(" MySQL not yet created (will be created by Bicep)") + } + + // ── Check for soft-deleted Key Vault ── + kvName := fmt.Sprintf("%skv%s", azureBaseName, suffix) + found, _ := azure.CheckSoftDeletedKeyVault(kvName) + if found { + fmt.Printf("\n🔑 Key Vault %q found in soft-deleted state, purging...\n", kvName) + if err := azure.PurgeKeyVault(kvName, azureLocation); err != nil { + return fmt.Errorf("failed to purge soft-deleted Key Vault %q: %w\nManual fix: az keyvault purge --name %s --location %s", kvName, err, kvName, azureLocation) + } + fmt.Println(" ✅ Key Vault purged") + } + + // ── Deploy infrastructure ── + fmt.Println("\n🚀 Deploying infrastructure with Bicep...") + templateName := "main.bicep" + if azureOfficial { + templateName = "main-official.bicep" + } + templatePath, cleanup, err := azure.WriteTemplate(templateName) + if err != nil { + return err + } + defer cleanup() + + params := map[string]string{ + "baseName": azureBaseName, + "uniqueSuffix": suffix, + "mysqlAdminPassword": mysqlPwd, + "encryptionSecret": encSecret, + } + if !azureOfficial { + params["acrName"] = acrName + } + + deployment, err := azure.DeployBicep(azureRG, templatePath, params) + if err != nil { + return fmt.Errorf("Bicep deployment failed: %w", err) + } + + printBanner("✅ Deployment Complete!") + fmt.Printf("\nEndpoints:\n") + fmt.Printf(" Backend API: %s\n", deployment.BackendEndpoint) + fmt.Printf(" Config UI: %s\n", deployment.ConfigUIEndpoint) + fmt.Printf(" Grafana: %s\n", deployment.GrafanaEndpoint) + + // ── Wait for backend and trigger migration ── + fmt.Println("\n⏳ Waiting for backend to start...") + backendReady := waitForReady(deployment.BackendEndpoint, 30, 10*time.Second) == nil + + if backendReady { + fmt.Println(" ✅ Backend is responding!") + if err := triggerAndWaitForMigration(deployment.BackendEndpoint); err != nil { + fmt.Printf(" ⚠️ %v\n", err) + fmt.Printf(" Trigger migration manually if needed: GET %s/proceed-db-migration\n", deployment.BackendEndpoint) + fmt.Println(" Migration may still be running — proceeding anyway") + } + } else { + fmt.Println(" Backend not ready after 30 attempts.") + fmt.Printf(" Trigger migration manually: GET %s/proceed-db-migration\n", deployment.BackendEndpoint) + } + + // ── Save state file ── + stateFile := filepath.Join(deployAzureDir, ".devlake-azure.json") + containers := []string{ + fmt.Sprintf("%s-backend-%s", azureBaseName, suffix), + fmt.Sprintf("%s-grafana-%s", azureBaseName, suffix), + fmt.Sprintf("%s-ui-%s", azureBaseName, suffix), + } + + kvName = deployment.KeyVaultName + if kvName == "" { + kvName = fmt.Sprintf("%skv%s", azureBaseName, suffix) + } + + // Write a combined state file: Azure-specific metadata + DevLake discovery fields + combinedState := map[string]any{ + "deployedAt": time.Now().Format(time.RFC3339), + "method": methodName(), + "subscription": acct.Name, + "subscriptionId": acct.ID, + "resourceGroup": azureRG, + "region": azureLocation, + "suffix": suffix, + "useOfficialImages": azureOfficial, + "resources": map[string]any{ + "acr": conditionalACR(), + "keyVault": kvName, + "mysql": mysqlName, + "database": "lake", + "containers": containers, + }, + "endpoints": map[string]string{ + "backend": deployment.BackendEndpoint, + "grafana": deployment.GrafanaEndpoint, + "configUi": deployment.ConfigUIEndpoint, + }, + } + + data, _ := json.MarshalIndent(combinedState, "", " ") + if err := os.WriteFile(stateFile, data, 0644); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Could not save state file: %v\n", err) + } else { + fmt.Printf("\n💾 State saved to %s\n", stateFile) + if deployAzureDir != "." { + fmt.Println(" Next commands should be run from this directory:") + fmt.Println(" PowerShell:") + fmt.Printf(" Set-Location \"%s\"\n", deployAzureDir) + fmt.Println(" Bash/Zsh:") + fmt.Printf(" cd \"%s\"\n", deployAzureDir) + } + } + + if !deployAzureQuiet { + fmt.Println("\nNext steps:") + fmt.Println(" 1. Wait 2-3 minutes for containers to start") + fmt.Printf(" 2. Open Config UI: %s\n", deployment.ConfigUIEndpoint) + fmt.Println(" 3. Configure your data sources") + fmt.Printf("\nTo cleanup: gh devlake cleanup --azure\n") + } + + return nil +} + +func findRepoRoot() (string, error) { + if azureRepoURL != "" { + tmpDir, err := os.MkdirTemp("", "devlake-clone-*") + if err != nil { + return "", err + } + fmt.Printf(" Cloning %s...\n", azureRepoURL) + if err := gitclone.Clone(azureRepoURL, tmpDir); err != nil { + return "", err + } + return tmpDir, nil + } + + // Walk up looking for backend/Dockerfile + dir, _ := os.Getwd() + for dir != "" && dir != filepath.Dir(dir) { + if _, err := os.Stat(filepath.Join(dir, "backend", "Dockerfile")); err == nil { + return dir, nil + } + dir = filepath.Dir(dir) + } + return "", fmt.Errorf("could not find DevLake repo root.\n" + + "Options:\n" + + " --repo-url Clone a fork with the custom Dockerfile\n" + + " --official Use official Apache images (no build needed)") +} + +func methodName() string { + if azureOfficial { + return "bicep-official" + } + return "bicep" +} + +func conditionalACR() any { + if azureOfficial { + return nil + } + return "devlakeacr" + azure.Suffix(azureRG) +} + +// savePartialAzureState writes a minimal state file immediately after the +// Resource Group is created so that cleanup --azure always has a breadcrumb, +// even when the deployment fails mid-flight (e.g. Docker build errors). +// The full state write at the end of a successful deployment overwrites this. +func savePartialAzureState(rg, region string) { + stateFile := ".devlake-azure.json" + partial := map[string]any{ + "deployedAt": time.Now().Format(time.RFC3339), + "resourceGroup": rg, + "region": region, + "partial": true, + } + data, _ := json.MarshalIndent(partial, "", " ") + if err := os.WriteFile(stateFile, data, 0644); err != nil { + fmt.Fprintf(os.Stderr, "⚠️ Could not save early state checkpoint: %v\n", err) + } +} diff --git a/internal/devlake/client.go b/internal/devlake/client.go index c9a3cd7..c77a557 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -1,548 +1,548 @@ -// Package devlake provides an HTTP client for the DevLake REST API. -package devlake - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "time" -) - -// Client wraps HTTP calls to the DevLake backend API. -type Client struct { - BaseURL string - HTTPClient *http.Client -} - -// NewClient creates a Client for the given base URL. -func NewClient(baseURL string) *Client { - return &Client{ - BaseURL: baseURL, - HTTPClient: &http.Client{ - Timeout: 90 * time.Second, - }, - } -} - -// Ping checks if the DevLake backend is reachable. -func (c *Client) Ping() error { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) - } - return nil -} - -// Connection represents a DevLake plugin connection. -type Connection struct { - ID int `json:"id"` - Name string `json:"name"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. -// Fields with omitempty are only included in the request when non-empty, -// enabling sparse updates (only changed fields are sent). -type ConnectionUpdateRequest struct { - Name string `json:"name,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionCreateRequest is the payload for creating a plugin connection. -type ConnectionCreateRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` - TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` - RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` -} - -// ConnectionTestRequest is the payload for testing a connection before creating. -type ConnectionTestRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Proxy string `json:"proxy"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionTestResult is the response from testing a connection. -type ConnectionTestResult struct { - Success bool `json:"success"` - Message string `json:"message"` -} - -// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). -func (c *Client) ListConnections(plugin string) ([]Connection, error) { - resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) - } - - var conns []Connection - if err := json.Unmarshal(body, &conns); err != nil { - return nil, err - } - return conns, nil -} - -// FindConnectionByName returns the first connection matching the given name, or nil. -func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { - conns, err := c.ListConnections(plugin) - if err != nil { - return nil, err - } - for _, conn := range conns { - if conn.Name == name { - return &conn, nil - } - } - return nil, nil -} - -// TestConnection tests connection parameters before creating. -func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { - return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) -} - -// CreateConnection creates a new connection for the given plugin. -func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { - return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) -} - -// DeleteConnection deletes a plugin connection by ID. -func (c *Client) DeleteConnection(plugin string, connID int) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) - } - return nil -} - -// TestSavedConnection tests an already-created connection by ID. -func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) - - reqBody := bytes.NewBufferString("{}") - resp, err := c.HTTPClient.Post(url, "application/json", reqBody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var result ConnectionTestResult - if err := json.Unmarshal(body, &result); err != nil { - // Non-JSON response is ok — treat as success if status 200 - if resp.StatusCode == http.StatusOK { - return &ConnectionTestResult{Success: true}, nil - } - return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) - } - return &result, nil -} - -// GetConnection retrieves a single connection by plugin and ID. -func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { - return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) -} - -// UpdateConnection patches an existing connection for the given plugin. -func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { - return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) -} - -// HealthStatus represents the response from /health or /ping. -type HealthStatus struct { - Status string `json:"status"` -} - -// Health returns the DevLake health status. -func (c *Client) Health() (*HealthStatus, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var hs HealthStatus - _ = json.Unmarshal(body, &hs) - if resp.StatusCode == http.StatusOK { - if hs.Status == "" { - hs.Status = "ok" - } - return &hs, nil - } - return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) -} - -// doPost is a generic helper for POST requests that return JSON. -func doPost[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - url := c.BaseURL + path - resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doGet is a generic helper for GET requests that return JSON. -func doGet[T any](c *Client, path string) (*T, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + path) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPut is a generic helper for PUT requests that return JSON. -func doPut[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPatch is a generic helper for PATCH requests that return JSON. -func doPatch[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// CreateScopeConfig creates a scope config for a plugin connection. -func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { - return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) -} - -// ListScopeConfigs returns all scope configs for a plugin connection. -func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { - result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) - if err != nil { - return nil, err - } - return *result, nil -} - -// PutScopes batch-upserts scopes for a plugin connection. -func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { - _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) - return err -} - -// ListScopes returns the scopes configured on a plugin connection. -func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { - return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) -} - -// ListProjects returns all DevLake projects. -func (c *Client) ListProjects() ([]Project, error) { - result, err := doGet[ProjectListResponse](c, "/projects") - if err != nil { - return nil, err - } - return result.Projects, nil -} - -// DeleteProject deletes a project by name. -func (c *Client) DeleteProject(name string) error { - url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("project not found: %s", name) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) - } - return nil -} - -// DeleteScope removes a scope from a plugin connection. -func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) - } - return nil -} - -// CreateProject creates a new DevLake project. -func (c *Client) CreateProject(project *Project) (*Project, error) { - return doPost[Project](c, "/projects", project) -} - -// GetProject retrieves a project by name. -func (c *Client) GetProject(name string) (*Project, error) { - return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) -} - -// PatchBlueprint updates a blueprint by ID. -func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { - return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) -} - -// TriggerBlueprint triggers a blueprint to run and returns the pipeline. -func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { - return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) -} - -// GetPipeline retrieves a pipeline by ID. -func (c *Client) GetPipeline(id int) (*Pipeline, error) { - return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) -} - -// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. -// groupID and pageToken are optional (pass "" to omit). -func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) - q := url.Values{} - if groupID != "" { - q.Set("groupId", groupID) - } - if pageToken != "" { - q.Set("pageToken", pageToken) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. -// page and pageSize control pagination; pass 0 to use DevLake defaults. -func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) - q := url.Values{} - if search != "" { - q.Set("search", search) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pageSize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// TriggerMigration triggers the DevLake database migration endpoint. -func (c *Client) TriggerMigration() error { - resp, err := c.HTTPClient.Get(c.BaseURL + "/proceed-db-migration") - if err != nil { - return fmt.Errorf("triggering migration: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return fmt.Errorf("triggering migration: DevLake returned status %d", resp.StatusCode) - } - return nil -} - -// PipelineListResponse is the response from GET /pipelines. -type PipelineListResponse struct { - Pipelines []Pipeline `json:"pipelines"` - Count int64 `json:"count"` -} - -// ListPipelines returns pipelines with optional query parameters. -// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. -// blueprintID filters by blueprint (0 = no filter). -// page and pageSize control pagination (0 = use defaults). -func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { - path := "/pipelines" - q := url.Values{} - if status != "" { - q.Set("status", status) - } - if blueprintID > 0 { - q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pagesize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[PipelineListResponse](c, path) -} +// Package devlake provides an HTTP client for the DevLake REST API. +package devlake + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// Client wraps HTTP calls to the DevLake backend API. +type Client struct { + BaseURL string + HTTPClient *http.Client +} + +// NewClient creates a Client for the given base URL. +func NewClient(baseURL string) *Client { + return &Client{ + BaseURL: baseURL, + HTTPClient: &http.Client{ + Timeout: 90 * time.Second, + }, + } +} + +// Ping checks if the DevLake backend is reachable. +func (c *Client) Ping() error { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) + } + return nil +} + +// Connection represents a DevLake plugin connection. +type Connection struct { + ID int `json:"id"` + Name string `json:"name"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. +// Fields with omitempty are only included in the request when non-empty, +// enabling sparse updates (only changed fields are sent). +type ConnectionUpdateRequest struct { + Name string `json:"name,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionCreateRequest is the payload for creating a plugin connection. +type ConnectionCreateRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` + TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` + RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` +} + +// ConnectionTestRequest is the payload for testing a connection before creating. +type ConnectionTestRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Proxy string `json:"proxy"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionTestResult is the response from testing a connection. +type ConnectionTestResult struct { + Success bool `json:"success"` + Message string `json:"message"` +} + +// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). +func (c *Client) ListConnections(plugin string) ([]Connection, error) { + resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) + } + + var conns []Connection + if err := json.Unmarshal(body, &conns); err != nil { + return nil, err + } + return conns, nil +} + +// FindConnectionByName returns the first connection matching the given name, or nil. +func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { + conns, err := c.ListConnections(plugin) + if err != nil { + return nil, err + } + for _, conn := range conns { + if conn.Name == name { + return &conn, nil + } + } + return nil, nil +} + +// TestConnection tests connection parameters before creating. +func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { + return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) +} + +// CreateConnection creates a new connection for the given plugin. +func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { + return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) +} + +// DeleteConnection deletes a plugin connection by ID. +func (c *Client) DeleteConnection(plugin string, connID int) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) + } + return nil +} + +// TestSavedConnection tests an already-created connection by ID. +func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) + + reqBody := bytes.NewBufferString("{}") + resp, err := c.HTTPClient.Post(url, "application/json", reqBody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var result ConnectionTestResult + if err := json.Unmarshal(body, &result); err != nil { + // Non-JSON response is ok — treat as success if status 200 + if resp.StatusCode == http.StatusOK { + return &ConnectionTestResult{Success: true}, nil + } + return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) + } + return &result, nil +} + +// GetConnection retrieves a single connection by plugin and ID. +func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { + return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) +} + +// UpdateConnection patches an existing connection for the given plugin. +func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { + return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) +} + +// HealthStatus represents the response from /health or /ping. +type HealthStatus struct { + Status string `json:"status"` +} + +// Health returns the DevLake health status. +func (c *Client) Health() (*HealthStatus, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var hs HealthStatus + _ = json.Unmarshal(body, &hs) + if resp.StatusCode == http.StatusOK { + if hs.Status == "" { + hs.Status = "ok" + } + return &hs, nil + } + return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) +} + +// doPost is a generic helper for POST requests that return JSON. +func doPost[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + url := c.BaseURL + path + resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doGet is a generic helper for GET requests that return JSON. +func doGet[T any](c *Client, path string) (*T, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + path) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPut is a generic helper for PUT requests that return JSON. +func doPut[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPatch is a generic helper for PATCH requests that return JSON. +func doPatch[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// CreateScopeConfig creates a scope config for a plugin connection. +func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { + return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) +} + +// ListScopeConfigs returns all scope configs for a plugin connection. +func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { + result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) + if err != nil { + return nil, err + } + return *result, nil +} + +// PutScopes batch-upserts scopes for a plugin connection. +func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { + _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) + return err +} + +// ListScopes returns the scopes configured on a plugin connection. +func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { + return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) +} + +// ListProjects returns all DevLake projects. +func (c *Client) ListProjects() ([]Project, error) { + result, err := doGet[ProjectListResponse](c, "/projects") + if err != nil { + return nil, err + } + return result.Projects, nil +} + +// DeleteProject deletes a project by name. +func (c *Client) DeleteProject(name string) error { + url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("project not found: %s", name) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) + } + return nil +} + +// DeleteScope removes a scope from a plugin connection. +func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) + } + return nil +} + +// CreateProject creates a new DevLake project. +func (c *Client) CreateProject(project *Project) (*Project, error) { + return doPost[Project](c, "/projects", project) +} + +// GetProject retrieves a project by name. +func (c *Client) GetProject(name string) (*Project, error) { + return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) +} + +// PatchBlueprint updates a blueprint by ID. +func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { + return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) +} + +// TriggerBlueprint triggers a blueprint to run and returns the pipeline. +func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { + return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) +} + +// GetPipeline retrieves a pipeline by ID. +func (c *Client) GetPipeline(id int) (*Pipeline, error) { + return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) +} + +// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. +// groupID and pageToken are optional (pass "" to omit). +func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) + q := url.Values{} + if groupID != "" { + q.Set("groupId", groupID) + } + if pageToken != "" { + q.Set("pageToken", pageToken) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. +// page and pageSize control pagination; pass 0 to use DevLake defaults. +func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) + q := url.Values{} + if search != "" { + q.Set("search", search) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pageSize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// TriggerMigration triggers the DevLake database migration endpoint. +func (c *Client) TriggerMigration() error { + resp, err := c.HTTPClient.Get(c.BaseURL + "/proceed-db-migration") + if err != nil { + return fmt.Errorf("triggering migration: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return fmt.Errorf("triggering migration: DevLake returned status %d", resp.StatusCode) + } + return nil +} + +// PipelineListResponse is the response from GET /pipelines. +type PipelineListResponse struct { + Pipelines []Pipeline `json:"pipelines"` + Count int64 `json:"count"` +} + +// ListPipelines returns pipelines with optional query parameters. +// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. +// blueprintID filters by blueprint (0 = no filter). +// page and pageSize control pagination (0 = use defaults). +func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { + path := "/pipelines" + q := url.Values{} + if status != "" { + q.Set("status", status) + } + if blueprintID > 0 { + q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pagesize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[PipelineListResponse](c, path) +} From f289fcd8663f72028df7fe8277eda4ee72c88a91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:55:38 +0000 Subject: [PATCH 03/10] fix: clean up migration error wrapping Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/a286b823-c9cc-41e5-9805-7ad02eb0248b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/helpers.go b/cmd/helpers.go index 1466804..7e6335a 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -274,7 +274,7 @@ func triggerAndWaitForMigrationWithClient(baseURL string, migClient *devlake.Cli } if err := waitForMigration(baseURL, waitAttempts, waitInterval); err != nil { if lastErr != nil { - return fmt.Errorf("migration trigger did not complete cleanly: %w; %w", lastErr, err) + return fmt.Errorf("migration trigger did not complete cleanly: %v; %w", lastErr, err) } return err } From bbbc0874ba8938490ca80382c94139887aee4b6a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:57:24 +0000 Subject: [PATCH 04/10] refactor: polish migration retry messaging Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/a286b823-c9cc-41e5-9805-7ad02eb0248b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/helpers.go | 14 ++++++------ cmd/helpers_migration_test.go | 8 +++++-- internal/devlake/client.go | 42 ++++++++++++++++++++--------------- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/cmd/helpers.go b/cmd/helpers.go index 7e6335a..9e8d610 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -232,11 +232,11 @@ func waitForMigration(baseURL string, maxAttempts int, interval time.Duration) e return nil } } + statusSuffix := "" if lastStatus != 0 { - fmt.Printf(" Migrating... (%d/%d, status=%d)\n", attempt, maxAttempts, lastStatus) - } else { - fmt.Printf(" Migrating... (%d/%d)\n", attempt, maxAttempts) + statusSuffix = fmt.Sprintf(", status=%d", lastStatus) } + fmt.Printf(" Migrating... (%d/%d%s)\n", attempt, maxAttempts, statusSuffix) time.Sleep(interval) } if lastStatus != 0 { @@ -249,18 +249,18 @@ func triggerAndWaitForMigration(baseURL string) error { return triggerAndWaitForMigrationWithClient(baseURL, devlake.NewClient(baseURL), 3, 10*time.Second, 60, 5*time.Second) } -func triggerAndWaitForMigrationWithClient(baseURL string, migClient *devlake.Client, triggerAttempts int, triggerInterval time.Duration, waitAttempts int, waitInterval time.Duration) error { +func triggerAndWaitForMigrationWithClient(baseURL string, devlakeClient *devlake.Client, triggerAttempts int, triggerInterval time.Duration, waitAttempts int, waitInterval time.Duration) error { fmt.Println("\n🔄 Triggering database migration...") var lastErr error for attempt := 1; attempt <= triggerAttempts; attempt++ { - if err := migClient.TriggerMigration(); err == nil { + if err := devlakeClient.TriggerMigration(); err == nil { fmt.Println(" ✅ Migration triggered") lastErr = nil break } else { lastErr = err - fmt.Printf(" ⚠️ Trigger attempt %d/%d did not complete cleanly: %v\n", attempt, triggerAttempts, err) + fmt.Printf(" ⚠️ Trigger attempt %d/%d failed: %v\n", attempt, triggerAttempts, err) if attempt < triggerAttempts { fmt.Println(" DevLake may still be starting or migration may already be running — retrying...") time.Sleep(triggerInterval) @@ -274,7 +274,7 @@ func triggerAndWaitForMigrationWithClient(baseURL string, migClient *devlake.Cli } if err := waitForMigration(baseURL, waitAttempts, waitInterval); err != nil { if lastErr != nil { - return fmt.Errorf("migration trigger did not complete cleanly: %v; %w", lastErr, err) + return fmt.Errorf("migration trigger failed: %v; %w", lastErr, err) } return err } diff --git a/cmd/helpers_migration_test.go b/cmd/helpers_migration_test.go index 03c4434..35a2105 100644 --- a/cmd/helpers_migration_test.go +++ b/cmd/helpers_migration_test.go @@ -32,8 +32,12 @@ func TestTriggerAndWaitForMigrationWithClient_CompletesAfterTriggerTimeout(t *te })) defer srv.Close() - client := devlake.NewClient(srv.URL) - client.HTTPClient.Timeout = 5 * time.Millisecond + client := &devlake.Client{ + BaseURL: srv.URL, + HTTPClient: &http.Client{ + Timeout: 5 * time.Millisecond, + }, + } err := triggerAndWaitForMigrationWithClient(srv.URL, client, 1, time.Millisecond, 3, time.Millisecond) if err != nil { diff --git a/internal/devlake/client.go b/internal/devlake/client.go index c77a557..682f45d 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -1,15 +1,16 @@ // Package devlake provides an HTTP client for the DevLake REST API. package devlake -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "time" -) +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) // Client wraps HTTP calls to the DevLake backend API. type Client struct { @@ -506,15 +507,20 @@ func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, pa // TriggerMigration triggers the DevLake database migration endpoint. func (c *Client) TriggerMigration() error { resp, err := c.HTTPClient.Get(c.BaseURL + "/proceed-db-migration") - if err != nil { - return fmt.Errorf("triggering migration: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return fmt.Errorf("triggering migration: DevLake returned status %d", resp.StatusCode) - } - return nil -} + if err != nil { + return fmt.Errorf("triggering migration: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + bodyText := strings.TrimSpace(string(body)) + if bodyText != "" { + return fmt.Errorf("triggering migration: DevLake returned status %d: %s", resp.StatusCode, bodyText) + } + return fmt.Errorf("triggering migration: DevLake returned status %d", resp.StatusCode) + } + return nil +} // PipelineListResponse is the response from GET /pipelines. type PipelineListResponse struct { From 63a85e03ddf19b5de79b2ea7dfbda129b54faf36 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:58:16 +0000 Subject: [PATCH 05/10] fix: finalize migration retry errors Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/a286b823-c9cc-41e5-9805-7ad02eb0248b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/helpers.go | 3 +-- internal/devlake/client.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/helpers.go b/cmd/helpers.go index 9e8d610..0a60f9f 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -256,7 +256,6 @@ func triggerAndWaitForMigrationWithClient(baseURL string, devlakeClient *devlake for attempt := 1; attempt <= triggerAttempts; attempt++ { if err := devlakeClient.TriggerMigration(); err == nil { fmt.Println(" ✅ Migration triggered") - lastErr = nil break } else { lastErr = err @@ -274,7 +273,7 @@ func triggerAndWaitForMigrationWithClient(baseURL string, devlakeClient *devlake } if err := waitForMigration(baseURL, waitAttempts, waitInterval); err != nil { if lastErr != nil { - return fmt.Errorf("migration trigger failed: %v; %w", lastErr, err) + return fmt.Errorf("migration trigger failed earlier (%v) and waiting for migration completion also failed: %w", lastErr, err) } return err } diff --git a/internal/devlake/client.go b/internal/devlake/client.go index 682f45d..e868567 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -515,9 +515,9 @@ func (c *Client) TriggerMigration() error { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) bodyText := strings.TrimSpace(string(body)) if bodyText != "" { - return fmt.Errorf("triggering migration: DevLake returned status %d: %s", resp.StatusCode, bodyText) + return fmt.Errorf("DevLake returned status %d: %s", resp.StatusCode, bodyText) } - return fmt.Errorf("triggering migration: DevLake returned status %d", resp.StatusCode) + return fmt.Errorf("DevLake returned status %d", resp.StatusCode) } return nil } From edb7da91206db87b5de3dab151cab10c6aa7c059 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:59:05 +0000 Subject: [PATCH 06/10] refactor: simplify migration status errors Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/a286b823-c9cc-41e5-9805-7ad02eb0248b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/helpers.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/helpers.go b/cmd/helpers.go index 0a60f9f..14bdc43 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -239,10 +239,11 @@ func waitForMigration(baseURL string, maxAttempts int, interval time.Duration) e fmt.Printf(" Migrating... (%d/%d%s)\n", attempt, maxAttempts, statusSuffix) time.Sleep(interval) } + statusSuffix := "" if lastStatus != 0 { - return fmt.Errorf("migration did not complete after %d attempts (last status %d)", maxAttempts, lastStatus) + statusSuffix = fmt.Sprintf(" (last status %d)", lastStatus) } - return fmt.Errorf("migration did not complete after %d attempts", maxAttempts) + return fmt.Errorf("migration did not complete after %d attempts%s", maxAttempts, statusSuffix) } func triggerAndWaitForMigration(baseURL string) error { From 4b4825fcccc65b3ad979ce82f3389dcbad3cc2df Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:00:06 +0000 Subject: [PATCH 07/10] refactor: flatten migration retry loop Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/a286b823-c9cc-41e5-9805-7ad02eb0248b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/helpers.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/helpers.go b/cmd/helpers.go index 14bdc43..f21580b 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -255,16 +255,16 @@ func triggerAndWaitForMigrationWithClient(baseURL string, devlakeClient *devlake var lastErr error for attempt := 1; attempt <= triggerAttempts; attempt++ { - if err := devlakeClient.TriggerMigration(); err == nil { + err := devlakeClient.TriggerMigration() + if err == nil { fmt.Println(" ✅ Migration triggered") break - } else { - lastErr = err - fmt.Printf(" ⚠️ Trigger attempt %d/%d failed: %v\n", attempt, triggerAttempts, err) - if attempt < triggerAttempts { - fmt.Println(" DevLake may still be starting or migration may already be running — retrying...") - time.Sleep(triggerInterval) - } + } + lastErr = err + fmt.Printf(" ⚠️ Trigger attempt %d/%d failed: %v\n", attempt, triggerAttempts, err) + if attempt < triggerAttempts { + fmt.Println(" DevLake may still be starting or migration may already be running — retrying...") + time.Sleep(triggerInterval) } } From 4cbb5e27ab56d0b206e2c84e37cf2d3baa6fd7cc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:24:52 +0000 Subject: [PATCH 08/10] fix: address migration review feedback Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/f7542969-e4d0-4329-bfa1-0cb12581ab4b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/helpers.go | 7 +- cmd/helpers_migration_test.go | 47 +- internal/devlake/client.go | 1070 ++++++++++++++++--------------- internal/devlake/client_test.go | 26 +- 4 files changed, 605 insertions(+), 545 deletions(-) diff --git a/cmd/helpers.go b/cmd/helpers.go index f21580b..fc38a65 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -247,16 +247,17 @@ func waitForMigration(baseURL string, maxAttempts int, interval time.Duration) e } func triggerAndWaitForMigration(baseURL string) error { - return triggerAndWaitForMigrationWithClient(baseURL, devlake.NewClient(baseURL), 3, 10*time.Second, 60, 5*time.Second) + return triggerAndWaitForMigrationWithClient(devlake.NewClient(baseURL), 3, 10*time.Second, 60, 5*time.Second) } -func triggerAndWaitForMigrationWithClient(baseURL string, devlakeClient *devlake.Client, triggerAttempts int, triggerInterval time.Duration, waitAttempts int, waitInterval time.Duration) error { +func triggerAndWaitForMigrationWithClient(devlakeClient *devlake.Client, triggerAttempts int, triggerInterval time.Duration, waitAttempts int, waitInterval time.Duration) error { fmt.Println("\n🔄 Triggering database migration...") var lastErr error for attempt := 1; attempt <= triggerAttempts; attempt++ { err := devlakeClient.TriggerMigration() if err == nil { + lastErr = nil fmt.Println(" ✅ Migration triggered") break } @@ -272,7 +273,7 @@ func triggerAndWaitForMigrationWithClient(baseURL string, devlakeClient *devlake if lastErr != nil { fmt.Println(" Continuing to monitor migration status anyway...") } - if err := waitForMigration(baseURL, waitAttempts, waitInterval); err != nil { + if err := waitForMigration(devlakeClient.BaseURL, waitAttempts, waitInterval); err != nil { if lastErr != nil { return fmt.Errorf("migration trigger failed earlier (%v) and waiting for migration completion also failed: %w", lastErr, err) } diff --git a/cmd/helpers_migration_test.go b/cmd/helpers_migration_test.go index 35a2105..aeb2bbb 100644 --- a/cmd/helpers_migration_test.go +++ b/cmd/helpers_migration_test.go @@ -3,6 +3,7 @@ package cmd import ( "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -39,7 +40,7 @@ func TestTriggerAndWaitForMigrationWithClient_CompletesAfterTriggerTimeout(t *te }, } - err := triggerAndWaitForMigrationWithClient(srv.URL, client, 1, time.Millisecond, 3, time.Millisecond) + err := triggerAndWaitForMigrationWithClient(client, 1, time.Millisecond, 3, time.Millisecond) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -75,7 +76,7 @@ func TestTriggerAndWaitForMigrationWithClient_RetriesBeforeWaiting(t *testing.T) client := devlake.NewClient(srv.URL) - err := triggerAndWaitForMigrationWithClient(srv.URL, client, 2, time.Millisecond, 2, time.Millisecond) + err := triggerAndWaitForMigrationWithClient(client, 2, time.Millisecond, 2, time.Millisecond) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -86,3 +87,45 @@ func TestTriggerAndWaitForMigrationWithClient_RetriesBeforeWaiting(t *testing.T) t.Fatalf("ping calls = %d, want 1", pingCalls) } } + +func TestTriggerAndWaitForMigrationWithClient_TriggerEventuallySucceedsBeforeWaitFails(t *testing.T) { + triggerCalls := 0 + pingCalls := 0 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/proceed-db-migration": + triggerCalls++ + if triggerCalls == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + case "/ping": + pingCalls++ + w.WriteHeader(http.StatusPreconditionRequired) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + client := devlake.NewClient(srv.URL) + + err := triggerAndWaitForMigrationWithClient(client, 2, 5*time.Millisecond, 2, 5*time.Millisecond) + if err == nil { + t.Fatal("expected error, got nil") + } + if strings.Contains(err.Error(), "migration trigger failed earlier") { + t.Fatalf("unexpected trigger failure in error: %v", err) + } + if !strings.Contains(err.Error(), "migration did not complete after 2 attempts") { + t.Fatalf("expected wait failure in error, got: %v", err) + } + if triggerCalls != 2 { + t.Fatalf("trigger calls = %d, want 2", triggerCalls) + } + if pingCalls != 2 { + t.Fatalf("ping calls = %d, want 2", pingCalls) + } +} diff --git a/internal/devlake/client.go b/internal/devlake/client.go index e868567..687bbd2 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -1,6 +1,6 @@ -// Package devlake provides an HTTP client for the DevLake REST API. -package devlake - +// Package devlake provides an HTTP client for the DevLake REST API. +package devlake + import ( "bytes" "encoding/json" @@ -11,544 +11,546 @@ import ( "strings" "time" ) - -// Client wraps HTTP calls to the DevLake backend API. -type Client struct { - BaseURL string - HTTPClient *http.Client -} - -// NewClient creates a Client for the given base URL. -func NewClient(baseURL string) *Client { - return &Client{ - BaseURL: baseURL, - HTTPClient: &http.Client{ - Timeout: 90 * time.Second, - }, - } -} - -// Ping checks if the DevLake backend is reachable. -func (c *Client) Ping() error { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) - } - return nil -} - -// Connection represents a DevLake plugin connection. -type Connection struct { - ID int `json:"id"` - Name string `json:"name"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. -// Fields with omitempty are only included in the request when non-empty, -// enabling sparse updates (only changed fields are sent). -type ConnectionUpdateRequest struct { - Name string `json:"name,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionCreateRequest is the payload for creating a plugin connection. -type ConnectionCreateRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` - TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` - RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` -} - -// ConnectionTestRequest is the payload for testing a connection before creating. -type ConnectionTestRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Proxy string `json:"proxy"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionTestResult is the response from testing a connection. -type ConnectionTestResult struct { - Success bool `json:"success"` - Message string `json:"message"` -} - -// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). -func (c *Client) ListConnections(plugin string) ([]Connection, error) { - resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) - } - - var conns []Connection - if err := json.Unmarshal(body, &conns); err != nil { - return nil, err - } - return conns, nil -} - -// FindConnectionByName returns the first connection matching the given name, or nil. -func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { - conns, err := c.ListConnections(plugin) - if err != nil { - return nil, err - } - for _, conn := range conns { - if conn.Name == name { - return &conn, nil - } - } - return nil, nil -} - -// TestConnection tests connection parameters before creating. -func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { - return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) -} - -// CreateConnection creates a new connection for the given plugin. -func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { - return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) -} - -// DeleteConnection deletes a plugin connection by ID. -func (c *Client) DeleteConnection(plugin string, connID int) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) - } - return nil -} - -// TestSavedConnection tests an already-created connection by ID. -func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) - - reqBody := bytes.NewBufferString("{}") - resp, err := c.HTTPClient.Post(url, "application/json", reqBody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var result ConnectionTestResult - if err := json.Unmarshal(body, &result); err != nil { - // Non-JSON response is ok — treat as success if status 200 - if resp.StatusCode == http.StatusOK { - return &ConnectionTestResult{Success: true}, nil - } - return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) - } - return &result, nil -} - -// GetConnection retrieves a single connection by plugin and ID. -func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { - return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) -} - -// UpdateConnection patches an existing connection for the given plugin. -func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { - return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) -} - -// HealthStatus represents the response from /health or /ping. -type HealthStatus struct { - Status string `json:"status"` -} - -// Health returns the DevLake health status. -func (c *Client) Health() (*HealthStatus, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var hs HealthStatus - _ = json.Unmarshal(body, &hs) - if resp.StatusCode == http.StatusOK { - if hs.Status == "" { - hs.Status = "ok" - } - return &hs, nil - } - return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) -} - -// doPost is a generic helper for POST requests that return JSON. -func doPost[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - url := c.BaseURL + path - resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doGet is a generic helper for GET requests that return JSON. -func doGet[T any](c *Client, path string) (*T, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + path) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPut is a generic helper for PUT requests that return JSON. -func doPut[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPatch is a generic helper for PATCH requests that return JSON. -func doPatch[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// CreateScopeConfig creates a scope config for a plugin connection. -func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { - return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) -} - -// ListScopeConfigs returns all scope configs for a plugin connection. -func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { - result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) - if err != nil { - return nil, err - } - return *result, nil -} - -// PutScopes batch-upserts scopes for a plugin connection. -func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { - _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) - return err -} - -// ListScopes returns the scopes configured on a plugin connection. -func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { - return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) -} - -// ListProjects returns all DevLake projects. -func (c *Client) ListProjects() ([]Project, error) { - result, err := doGet[ProjectListResponse](c, "/projects") - if err != nil { - return nil, err - } - return result.Projects, nil -} - -// DeleteProject deletes a project by name. -func (c *Client) DeleteProject(name string) error { - url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("project not found: %s", name) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) - } - return nil -} - -// DeleteScope removes a scope from a plugin connection. -func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) - } - return nil -} - -// CreateProject creates a new DevLake project. -func (c *Client) CreateProject(project *Project) (*Project, error) { - return doPost[Project](c, "/projects", project) -} - -// GetProject retrieves a project by name. -func (c *Client) GetProject(name string) (*Project, error) { - return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) -} - -// PatchBlueprint updates a blueprint by ID. -func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { - return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) -} - -// TriggerBlueprint triggers a blueprint to run and returns the pipeline. -func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { - return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) -} - -// GetPipeline retrieves a pipeline by ID. -func (c *Client) GetPipeline(id int) (*Pipeline, error) { - return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) -} - -// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. -// groupID and pageToken are optional (pass "" to omit). -func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) - q := url.Values{} - if groupID != "" { - q.Set("groupId", groupID) - } - if pageToken != "" { - q.Set("pageToken", pageToken) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. -// page and pageSize control pagination; pass 0 to use DevLake defaults. -func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) - q := url.Values{} - if search != "" { - q.Set("search", search) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pageSize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// TriggerMigration triggers the DevLake database migration endpoint. -func (c *Client) TriggerMigration() error { - resp, err := c.HTTPClient.Get(c.BaseURL + "/proceed-db-migration") - if err != nil { - return fmt.Errorf("triggering migration: %w", err) + +// Client wraps HTTP calls to the DevLake backend API. +type Client struct { + BaseURL string + HTTPClient *http.Client +} + +// NewClient creates a Client for the given base URL. +func NewClient(baseURL string) *Client { + return &Client{ + BaseURL: baseURL, + HTTPClient: &http.Client{ + Timeout: 90 * time.Second, + }, + } +} + +// Ping checks if the DevLake backend is reachable. +func (c *Client) Ping() error { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) + } + return nil +} + +// Connection represents a DevLake plugin connection. +type Connection struct { + ID int `json:"id"` + Name string `json:"name"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. +// Fields with omitempty are only included in the request when non-empty, +// enabling sparse updates (only changed fields are sent). +type ConnectionUpdateRequest struct { + Name string `json:"name,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionCreateRequest is the payload for creating a plugin connection. +type ConnectionCreateRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` + TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` + RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` +} + +// ConnectionTestRequest is the payload for testing a connection before creating. +type ConnectionTestRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Proxy string `json:"proxy"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionTestResult is the response from testing a connection. +type ConnectionTestResult struct { + Success bool `json:"success"` + Message string `json:"message"` +} + +// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). +func (c *Client) ListConnections(plugin string) ([]Connection, error) { + resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) + } + + var conns []Connection + if err := json.Unmarshal(body, &conns); err != nil { + return nil, err + } + return conns, nil +} + +// FindConnectionByName returns the first connection matching the given name, or nil. +func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { + conns, err := c.ListConnections(plugin) + if err != nil { + return nil, err + } + for _, conn := range conns { + if conn.Name == name { + return &conn, nil + } + } + return nil, nil +} + +// TestConnection tests connection parameters before creating. +func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { + return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) +} + +// CreateConnection creates a new connection for the given plugin. +func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { + return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) +} + +// DeleteConnection deletes a plugin connection by ID. +func (c *Client) DeleteConnection(plugin string, connID int) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) + } + return nil +} + +// TestSavedConnection tests an already-created connection by ID. +func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) + + reqBody := bytes.NewBufferString("{}") + resp, err := c.HTTPClient.Post(url, "application/json", reqBody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var result ConnectionTestResult + if err := json.Unmarshal(body, &result); err != nil { + // Non-JSON response is ok — treat as success if status 200 + if resp.StatusCode == http.StatusOK { + return &ConnectionTestResult{Success: true}, nil + } + return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) + } + return &result, nil +} + +// GetConnection retrieves a single connection by plugin and ID. +func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { + return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) +} + +// UpdateConnection patches an existing connection for the given plugin. +func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { + return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) +} + +// HealthStatus represents the response from /health or /ping. +type HealthStatus struct { + Status string `json:"status"` +} + +// Health returns the DevLake health status. +func (c *Client) Health() (*HealthStatus, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var hs HealthStatus + _ = json.Unmarshal(body, &hs) + if resp.StatusCode == http.StatusOK { + if hs.Status == "" { + hs.Status = "ok" + } + return &hs, nil + } + return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) +} + +// doPost is a generic helper for POST requests that return JSON. +func doPost[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + url := c.BaseURL + path + resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doGet is a generic helper for GET requests that return JSON. +func doGet[T any](c *Client, path string) (*T, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + path) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPut is a generic helper for PUT requests that return JSON. +func doPut[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPatch is a generic helper for PATCH requests that return JSON. +func doPatch[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// CreateScopeConfig creates a scope config for a plugin connection. +func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { + return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) +} + +// ListScopeConfigs returns all scope configs for a plugin connection. +func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { + result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) + if err != nil { + return nil, err + } + return *result, nil +} + +// PutScopes batch-upserts scopes for a plugin connection. +func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { + _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) + return err +} + +// ListScopes returns the scopes configured on a plugin connection. +func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { + return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) +} + +// ListProjects returns all DevLake projects. +func (c *Client) ListProjects() ([]Project, error) { + result, err := doGet[ProjectListResponse](c, "/projects") + if err != nil { + return nil, err + } + return result.Projects, nil +} + +// DeleteProject deletes a project by name. +func (c *Client) DeleteProject(name string) error { + url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("project not found: %s", name) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) + } + return nil +} + +// DeleteScope removes a scope from a plugin connection. +func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) + } + return nil +} + +// CreateProject creates a new DevLake project. +func (c *Client) CreateProject(project *Project) (*Project, error) { + return doPost[Project](c, "/projects", project) +} + +// GetProject retrieves a project by name. +func (c *Client) GetProject(name string) (*Project, error) { + return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) +} + +// PatchBlueprint updates a blueprint by ID. +func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { + return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) +} + +// TriggerBlueprint triggers a blueprint to run and returns the pipeline. +func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { + return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) +} + +// GetPipeline retrieves a pipeline by ID. +func (c *Client) GetPipeline(id int) (*Pipeline, error) { + return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) +} + +// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. +// groupID and pageToken are optional (pass "" to omit). +func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) + q := url.Values{} + if groupID != "" { + q.Set("groupId", groupID) + } + if pageToken != "" { + q.Set("pageToken", pageToken) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. +// page and pageSize control pagination; pass 0 to use DevLake defaults. +func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) + q := url.Values{} + if search != "" { + q.Set("search", search) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pageSize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// TriggerMigration triggers the DevLake database migration endpoint. +func (c *Client) TriggerMigration() error { + const path = "/proceed-db-migration" + + resp, err := c.HTTPClient.Get(c.BaseURL + path) + if err != nil { + return fmt.Errorf("GET %s: triggering migration: %w", path, err) } defer resp.Body.Close() if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) bodyText := strings.TrimSpace(string(body)) if bodyText != "" { - return fmt.Errorf("DevLake returned status %d: %s", resp.StatusCode, bodyText) + return fmt.Errorf("GET %s: DevLake returned %s: %s", path, resp.Status, bodyText) } - return fmt.Errorf("DevLake returned status %d", resp.StatusCode) + return fmt.Errorf("GET %s: DevLake returned %s", path, resp.Status) } return nil } - -// PipelineListResponse is the response from GET /pipelines. -type PipelineListResponse struct { - Pipelines []Pipeline `json:"pipelines"` - Count int64 `json:"count"` -} - -// ListPipelines returns pipelines with optional query parameters. -// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. -// blueprintID filters by blueprint (0 = no filter). -// page and pageSize control pagination (0 = use defaults). -func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { - path := "/pipelines" - q := url.Values{} - if status != "" { - q.Set("status", status) - } - if blueprintID > 0 { - q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pagesize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[PipelineListResponse](c, path) -} + +// PipelineListResponse is the response from GET /pipelines. +type PipelineListResponse struct { + Pipelines []Pipeline `json:"pipelines"` + Count int64 `json:"count"` +} + +// ListPipelines returns pipelines with optional query parameters. +// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. +// blueprintID filters by blueprint (0 = no filter). +// page and pageSize control pagination (0 = use defaults). +func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { + path := "/pipelines" + q := url.Values{} + if status != "" { + q.Set("status", status) + } + if blueprintID > 0 { + q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pagesize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[PipelineListResponse](c, path) +} diff --git a/internal/devlake/client_test.go b/internal/devlake/client_test.go index 950c128..09ceb8a 100644 --- a/internal/devlake/client_test.go +++ b/internal/devlake/client_test.go @@ -911,9 +911,11 @@ func TestHealth(t *testing.T) { func TestTriggerMigration(t *testing.T) { tests := []struct { - name string - statusCode int - wantErr bool + name string + statusCode int + body string + wantErr bool + wantErrText string }{ { name: "success", @@ -924,9 +926,17 @@ func TestTriggerMigration(t *testing.T) { statusCode: http.StatusNoContent, }, { - name: "server error", - statusCode: http.StatusServiceUnavailable, - wantErr: true, + name: "server error with body", + statusCode: http.StatusServiceUnavailable, + body: "warming up", + wantErr: true, + wantErrText: "GET /proceed-db-migration: DevLake returned 503 Service Unavailable: warming up", + }, + { + name: "server error without body", + statusCode: http.StatusBadGateway, + wantErr: true, + wantErrText: "GET /proceed-db-migration: DevLake returned 502 Bad Gateway", }, } @@ -937,6 +947,7 @@ func TestTriggerMigration(t *testing.T) { t.Errorf("path = %s, want /proceed-db-migration", r.URL.Path) } w.WriteHeader(tt.statusCode) + _, _ = w.Write([]byte(tt.body)) })) defer srv.Close() @@ -947,6 +958,9 @@ func TestTriggerMigration(t *testing.T) { if err == nil { t.Fatal("expected error, got nil") } + if err.Error() != tt.wantErrText { + t.Fatalf("error = %q, want %q", err.Error(), tt.wantErrText) + } return } if err != nil { From 1efb3a3b376e2d166ce3abd79d09b7ef4ad9adf4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:25:34 +0000 Subject: [PATCH 09/10] fix: restore client line endings Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/f7542969-e4d0-4329-bfa1-0cb12581ab4b Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- internal/devlake/client.go | 1112 ++++++++++++++++++------------------ 1 file changed, 556 insertions(+), 556 deletions(-) diff --git a/internal/devlake/client.go b/internal/devlake/client.go index 687bbd2..ac27525 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -1,556 +1,556 @@ -// Package devlake provides an HTTP client for the DevLake REST API. -package devlake - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" -) - -// Client wraps HTTP calls to the DevLake backend API. -type Client struct { - BaseURL string - HTTPClient *http.Client -} - -// NewClient creates a Client for the given base URL. -func NewClient(baseURL string) *Client { - return &Client{ - BaseURL: baseURL, - HTTPClient: &http.Client{ - Timeout: 90 * time.Second, - }, - } -} - -// Ping checks if the DevLake backend is reachable. -func (c *Client) Ping() error { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) - } - return nil -} - -// Connection represents a DevLake plugin connection. -type Connection struct { - ID int `json:"id"` - Name string `json:"name"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. -// Fields with omitempty are only included in the request when non-empty, -// enabling sparse updates (only changed fields are sent). -type ConnectionUpdateRequest struct { - Name string `json:"name,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod,omitempty"` - Token string `json:"token,omitempty"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionCreateRequest is the payload for creating a plugin connection. -type ConnectionCreateRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - Proxy string `json:"proxy,omitempty"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` - TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` - RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` -} - -// ConnectionTestRequest is the payload for testing a connection before creating. -type ConnectionTestRequest struct { - Name string `json:"name"` - Endpoint string `json:"endpoint"` - AuthMethod string `json:"authMethod"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - EnableGraphql bool `json:"enableGraphql,omitempty"` - RateLimitPerHour int `json:"rateLimitPerHour"` - Proxy string `json:"proxy"` - Organization string `json:"organization,omitempty"` - Enterprise string `json:"enterprise,omitempty"` -} - -// ConnectionTestResult is the response from testing a connection. -type ConnectionTestResult struct { - Success bool `json:"success"` - Message string `json:"message"` -} - -// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). -func (c *Client) ListConnections(plugin string) ([]Connection, error) { - resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) - } - - var conns []Connection - if err := json.Unmarshal(body, &conns); err != nil { - return nil, err - } - return conns, nil -} - -// FindConnectionByName returns the first connection matching the given name, or nil. -func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { - conns, err := c.ListConnections(plugin) - if err != nil { - return nil, err - } - for _, conn := range conns { - if conn.Name == name { - return &conn, nil - } - } - return nil, nil -} - -// TestConnection tests connection parameters before creating. -func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { - return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) -} - -// CreateConnection creates a new connection for the given plugin. -func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { - return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) -} - -// DeleteConnection deletes a plugin connection by ID. -func (c *Client) DeleteConnection(plugin string, connID int) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) - } - return nil -} - -// TestSavedConnection tests an already-created connection by ID. -func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) - - reqBody := bytes.NewBufferString("{}") - resp, err := c.HTTPClient.Post(url, "application/json", reqBody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var result ConnectionTestResult - if err := json.Unmarshal(body, &result); err != nil { - // Non-JSON response is ok — treat as success if status 200 - if resp.StatusCode == http.StatusOK { - return &ConnectionTestResult{Success: true}, nil - } - return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) - } - return &result, nil -} - -// GetConnection retrieves a single connection by plugin and ID. -func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { - return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) -} - -// UpdateConnection patches an existing connection for the given plugin. -func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { - return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) -} - -// HealthStatus represents the response from /health or /ping. -type HealthStatus struct { - Status string `json:"status"` -} - -// Health returns the DevLake health status. -func (c *Client) Health() (*HealthStatus, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - var hs HealthStatus - _ = json.Unmarshal(body, &hs) - if resp.StatusCode == http.StatusOK { - if hs.Status == "" { - hs.Status = "ok" - } - return &hs, nil - } - return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) -} - -// doPost is a generic helper for POST requests that return JSON. -func doPost[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - url := c.BaseURL + path - resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doGet is a generic helper for GET requests that return JSON. -func doGet[T any](c *Client, path string) (*T, error) { - resp, err := c.HTTPClient.Get(c.BaseURL + path) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPut is a generic helper for PUT requests that return JSON. -func doPut[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// doPatch is a generic helper for PATCH requests that return JSON. -func doPatch[T any](c *Client, path string, payload any) (*T, error) { - jsonBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) - } - - var result T - if err := json.Unmarshal(body, &result); err != nil { - return nil, err - } - return &result, nil -} - -// CreateScopeConfig creates a scope config for a plugin connection. -func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { - return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) -} - -// ListScopeConfigs returns all scope configs for a plugin connection. -func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { - result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) - if err != nil { - return nil, err - } - return *result, nil -} - -// PutScopes batch-upserts scopes for a plugin connection. -func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { - _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) - return err -} - -// ListScopes returns the scopes configured on a plugin connection. -func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { - return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) -} - -// ListProjects returns all DevLake projects. -func (c *Client) ListProjects() ([]Project, error) { - result, err := doGet[ProjectListResponse](c, "/projects") - if err != nil { - return nil, err - } - return result.Projects, nil -} - -// DeleteProject deletes a project by name. -func (c *Client) DeleteProject(name string) error { - url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("project not found: %s", name) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) - } - return nil -} - -// DeleteScope removes a scope from a plugin connection. -func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { - url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return err - } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response: %w", err) - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) - } - return nil -} - -// CreateProject creates a new DevLake project. -func (c *Client) CreateProject(project *Project) (*Project, error) { - return doPost[Project](c, "/projects", project) -} - -// GetProject retrieves a project by name. -func (c *Client) GetProject(name string) (*Project, error) { - return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) -} - -// PatchBlueprint updates a blueprint by ID. -func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { - return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) -} - -// TriggerBlueprint triggers a blueprint to run and returns the pipeline. -func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { - return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) -} - -// GetPipeline retrieves a pipeline by ID. -func (c *Client) GetPipeline(id int) (*Pipeline, error) { - return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) -} - -// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. -// groupID and pageToken are optional (pass "" to omit). -func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) - q := url.Values{} - if groupID != "" { - q.Set("groupId", groupID) - } - if pageToken != "" { - q.Set("pageToken", pageToken) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. -// page and pageSize control pagination; pass 0 to use DevLake defaults. -func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { - path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) - q := url.Values{} - if search != "" { - q.Set("search", search) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pageSize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[RemoteScopeResponse](c, path) -} - -// TriggerMigration triggers the DevLake database migration endpoint. -func (c *Client) TriggerMigration() error { - const path = "/proceed-db-migration" - - resp, err := c.HTTPClient.Get(c.BaseURL + path) - if err != nil { - return fmt.Errorf("GET %s: triggering migration: %w", path, err) - } - defer resp.Body.Close() - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - bodyText := strings.TrimSpace(string(body)) - if bodyText != "" { - return fmt.Errorf("GET %s: DevLake returned %s: %s", path, resp.Status, bodyText) - } - return fmt.Errorf("GET %s: DevLake returned %s", path, resp.Status) - } - return nil -} - -// PipelineListResponse is the response from GET /pipelines. -type PipelineListResponse struct { - Pipelines []Pipeline `json:"pipelines"` - Count int64 `json:"count"` -} - -// ListPipelines returns pipelines with optional query parameters. -// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. -// blueprintID filters by blueprint (0 = no filter). -// page and pageSize control pagination (0 = use defaults). -func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { - path := "/pipelines" - q := url.Values{} - if status != "" { - q.Set("status", status) - } - if blueprintID > 0 { - q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) - } - if page > 0 { - q.Set("page", fmt.Sprintf("%d", page)) - } - if pageSize > 0 { - q.Set("pagesize", fmt.Sprintf("%d", pageSize)) - } - if len(q) > 0 { - path += "?" + q.Encode() - } - return doGet[PipelineListResponse](c, path) -} +// Package devlake provides an HTTP client for the DevLake REST API. +package devlake + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// Client wraps HTTP calls to the DevLake backend API. +type Client struct { + BaseURL string + HTTPClient *http.Client +} + +// NewClient creates a Client for the given base URL. +func NewClient(baseURL string) *Client { + return &Client{ + BaseURL: baseURL, + HTTPClient: &http.Client{ + Timeout: 90 * time.Second, + }, + } +} + +// Ping checks if the DevLake backend is reachable. +func (c *Client) Ping() error { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return fmt.Errorf("cannot reach DevLake at %s/ping: %w", c.BaseURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("DevLake returned status %d from /ping", resp.StatusCode) + } + return nil +} + +// Connection represents a DevLake plugin connection. +type Connection struct { + ID int `json:"id"` + Name string `json:"name"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionUpdateRequest is the payload for PATCH /plugins/{plugin}/connections/{id}. +// Fields with omitempty are only included in the request when non-empty, +// enabling sparse updates (only changed fields are sent). +type ConnectionUpdateRequest struct { + Name string `json:"name,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod,omitempty"` + Token string `json:"token,omitempty"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionCreateRequest is the payload for creating a plugin connection. +type ConnectionCreateRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy,omitempty"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` + TokenExpiresAt string `json:"tokenExpiresAt,omitempty"` + RefreshTokenExpiresAt string `json:"refreshTokenExpiresAt,omitempty"` +} + +// ConnectionTestRequest is the payload for testing a connection before creating. +type ConnectionTestRequest struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + AuthMethod string `json:"authMethod"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + EnableGraphql bool `json:"enableGraphql,omitempty"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Proxy string `json:"proxy"` + Organization string `json:"organization,omitempty"` + Enterprise string `json:"enterprise,omitempty"` +} + +// ConnectionTestResult is the response from testing a connection. +type ConnectionTestResult struct { + Success bool `json:"success"` + Message string `json:"message"` +} + +// ListConnections returns all connections for a plugin (e.g. "github", "gh-copilot"). +func (c *Client) ListConnections(plugin string) ([]Connection, error) { + resp, err := c.HTTPClient.Get(fmt.Sprintf("%s/plugins/%s/connections", c.BaseURL, plugin)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("list connections returned %d: %s", resp.StatusCode, body) + } + + var conns []Connection + if err := json.Unmarshal(body, &conns); err != nil { + return nil, err + } + return conns, nil +} + +// FindConnectionByName returns the first connection matching the given name, or nil. +func (c *Client) FindConnectionByName(plugin, name string) (*Connection, error) { + conns, err := c.ListConnections(plugin) + if err != nil { + return nil, err + } + for _, conn := range conns { + if conn.Name == name { + return &conn, nil + } + } + return nil, nil +} + +// TestConnection tests connection parameters before creating. +func (c *Client) TestConnection(plugin string, req *ConnectionTestRequest) (*ConnectionTestResult, error) { + return doPost[ConnectionTestResult](c, fmt.Sprintf("/plugins/%s/test", plugin), req) +} + +// CreateConnection creates a new connection for the given plugin. +func (c *Client) CreateConnection(plugin string, req *ConnectionCreateRequest) (*Connection, error) { + return doPost[Connection](c, fmt.Sprintf("/plugins/%s/connections", plugin), req) +} + +// DeleteConnection deletes a plugin connection by ID. +func (c *Client) DeleteConnection(plugin string, connID int) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d", c.BaseURL, plugin, connID) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("connection not found: plugin=%s id=%d", plugin, connID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d returned %d: %s", plugin, connID, resp.StatusCode, body) + } + return nil +} + +// TestSavedConnection tests an already-created connection by ID. +func (c *Client) TestSavedConnection(plugin string, connID int) (*ConnectionTestResult, error) { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/test", c.BaseURL, plugin, connID) + + reqBody := bytes.NewBufferString("{}") + resp, err := c.HTTPClient.Post(url, "application/json", reqBody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var result ConnectionTestResult + if err := json.Unmarshal(body, &result); err != nil { + // Non-JSON response is ok — treat as success if status 200 + if resp.StatusCode == http.StatusOK { + return &ConnectionTestResult{Success: true}, nil + } + return nil, fmt.Errorf("test connection returned %d: %s", resp.StatusCode, body) + } + return &result, nil +} + +// GetConnection retrieves a single connection by plugin and ID. +func (c *Client) GetConnection(plugin string, connID int) (*Connection, error) { + return doGet[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID)) +} + +// UpdateConnection patches an existing connection for the given plugin. +func (c *Client) UpdateConnection(plugin string, connID int, req *ConnectionUpdateRequest) (*Connection, error) { + return doPatch[Connection](c, fmt.Sprintf("/plugins/%s/connections/%d", plugin, connID), req) +} + +// HealthStatus represents the response from /health or /ping. +type HealthStatus struct { + Status string `json:"status"` +} + +// Health returns the DevLake health status. +func (c *Client) Health() (*HealthStatus, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + "/ping") + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + var hs HealthStatus + _ = json.Unmarshal(body, &hs) + if resp.StatusCode == http.StatusOK { + if hs.Status == "" { + hs.Status = "ok" + } + return &hs, nil + } + return nil, fmt.Errorf("health check returned %d: %s", resp.StatusCode, body) +} + +// doPost is a generic helper for POST requests that return JSON. +func doPost[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + url := c.BaseURL + path + resp, err := c.HTTPClient.Post(url, "application/json", bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("POST %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doGet is a generic helper for GET requests that return JSON. +func doGet[T any](c *Client, path string) (*T, error) { + resp, err := c.HTTPClient.Get(c.BaseURL + path) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPut is a generic helper for PUT requests that return JSON. +func doPut[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPut, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("PUT %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// doPatch is a generic helper for PATCH requests that return JSON. +func doPatch[T any](c *Client, path string, payload any) (*T, error) { + jsonBody, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPatch, c.BaseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("PATCH %s returned %d: %s", path, resp.StatusCode, body) + } + + var result T + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// CreateScopeConfig creates a scope config for a plugin connection. +func (c *Client) CreateScopeConfig(plugin string, connID int, cfg *ScopeConfig) (*ScopeConfig, error) { + return doPost[ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID), cfg) +} + +// ListScopeConfigs returns all scope configs for a plugin connection. +func (c *Client) ListScopeConfigs(plugin string, connID int) ([]ScopeConfig, error) { + result, err := doGet[[]ScopeConfig](c, fmt.Sprintf("/plugins/%s/connections/%d/scope-configs", plugin, connID)) + if err != nil { + return nil, err + } + return *result, nil +} + +// PutScopes batch-upserts scopes for a plugin connection. +func (c *Client) PutScopes(plugin string, connID int, req *ScopeBatchRequest) error { + _, err := doPut[json.RawMessage](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes", plugin, connID), req) + return err +} + +// ListScopes returns the scopes configured on a plugin connection. +func (c *Client) ListScopes(plugin string, connID int) (*ScopeListResponse, error) { + return doGet[ScopeListResponse](c, fmt.Sprintf("/plugins/%s/connections/%d/scopes?pageSize=100&page=1", plugin, connID)) +} + +// ListProjects returns all DevLake projects. +func (c *Client) ListProjects() ([]Project, error) { + result, err := doGet[ProjectListResponse](c, "/projects") + if err != nil { + return nil, err + } + return result.Projects, nil +} + +// DeleteProject deletes a project by name. +func (c *Client) DeleteProject(name string) error { + url := fmt.Sprintf("%s/projects/%s", c.BaseURL, name) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("project not found: %s", name) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /projects/%s returned %d: %s", name, resp.StatusCode, body) + } + return nil +} + +// DeleteScope removes a scope from a plugin connection. +func (c *Client) DeleteScope(plugin string, connID int, scopeID string) error { + url := fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", c.BaseURL, plugin, connID, url.PathEscape(scopeID)) + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("scope not found: plugin=%s connID=%d scopeID=%s", plugin, connID, scopeID) + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("DELETE /plugins/%s/connections/%d/scopes/%s returned %d: %s", plugin, connID, scopeID, resp.StatusCode, body) + } + return nil +} + +// CreateProject creates a new DevLake project. +func (c *Client) CreateProject(project *Project) (*Project, error) { + return doPost[Project](c, "/projects", project) +} + +// GetProject retrieves a project by name. +func (c *Client) GetProject(name string) (*Project, error) { + return doGet[Project](c, fmt.Sprintf("/projects/%s", name)) +} + +// PatchBlueprint updates a blueprint by ID. +func (c *Client) PatchBlueprint(id int, patch *BlueprintPatch) (*Blueprint, error) { + return doPatch[Blueprint](c, fmt.Sprintf("/blueprints/%d", id), patch) +} + +// TriggerBlueprint triggers a blueprint to run and returns the pipeline. +func (c *Client) TriggerBlueprint(id int) (*Pipeline, error) { + return doPost[Pipeline](c, fmt.Sprintf("/blueprints/%d/trigger", id), struct{}{}) +} + +// GetPipeline retrieves a pipeline by ID. +func (c *Client) GetPipeline(id int) (*Pipeline, error) { + return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id)) +} + +// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection. +// groupID and pageToken are optional (pass "" to omit). +func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID) + q := url.Values{} + if groupID != "" { + q.Set("groupId", groupID) + } + if pageToken != "" { + q.Set("pageToken", pageToken) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection. +// page and pageSize control pagination; pass 0 to use DevLake defaults. +func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) { + path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID) + q := url.Values{} + if search != "" { + q.Set("search", search) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pageSize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[RemoteScopeResponse](c, path) +} + +// TriggerMigration triggers the DevLake database migration endpoint. +func (c *Client) TriggerMigration() error { + const path = "/proceed-db-migration" + + resp, err := c.HTTPClient.Get(c.BaseURL + path) + if err != nil { + return fmt.Errorf("GET %s: triggering migration: %w", path, err) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + bodyText := strings.TrimSpace(string(body)) + if bodyText != "" { + return fmt.Errorf("GET %s: DevLake returned %s: %s", path, resp.Status, bodyText) + } + return fmt.Errorf("GET %s: DevLake returned %s", path, resp.Status) + } + return nil +} + +// PipelineListResponse is the response from GET /pipelines. +type PipelineListResponse struct { + Pipelines []Pipeline `json:"pipelines"` + Count int64 `json:"count"` +} + +// ListPipelines returns pipelines with optional query parameters. +// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. +// blueprintID filters by blueprint (0 = no filter). +// page and pageSize control pagination (0 = use defaults). +func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { + path := "/pipelines" + q := url.Values{} + if status != "" { + q.Set("status", status) + } + if blueprintID > 0 { + q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) + } + if page > 0 { + q.Set("page", fmt.Sprintf("%d", page)) + } + if pageSize > 0 { + q.Set("pagesize", fmt.Sprintf("%d", pageSize)) + } + if len(q) > 0 { + path += "?" + q.Encode() + } + return doGet[PipelineListResponse](c, path) +} From 359b74912ad4d557123160ac22d8c58d452147e7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 30 Mar 2026 19:01:56 +0000 Subject: [PATCH 10/10] refactor: use helper for migration trigger Agent-Logs-Url: https://github.com/DevExpGbb/gh-devlake/sessions/bd7e7a9e-5fa5-45a2-87b4-63939ccb1693 Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- internal/devlake/client.go | 40 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/internal/devlake/client.go b/internal/devlake/client.go index ac27525..dffd1dc 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -290,6 +290,29 @@ func doGet[T any](c *Client, path string) (*T, error) { return &result, nil } +// doGetNoBody is a helper for GET requests that only need a successful 2xx status. +func doGetNoBody(c *Client, path string) error { + resp, err := c.HTTPClient.Get(c.BaseURL + path) + if err != nil { + return fmt.Errorf("GET %s: %w", path, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 512)) + if err != nil { + return fmt.Errorf("GET %s: reading response: %w", path, err) + } + if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices { + return nil + } + + bodyText := strings.TrimSpace(string(body)) + if bodyText != "" { + return fmt.Errorf("GET %s: DevLake returned %s: %s", path, resp.Status, bodyText) + } + return fmt.Errorf("GET %s: DevLake returned %s", path, resp.Status) +} + // doPut is a generic helper for PUT requests that return JSON. func doPut[T any](c *Client, path string, payload any) (*T, error) { jsonBody, err := json.Marshal(payload) @@ -506,22 +529,7 @@ func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, pa // TriggerMigration triggers the DevLake database migration endpoint. func (c *Client) TriggerMigration() error { - const path = "/proceed-db-migration" - - resp, err := c.HTTPClient.Get(c.BaseURL + path) - if err != nil { - return fmt.Errorf("GET %s: triggering migration: %w", path, err) - } - defer resp.Body.Close() - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - bodyText := strings.TrimSpace(string(body)) - if bodyText != "" { - return fmt.Errorf("GET %s: DevLake returned %s: %s", path, resp.Status, bodyText) - } - return fmt.Errorf("GET %s: DevLake returned %s", path, resp.Status) - } - return nil + return doGetNoBody(c, "/proceed-db-migration") } // PipelineListResponse is the response from GET /pipelines.