From b9d85b8b59b76f0bd600995ea00a39aa10326ed3 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 19 Mar 2025 16:23:37 +0000 Subject: [PATCH 1/4] `vendor/modules.txt` should list package paths It previously just listed the module path from the go.mod file. --- vendor.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/vendor.go b/vendor.go index 4c437d9..23dfd20 100644 --- a/vendor.go +++ b/vendor.go @@ -4,10 +4,13 @@ package main import ( "bytes" + "go/parser" + "go/token" "io/ioutil" "log" "os" "path/filepath" + "regexp" "strings" "golang.org/x/mod/modfile" @@ -85,24 +88,30 @@ func stubModulesTxt() { } modFile := loadModFile(filepath.Join(modRoot, "go.mod")) - vdir := filepath.Join(modRoot, "vendor") if gv := modFile.Go; gv != nil && semver.Compare("v"+gv.Version, "v1.14") >= 0 { - // If the Go version is at least 1.14, generate a dummy modules.txt using only the information - // in the go.mod file + // Find imports from all Go files in the project + usedPackages := findPackagesInSourceCode(modRoot) generated := make(map[module.Version]bool) var buf bytes.Buffer for _, r := range modFile.Require { - // TODO: support replace lines generated[r.Mod] = true line := moduleLine(r.Mod, module.Version{}) buf.WriteString(line) - buf.WriteString("## explicit\n") - buf.WriteString(r.Mod.Path + "\n") + // List package paths that are used in the source code + packagesForModule := findPackagesForModule(r.Mod.Path, usedPackages) + if len(packagesForModule) > 0 { + for _, pkg := range packagesForModule { + buf.WriteString(pkg + "\n") + } + } else { + // If we can't find any packages then just list the module path itself + buf.WriteString(r.Mod.Path + "\n") + } } // Record unused and wildcard replacements at the end of the modules.txt file: @@ -133,3 +142,61 @@ func stubModulesTxt() { } } } + +// findPackagesInSourceCode scans all Go files in the directory tree and extracts import paths +func findPackagesInSourceCode(root string) map[string]bool { + packages := make(map[string]bool) + + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Skip vendor directory and hidden directories + if info.IsDir() && (info.Name() == "vendor" || strings.HasPrefix(info.Name(), ".")) { + return filepath.SkipDir + } + + // Only process Go files + if !info.IsDir() && strings.HasSuffix(path, ".go") { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly) + if err != nil { + return err + } + + // Extract import paths from the AST + for _, imp := range file.Imports { + pkgPath := strings.Trim(imp.Path.Value, "\"") + packages[pkgPath] = true + } + } + return nil + }) + + if err != nil { + log.Printf("Warning: error walking source directory: %v", err) + } + + return packages +} + +// findPackagesForModule returns the submodules of a given module that are actually used in the source code +func findPackagesForModule(modulePath string, usedPackages map[string]bool) []string { + var packages []string + + for pkg := range usedPackages { + // Check if this package belongs to the module + if strings.HasPrefix(pkg, modulePath) { + // Extract the part after modulePath + suffix := pkg[len(modulePath):] + matched, _ := regexp.MatchString(`^/v[1-9][0-9]*(/|$)`, suffix) + + if !matched { + packages = append(packages, pkg) + } + } + } + + return packages +} From d3b28a5fb6a2fbb7e951d5fe6632a25f8b043d33 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 19 Mar 2025 16:48:03 +0000 Subject: [PATCH 2/4] Compile regex and explain why it is used --- vendor.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vendor.go b/vendor.go index 23dfd20..9773907 100644 --- a/vendor.go +++ b/vendor.go @@ -181,6 +181,9 @@ func findPackagesInSourceCode(root string) map[string]bool { return packages } +// Compile the regular expression once +var majorVersionSuffixRegex = regexp.MustCompile(`^/v[1-9][0-9]*(/|$)`) + // findPackagesForModule returns the submodules of a given module that are actually used in the source code func findPackagesForModule(modulePath string, usedPackages map[string]bool) []string { var packages []string @@ -190,9 +193,12 @@ func findPackagesForModule(modulePath string, usedPackages map[string]bool) []st if strings.HasPrefix(pkg, modulePath) { // Extract the part after modulePath suffix := pkg[len(modulePath):] - matched, _ := regexp.MatchString(`^/v[1-9][0-9]*(/|$)`, suffix) - if !matched { + // If `suffix` begins with a major version suffix then we do not have the right module + // path. For example, if the module path is `example.com/mymodule` and the package path + // is `example.com/mymodule/v2/submodule` then we should not consider it a match - it + // is really a match for the module `example.com/mymodule/v2`. + if !majorVersionSuffixRegex.MatchString(suffix) { packages = append(packages, pkg) } } From 3ac77e09a2a23d865e7521775553b0f50c94a885 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Wed, 19 Mar 2025 16:53:33 +0000 Subject: [PATCH 3/4] Do not use deprecated `io/ioutil` package --- depstubber.go | 3 +-- reflect.go | 7 +++---- util.go | 3 +-- vendor.go | 5 ++--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/depstubber.go b/depstubber.go index 1b0c046..d9907b0 100644 --- a/depstubber.go +++ b/depstubber.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -153,7 +152,7 @@ func createStubs(packageName string, typeNames []string, funcAndVarNames []strin g.srcFunctions = strings.Join(funcAndVarNames, ",") if *copyrightFile != "" { - header, err := ioutil.ReadFile(*copyrightFile) + header, err := os.ReadFile(*copyrightFile) if err != nil { log.Fatalf("Failed reading copyright file: %v", err) } diff --git a/reflect.go b/reflect.go index 6ad6087..f29b8df 100644 --- a/reflect.go +++ b/reflect.go @@ -8,7 +8,6 @@ import ( "flag" "fmt" "go/build" - "io/ioutil" "log" "os" "os/exec" @@ -44,7 +43,7 @@ func writeProgram(importPath string, types []string, values []string) ([]byte, e // run the given program and parse the output as a model.Package. func run(program string) (*model.PackedPkg, error) { - f, err := ioutil.TempFile("", "") + f, err := os.CreateTemp("", "") if err != nil { return nil, err } @@ -85,7 +84,7 @@ func run(program string) (*model.PackedPkg, error) { // parses the output as a model.Package. func runInDir(program []byte, dir string) (*model.PackedPkg, error) { // We use TempDir instead of TempFile so we can control the filename. - tmpDir, err := ioutil.TempDir(dir, "depstubber_reflect_") + tmpDir, err := os.MkdirTemp(dir, "depstubber_reflect_") if err != nil { return nil, err } @@ -101,7 +100,7 @@ func runInDir(program []byte, dir string) (*model.PackedPkg, error) { progBinary += ".exe" } - if err := ioutil.WriteFile(filepath.Join(tmpDir, progSource), program, 0600); err != nil { + if err := os.WriteFile(filepath.Join(tmpDir, progSource), program, 0600); err != nil { return nil, err } diff --git a/util.go b/util.go index 6d9fe33..8de60cc 100644 --- a/util.go +++ b/util.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "os" "runtime/debug" @@ -23,7 +22,7 @@ func removeDot(s string) string { // packageNameOfDir get package import path via dir func packageNameOfDir(srcDir string) (string, error) { - files, err := ioutil.ReadDir(srcDir) + files, err := os.ReadDir(srcDir) if err != nil { log.Fatal(err) } diff --git a/vendor.go b/vendor.go index 9773907..9fd12d9 100644 --- a/vendor.go +++ b/vendor.go @@ -6,7 +6,6 @@ import ( "bytes" "go/parser" "go/token" - "io/ioutil" "log" "os" "path/filepath" @@ -41,7 +40,7 @@ func findModuleRoot(dir string) (root string) { } func loadModFile(filename string) *modfile.File { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { panic(err) } @@ -137,7 +136,7 @@ func stubModulesTxt() { log.Fatalf("go mod vendor: %v", err) } - if err := ioutil.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil { + if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil { log.Fatalf("go mod vendor: %v", err) } } From 1e8a138fb4835dff9d4cd4b05925fd13b498afe4 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Thu, 20 Mar 2025 21:59:20 +0000 Subject: [PATCH 4/4] Sort packages for each module --- vendor.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vendor.go b/vendor.go index 9fd12d9..a9bde9d 100644 --- a/vendor.go +++ b/vendor.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "golang.org/x/mod/modfile" @@ -203,5 +204,7 @@ func findPackagesForModule(modulePath string, usedPackages map[string]bool) []st } } + // Sort packages for consistent output + sort.Strings(packages) return packages }