feat: support for Go modules to imports

Author: switchupcb

README.md

@@ -15,6 +15,7 @@ It powers executable Go scripts and plugins, in embedded interpreters or interac
 * Complete support of [Go specification][specs]
 * Written in pure Go, using only the standard library
 * Simple interpreter API: `New()`, `Eval()`, `Use()`
+* Supports Go Modules
 * Works everywhere Go works
 * All Go & runtime resources accessible from script (with control)
 * Security: `unsafe` and `syscall` packages neither used nor exported by default
@@ -95,6 +96,11 @@ func Bar(s string) string { return s + "-Foo" }`
 func main() {
 	i := interp.New(interp.Options{})
 
+	// import the standard library (Go Modules)
+	if err := i.Use(stdlib.Symbols); err != nil {
+		t.Fatal(err)
+	}
+
 	_, err := i.Eval(src)
 	if err != nil {
 		panic(err)

go.mod

@@ -1,3 +1,10 @@
 module github.com/traefik/yaegi
 
 go 1.18
+
+require golang.org/x/tools v0.1.12
+
+require (
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+)

go.sum

@@ -0,0 +1,6 @@
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

interp/doc.go

@@ -9,8 +9,7 @@ Importing packages
 Packages can be imported in source or binary form, using the standard
 Go import statement. In source form, packages are searched first in the
 vendor directory, the preferred way to store source dependencies. If not
-found in vendor, sources modules will be searched in GOPATH. Go modules
-are not supported yet by yaegi.
+found in vendor, sources modules will be searched in GOPATH.
 
 Binary form packages are compiled and linked with the interpreter
 executable, and exposed to scripts with the Use method. The extract

interp/interp.go

@@ -174,6 +174,8 @@ type opt struct {
 	// dotCmd is the command to process the dot graph produced when astDot and/or
 	// cfgDot is enabled. It defaults to 'dot -Tdot -o <filename>.dot'.
 	dotCmd       string
+	noRun        bool              // compile, but do not run
+	fastChan     bool              // disable cancellable chan operations
 	context      build.Context     // build context: GOPATH, build constraints
 	stdin        io.Reader         // standard input
 	stdout       io.Writer         // standard output
@@ -183,8 +185,6 @@ type opt struct {
 	filesystem   fs.FS             // filesystem containing sources
 	astDot       bool              // display AST graph (debug)
 	cfgDot       bool              // display CFG graph (debug)
-	noRun        bool              // compile, but do not run
-	fastChan     bool              // disable cancellable chan operations
 	specialStdio bool              // allows os.Stdin, os.Stdout, os.Stderr to not be file descriptors
 	unrestricted bool              // allow use of non sandboxed symbols
 }
@@ -301,6 +301,12 @@ type Options struct {
 	// GoPath sets GOPATH for the interpreter.
 	GoPath string
 
+	// GoCache sets GOCACHE for the interpreter.
+	GoCache string
+
+	// GoToolDir sets the GOTOOLDIR for the interpreter.
+	GoToolDir string
+
 	// BuildTags sets build constraints for the interpreter.
 	BuildTags []string
 
@@ -328,7 +334,12 @@ type Options struct {
 // New returns a new interpreter.
 func New(options Options) *Interpreter {
 	i := Interpreter{
-		opt:      opt{context: build.Default, filesystem: &realFS{}, env: map[string]string{}},
+		opt: opt{
+			context: build.Default, filesystem: &realFS{}, env: map[string]string{
+				"goCache":   options.GoCache,
+				"goToolDir": options.GoToolDir,
+			},
+		},
 		frame:    newFrame(nil, 0, 0),
 		fset:     token.NewFileSet(),
 		universe: initUniverse(),

interp/src.go

@@ -3,12 +3,13 @@ package interp
 import (
 	"fmt"
 	"io/fs"
-	"os"
 	"path/filepath"
 	"strings"
+
+	"golang.org/x/tools/go/packages"
 )
 
-// importSrc calls gta on the source code for the package identified by
+// importSrc calls global tag analysis on the source code for the package identified by
 // importPath. rPath is the relative path to the directory containing the source
 // code for the package. It can also be "main" as a special value.
 func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (string, error) {
@@ -23,24 +24,14 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s
 		return name, nil
 	}
 
-	// For relative import paths in the form "./xxx" or "../xxx", the initial
-	// base path is the directory of the interpreter input file, or "." if no file
-	// was provided.
-	// In all other cases, absolute import paths are resolved from the GOPATH
-	// and the nested "vendor" directories.
+	// resolve relative and absolute import paths
 	if isPathRelative(importPath) {
 		if rPath == mainID {
 			rPath = "."
 		}
 		dir = filepath.Join(filepath.Dir(interp.name), rPath, importPath)
-	} else if dir, rPath, err = interp.pkgDir(interp.context.GOPATH, rPath, importPath); err != nil {
-		// Try again, assuming a root dir at the source location.
-		if rPath, err = interp.rootFromSourceLocation(); err != nil {
-			return "", err
-		}
-		if dir, rPath, err = interp.pkgDir(interp.context.GOPATH, rPath, importPath); err != nil {
-			return "", err
-		}
+	} else if dir, err = interp.getPackageDir(importPath); err != nil {
+		return "", err
 	}
 
 	if interp.rdir[importPath] {
@@ -171,119 +162,87 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s
 	return pkgName, nil
 }
 
-// rootFromSourceLocation returns the path to the directory containing the input
-// Go file given to the interpreter, relative to $GOPATH/src.
-// It is meant to be called in the case when the initial input is a main package.
-func (interp *Interpreter) rootFromSourceLocation() (string, error) {
-	sourceFile := interp.name
-	if sourceFile == DefaultSourceName {
-		return "", nil
-	}
-	wd, err := os.Getwd()
+// getPackageDir uses the GOPATH to find the absolute path of an import path.
+func (interp *Interpreter) getPackageDir(importPath string) (string, error) {
+	// search the standard library and Go modules.
+	config := packages.Config{}
+	config.Env = append(config.Env, "GOPATH="+interp.context.GOPATH, "GOCACHE="+interp.opt.env["goCache"], "GOTOOLDIR="+interp.opt.env["goToolDir"])
+	pkgs, err := packages.Load(&config, importPath)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("an error occurred retrieving a package from the GOPATH: %v\n%v\nIf Access is denied, run in administrator", importPath, err)
 	}
-	pkgDir := filepath.Join(wd, filepath.Dir(sourceFile))
-	root := strings.TrimPrefix(pkgDir, filepath.Join(interp.context.GOPATH, "src")+"/")
-	if root == wd {
-		return "", fmt.Errorf("package location %s not in GOPATH", pkgDir)
+
+	// confirm the import path is found.
+	for _, pkg := range pkgs {
+		for _, goFile := range pkg.GoFiles {
+			if strings.Contains(filepath.Dir(goFile), pkg.Name) {
+				return filepath.Dir(goFile), nil
+			}
+		}
 	}
-	return root, nil
-}
 
-// pkgDir returns the absolute path in filesystem for a package given its import path
-// and the root of the subtree dependencies.
-func (interp *Interpreter) pkgDir(goPath string, root, importPath string) (string, string, error) {
-	rPath := filepath.Join(root, "vendor")
-	dir := filepath.Join(goPath, "src", rPath, importPath)
+	// check for certain go tools located in GOTOOLDIR.
+	if interp.opt.env["goToolDir"] != "" {
+		// search for the go directory before searching for packages.
+		// this approach prevents the computer from searching the entire filesystem.
+		godir, err := searchUpDirPath(interp.opt.env["goToolDir"], "go", false)
+		if err != nil {
+			return "", fmt.Errorf("an import source could not be found: %q\nThe current GOPATH=%v, GOCACHE=%v, GOTOOLDIR=%v\n%v", importPath, interp.context.GOPATH, interp.opt.env["goCache"], interp.opt.env["goToolDir"], err)
+		}
 
-	if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil {
-		return dir, rPath, nil // found!
+		absimportpath, err := searchDirs(godir, importPath)
+		if err != nil {
+			return "", fmt.Errorf("an import source could not be found: %q\nThe current GOPATH=%v, GOCACHE=%v, GOTOOLDIR=%v\n%v", importPath, interp.context.GOPATH, interp.opt.env["goCache"], interp.opt.env["goToolDir"], err)
+		}
+		return absimportpath, nil
 	}
 
-	dir = filepath.Join(goPath, "src", effectivePkg(root, importPath))
+	return "", fmt.Errorf("an import source could not be found: %q. Set the GOPATH and/or GOTOOLDIR environment variable from Interpreter.Options", importPath)
+}
 
-	if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil {
-		return dir, root, nil // found!
+// searchUpDirPath searches up a directory path in order to find a target directory.
+func searchUpDirPath(initial string, target string, isCaseSensitive bool) (string, error) {
+	// strings.Split always returns [:0] as filepath.Dir returns "." or the last directory.
+	splitdir := strings.Split(filepath.Clean(initial), string(filepath.Separator))
+	if len(splitdir) == 1 {
+		return "", fmt.Errorf("the target directory %q is not within the path %q", target, initial)
 	}
 
-	if len(root) == 0 {
-		if interp.context.GOPATH == "" {
-			return "", "", fmt.Errorf("unable to find source related to: %q. Either the GOPATH environment variable, or the Interpreter.Options.GoPath needs to be set", importPath)
-		}
-		return "", "", fmt.Errorf("unable to find source related to: %q", importPath)
+	updir := splitdir[len(splitdir)-1]
+	if !isCaseSensitive {
+		updir = strings.ToLower(updir)
 	}
-
-	rootPath := filepath.Join(goPath, "src", root)
-	prevRoot, err := previousRoot(interp.opt.filesystem, rootPath, root)
-	if err != nil {
-		return "", "", err
+	if updir == target {
+		return initial, nil
 	}
-
-	return interp.pkgDir(goPath, prevRoot, importPath)
+	return searchUpDirPath(filepath.Dir(initial), target, isCaseSensitive)
 }
 
-const vendor = "vendor"
-
-// Find the previous source root (vendor > vendor > ... > GOPATH).
-func previousRoot(filesystem fs.FS, rootPath, root string) (string, error) {
-	rootPath = filepath.Clean(rootPath)
-	parent, final := filepath.Split(rootPath)
-	parent = filepath.Clean(parent)
-
-	// TODO(mpl): maybe it works for the special case main, but can't be bothered for now.
-	if root != mainID && final != vendor {
-		root = strings.TrimSuffix(root, string(filepath.Separator))
-		prefix := strings.TrimSuffix(strings.TrimSuffix(rootPath, root), string(filepath.Separator))
-
-		// look for the closest vendor in one of our direct ancestors, as it takes priority.
-		var vendored string
-		for {
-			fi, err := fs.Stat(filesystem, filepath.Join(parent, vendor))
-			if err == nil && fi.IsDir() {
-				vendored = strings.TrimPrefix(strings.TrimPrefix(parent, prefix), string(filepath.Separator))
-				break
-			}
-			if !os.IsNotExist(err) {
-				return "", err
-			}
-
-			// stop when we reach GOPATH/src/blah
-			parent = filepath.Dir(parent)
-			if parent == prefix {
-				break
-			}
+// searchDirs searches within a directory (and its subdirectories) in an attempt to find a filepath.
+func searchDirs(initial string, target string) (string, error) {
+	absfilepath, err := filepath.Abs(initial)
+	if err != nil {
+		return "", err
+	}
 
-			// just an additional failsafe, stop if we reach the filesystem root, or dot (if
-			// we are dealing with relative paths).
-			// TODO(mpl): It should probably be a critical error actually,
-			// as we shouldn't have gone that high up in the tree.
-			if parent == string(filepath.Separator) || parent == "." {
-				break
+	// find the go directory.
+	var foundpath string
+	filter := func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			if d.Name() == target {
+				foundpath = path
 			}
 		}
-
-		if vendored != "" {
-			return vendored, nil
-		}
+		return nil
 	}
-
-	// TODO(mpl): the algorithm below might be redundant with the one above,
-	// but keeping it for now. Investigate/simplify/remove later.
-	splitRoot := strings.Split(root, string(filepath.Separator))
-	var index int
-	for i := len(splitRoot) - 1; i >= 0; i-- {
-		if splitRoot[i] == "vendor" {
-			index = i
-			break
-		}
+	if err = filepath.WalkDir(absfilepath, filter); err != nil {
+		return "", fmt.Errorf("An error occurred searching for a directory.\n%v", err)
 	}
 
-	if index == 0 {
-		return "", nil
+	if foundpath != "" {
+		return foundpath, nil
 	}
-
-	return filepath.Join(splitRoot[:index]...), nil
+	return "", fmt.Errorf("The target filepath %q is not within the path %q", target, initial)
 }
 
 func effectivePkg(root, path string) string {

interp/src_test.go

@@ -183,7 +183,7 @@ func Test_pkgDir(t *testing.T) {
 				}
 			}
 
-			dir, rPath, err := interp.pkgDir(goPath, test.root, test.path)
+			dir, err := interp.getPackageDir(test.path)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -192,71 +192,8 @@ func Test_pkgDir(t *testing.T) {
 				t.Errorf("[dir] got: %s, want: %s", dir, test.expected.dir)
 			}
 
-			if rPath != test.expected.rpath {
-				t.Errorf(" [rpath] got: %s, want: %s", rPath, test.expected.rpath)
-			}
-		})
-	}
-}
-
-func Test_previousRoot(t *testing.T) {
-	testCases := []struct {
-		desc           string
-		root           string
-		rootPathSuffix string
-		expected       string
-	}{
-		{
-			desc:     "GOPATH",
-			root:     "github.com/foo/pkg/",
-			expected: "",
-		},
-		{
-			desc:     "vendor level 1",
-			root:     "github.com/foo/pkg/vendor/guthib.com/traefik/fromage",
-			expected: "github.com/foo/pkg",
-		},
-		{
-			desc:     "vendor level 2",
-			root:     "github.com/foo/pkg/vendor/guthib.com/traefik/fromage/vendor/guthib.com/traefik/fuu",
-			expected: "github.com/foo/pkg/vendor/guthib.com/traefik/fromage",
-		},
-		{
-			desc:           "vendor is sibling",
-			root:           "github.com/foo/bar",
-			rootPathSuffix: "testdata/src/github.com/foo/bar",
-			expected:       "github.com/foo",
-		},
-		{
-			desc:           "vendor is uncle",
-			root:           "github.com/foo/bar/baz",
-			rootPathSuffix: "testdata/src/github.com/foo/bar/baz",
-			expected:       "github.com/foo",
-		},
-	}
-
-	for _, test := range testCases {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			t.Parallel()
-
-			var rootPath string
-			if test.rootPathSuffix != "" {
-				wd, err := os.Getwd()
-				if err != nil {
-					t.Fatal(err)
-				}
-				rootPath = filepath.Join(wd, test.rootPathSuffix)
-			} else {
-				rootPath = vendor
-			}
-			p, err := previousRoot(&realFS{}, rootPath, test.root)
-			if err != nil {
-				t.Error(err)
-			}
-
-			if p != test.expected {
-				t.Errorf("got: %s, want: %s", p, test.expected)
+			if test.root != test.expected.rpath {
+				t.Errorf(" [rpath] got: %s, want: %s", test.root, test.expected.rpath)
 			}
 		})
 	}