Add tools distribution for python

go.mod

@@ -10,8 +10,10 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/hexops/autogold/v2 v2.1.0
 	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
+	github.com/mholt/archiver/v4 v4.0.0-alpha.8
 	github.com/olahol/melody v1.1.4
 	github.com/rs/cors v1.10.1
+	github.com/samber/lo v1.38.1
 	github.com/sashabaranov/go-openai v1.20.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
@@ -22,33 +24,49 @@ require (
 
 require (
 	github.com/acorn-io/baaah v0.0.0-20240119160309-2a58ee757bbd // indirect
+	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/bodgit/plumbing v1.2.0 // indirect
+	github.com/bodgit/sevenzip v1.3.0 // indirect
+	github.com/bodgit/windows v1.0.0 // indirect
 	github.com/bombsimon/logrusr/v4 v4.0.0 // indirect
+	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-containerregistry v0.16.1 // indirect
 	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hexops/gotextdiff v1.0.3 // indirect
 	github.com/hexops/valast v1.4.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/nightlyone/lockfile v1.0.0 // indirect
+	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onsi/ginkgo/v2 v2.13.0 // indirect
 	github.com/onsi/gomega v1.29.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/samber/lo v1.38.1 // indirect
 	github.com/samber/slog-logrus v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
+	github.com/therootcompany/xz v1.0.1 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
+	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
 	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.17.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect

pkg/cache/cache.go

@@ -22,7 +22,7 @@ type Options struct {
 	CacheDir string `usage:"Directory to store cache (default: $XDG_CACHE_HOME/gptscript)"`
 }
 
-func complete(opts ...Options) (result Options) {
+func Complete(opts ...Options) (result Options) {
 	for _, opt := range opts {
 		result.CacheDir = types.FirstSet(opt.CacheDir, result.CacheDir)
 		result.Cache = types.FirstSet(opt.Cache, result.Cache)
@@ -48,7 +48,7 @@ func WithNoCache(ctx context.Context) context.Context {
 }
 
 func New(opts ...Options) (*Client, error) {
-	opt := complete(opts...)
+	opt := Complete(opts...)
 	if err := os.MkdirAll(opt.CacheDir, 0755); err != nil {
 		return nil, err
 	}

pkg/cli/gptscript.go

@@ -18,6 +18,7 @@ import (
 	"github.com/gptscript-ai/gptscript/pkg/monitor"
 	"github.com/gptscript-ai/gptscript/pkg/mvl"
 	"github.com/gptscript-ai/gptscript/pkg/openai"
+	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes"
 	"github.com/gptscript-ai/gptscript/pkg/runner"
 	"github.com/gptscript-ai/gptscript/pkg/server"
 	"github.com/gptscript-ai/gptscript/pkg/types"
@@ -225,6 +226,7 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error {
 		MonitorFactory: monitor.NewConsole(monitor.Options(r.DisplayOptions), monitor.Options{
 			DisplayProgress: !*r.Quiet,
 		}),
+		RuntimeManager: runtimes.Default(cache.Complete(cache.Options(r.CacheOptions)).CacheDir),
 	})
 	if err != nil {
 		return err

pkg/debugcmd/debug.go

@@ -0,0 +1,20 @@
+package debugcmd
+
+import (
+	"context"
+	"os"
+	"os/exec"
+)
+
+func New(ctx context.Context, arg string, args ...string) *exec.Cmd {
+	cmd := exec.CommandContext(ctx, arg, args...)
+	SetupDebug(cmd)
+	return cmd
+}
+
+func SetupDebug(cmd *exec.Cmd) {
+	if log.IsDebug() {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.Stderr = os.Stderr
+}

pkg/debugcmd/log.go

@@ -0,0 +1,5 @@
+package debugcmd
+
+import "github.com/gptscript-ai/gptscript/pkg/mvl"
+
+var log = mvl.Package()

pkg/engine/cmd.go

@@ -8,6 +8,8 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"sort"
 	"strings"
 	"sync/atomic"
 
@@ -40,12 +42,7 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
 		return tool.BuiltinFunc(ctx, e.Env, input)
 	}
 
-	var extraEnv []string
-	if tool.WorkingDir != "" {
-		extraEnv = append(extraEnv, "GPTSCRIPT_TOOL_DIR="+tool.WorkingDir)
-	}
-
-	cmd, stop, err := e.newCommand(ctx, extraEnv, tool.Instructions, input)
+	cmd, stop, err := e.newCommand(ctx, nil, tool, input)
 	if err != nil {
 		return "", err
 	}
@@ -74,62 +71,106 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string)
 	return output.String(), nil
 }
 
-func (e *Engine) newCommand(ctx context.Context, extraEnv []string, instructions, input string) (*exec.Cmd, func(), error) {
-	env := append(e.Env[:], extraEnv...)
-	data := map[string]any{}
-
-	dec := json.NewDecoder(bytes.NewReader([]byte(input)))
-	dec.UseNumber()
+func (e *Engine) getRuntimeEnv(ctx context.Context, tool types.Tool, cmd, env []string) ([]string, error) {
+	var (
+		workdir = tool.WorkingDir
+		err     error
+	)
+	if e.RuntimeManager != nil {
+		workdir, env, err = e.RuntimeManager.GetContext(ctx, tool, cmd, env)
+		if err != nil {
+			return nil, err
+		}
+		workdir = filepath.Join(workdir, tool.Source.Repo.Path)
+	}
+	return append(env, "GPTSCRIPT_TOOL_DIR="+workdir), nil
+}
 
+func envAsMapAndDeDup(env []string) (sortedEnv []string, _ map[string]string) {
 	envMap := map[string]string{}
+	var keys []string
 	for _, env := range env {
 		key, value, _ := strings.Cut(env, "=")
+		if _, existing := envMap[key]; !existing {
+			keys = append(keys, key)
+		}
 		envMap[key] = value
 	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		sortedEnv = append(sortedEnv, key+"="+envMap[key])
+	}
+
+	return sortedEnv, envMap
+}
+
+var ignoreENV = map[string]struct{}{
+	"PATH":               {},
+	"GPTSCRIPT_TOOL_DIR": {},
+}
+
+func appendEnv(env []string, k, v string) []string {
+	for _, k := range []string{k, strings.ToUpper(strings.ReplaceAll(k, "-", "_"))} {
+		if _, ignore := ignoreENV[k]; !ignore {
+			env = append(env, k+"="+v)
+		}
+	}
+	return env
+}
+
+func appendInputAsEnv(env []string, input string) []string {
+	data := map[string]any{}
+	dec := json.NewDecoder(bytes.NewReader([]byte(input)))
+	dec.UseNumber()
+
+	if err := json.Unmarshal([]byte(input), &data); err != nil {
+		// ignore invalid JSON
+		return env
+	}
 
-	if err := json.Unmarshal([]byte(input), &data); err == nil {
-		for k, v := range data {
-			envName := strings.ToUpper(strings.ReplaceAll(k, "-", "_"))
-			switch val := v.(type) {
-			case string:
-				envMap[envName] = val
-				env = append(env, envName+"="+val)
-				envMap[k] = val
-				env = append(env, k+"="+val)
-			case json.Number:
-				envMap[envName] = string(val)
-				env = append(env, envName+"="+string(val))
-				envMap[k] = string(val)
-				env = append(env, k+"="+string(val))
-			case bool:
-				envMap[envName] = fmt.Sprint(val)
-				env = append(env, envName+"="+fmt.Sprint(val))
-				envMap[k] = fmt.Sprint(val)
-				env = append(env, k+"="+fmt.Sprint(val))
-			default:
-				data, err := json.Marshal(val)
-				if err == nil {
-					envMap[envName] = string(data)
-					env = append(env, envName+"="+string(data))
-					envMap[k] = string(data)
-					env = append(env, k+"="+string(data))
-				}
+	for k, v := range data {
+		switch val := v.(type) {
+		case string:
+			env = appendEnv(env, k, val)
+		case json.Number:
+			env = appendEnv(env, k, string(val))
+		case bool:
+			env = appendEnv(env, k, fmt.Sprint(val))
+		default:
+			data, err := json.Marshal(val)
+			if err == nil {
+				env = appendEnv(env, k, string(data))
 			}
 		}
 	}
 
-	interpreter, rest, _ := strings.Cut(instructions, "\n")
-	interpreter = strings.TrimSpace(interpreter)[2:]
+	return env
+}
 
-	interpreter = os.Expand(interpreter, func(s string) string {
-		return envMap[s]
-	})
+func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.Tool, input string) (*exec.Cmd, func(), error) {
+	env := append(e.Env[:], extraEnv...)
+	env = appendInputAsEnv(env, input)
+
+	interpreter, rest, _ := strings.Cut(tool.Instructions, "\n")
+	interpreter = strings.TrimSpace(interpreter)[2:]
 
 	args, err := shlex.Split(interpreter)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	env, err = e.getRuntimeEnv(ctx, tool, args, env)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	env, envMap := envAsMapAndDeDup(env)
+	for i, arg := range args {
+		args[i] = os.Expand(arg, func(s string) string {
+			return envMap[s]
+		})
+	}
+
 	var (
 		cmdArgs = args[1:]
 		stop    = func() {}

pkg/engine/daemon.go

@@ -74,6 +74,7 @@ func (e *Engine) startDaemon(_ context.Context, tool types.Tool) (string, error)
 
 	instructions := strings.TrimPrefix(tool.Instructions, types.DaemonPrefix)
 	instructions, path := getPath(instructions)
+	tool.Instructions = types.CommandPrefix + instructions
 
 	port, ok := daemonPorts[tool.ID]
 	url := fmt.Sprintf("http://127.0.0.1:%d%s", port, path)
@@ -92,7 +93,7 @@ func (e *Engine) startDaemon(_ context.Context, tool types.Tool) (string, error)
 	cmd, stop, err := e.newCommand(ctx, []string{
 		fmt.Sprintf("PORT=%d", port),
 	},
-		types.CommandPrefix+instructions,
+		tool,
 		"{}",
 	)
 	if err != nil {

pkg/engine/engine.go

@@ -26,10 +26,15 @@ type Model interface {
 	Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error)
 }
 
+type RuntimeManager interface {
+	GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error)
+}
+
 type Engine struct {
-	Model    Model
-	Env      []string
-	Progress chan<- types.CompletionStatus
+	Model          Model
+	RuntimeManager RuntimeManager
+	Env            []string
+	Progress       chan<- types.CompletionStatus
 }
 
 type State struct {

pkg/hash/sha256.go

@@ -6,13 +6,25 @@ import (
 	"encoding/json"
 )
 
+func ID(parts ...string) string {
+	d := sha256.New()
+	for i, part := range parts {
+		if i > 0 {
+			d.Write([]byte{0x00})
+		}
+		d.Write([]byte(part))
+	}
+	hash := d.Sum(nil)
+	return hex.EncodeToString(hash[:])
+}
+
 func Digest(obj any) string {
 	data, err := json.Marshal(obj)
 	if err != nil {
 		panic(err)
 	}
 
-	hash := sha256.Sum224(data)
+	hash := sha256.Sum256(data)
 	return hex.EncodeToString(hash[:])
 }
 
@@ -32,6 +44,6 @@ func Encode(obj any) string {
 		panic(err)
 	}
 
-	hash := sha256.Sum224(data)
+	hash := sha256.Sum256(data)
 	return hex.EncodeToString(hash[:])
 }