set up different credential types and prompt for model providers

Signed-off-by: Grant Linville <[email protected]>

Author: g-linville

pkg/credentials/credential.go

@@ -8,9 +8,17 @@ import (
 	"github.com/docker/cli/cli/config/types"
 )
 
+type CredentialType string
+
+const (
+	CredentialTypeTool          CredentialType = "tool"
+	CredentialTypeModelProvider CredentialType = "modelprovider"
+)
+
 type Credential struct {
 	Context  string            `json:"context"`
 	ToolName string            `json:"toolName"`
+	Type     CredentialType    `json:"type"`
 	Env      map[string]string `json:"env"`
 }
 
@@ -21,7 +29,7 @@ func (c Credential) toDockerAuthConfig() (types.AuthConfig, error) {
 	}
 
 	return types.AuthConfig{
-		Username:      "gptscript", // Username is required, but not used
+		Username:      string(c.Type),
 		Password:      string(env),
 		ServerAddress: toolNameWithCtx(c.ToolName, c.Context),
 	}, nil
@@ -33,14 +41,28 @@ func credentialFromDockerAuthConfig(authCfg types.AuthConfig) (Credential, error
 		return Credential{}, err
 	}
 
-	tool, ctx, err := toolNameAndCtxFromAddress(strings.TrimPrefix(authCfg.ServerAddress, "https://"))
+	// We used to hardcode the username as "gptscript" before CredentialType was introduced, so
+	// check for that here.
+	credType := authCfg.Username
+	if credType == "gptscript" {
+		credType = string(CredentialTypeTool)
+	}
+
+	// If it's a tool credential, remove the http[s] prefix.
+	address := authCfg.ServerAddress
+	if credType == string(CredentialTypeTool) {
+		address = strings.TrimPrefix(strings.TrimPrefix(address, "https://"), "http://")
+	}
+
+	tool, ctx, err := toolNameAndCtxFromAddress(address)
 	if err != nil {
 		return Credential{}, err
 	}
 
 	return Credential{
 		Context:  ctx,
 		ToolName: tool,
+		Type:     CredentialType(credType),
 		Env:      env,
 	}, nil
 }

pkg/credentials/helper.go

@@ -2,6 +2,8 @@ package credentials
 
 import (
 	"errors"
+	"regexp"
+	"strings"
 
 	"github.com/docker/cli/cli/config/credentials"
 	"github.com/docker/cli/cli/config/types"
@@ -55,7 +57,29 @@ func (h *HelperStore) GetAll() (map[string]types.AuthConfig, error) {
 		return nil, err
 	}
 
-	for serverAddress := range serverAddresses {
+	newCredAddresses := make(map[string]string, len(serverAddresses))
+	for serverAddress, val := range serverAddresses {
+		// If the serverAddress contains a port, we need to put it back in the right spot.
+		// For some reason, even when a credential is stored properly as http://hostname:8080///credctx,
+		// the list function will return http://hostname///credctx:8080. This is something wrong
+		// with macOS's built-in libraries. So we need to fix it here.
+		toolName, ctx, err := toolNameAndCtxFromAddress(serverAddress)
+		if err != nil {
+			return nil, err
+		}
+
+		possiblePortNumber := strings.Split(ctx, ":")[len(strings.Split(ctx, ":"))-1]
+		if regexp.MustCompile(`\d+$`).MatchString(possiblePortNumber) {
+			// port number confirmed
+			toolName = toolName + ":" + possiblePortNumber
+			ctx = strings.TrimSuffix(ctx, ":"+possiblePortNumber)
+		}
+
+		newCredAddresses[toolNameWithCtx(toolName, ctx)] = val
+		delete(serverAddresses, serverAddress)
+	}
+
+	for serverAddress := range newCredAddresses {
 		ac, err := h.Get(serverAddress)
 		if err != nil {
 			return nil, err

pkg/llm/registry.go

@@ -33,21 +33,7 @@ func (r *Registry) ListModels(ctx context.Context, providers ...string) (result
 	for _, v := range r.clients {
 		models, err := v.ListModels(ctx, providers...)
 		if err != nil {
-			// If we got back an InvalidAuthError, then we know it came from the OpenAI client, and we can
-			// try to get the credential from the cred store.
-			if errors.Is(err, openai.InvalidAuthError{}) {
-				if err := v.(*openai.Client).RetrieveAPIKey(); err != nil {
-					return nil, err
-				}
-
-				// Now that the API key has been retrieved, try to list models again.
-				models, err = v.ListModels(ctx, providers...)
-				if err != nil {
-					return nil, err
-				}
-			} else {
-				return nil, err
-			}
+			return nil, err
 		}
 		result = append(result, models...)
 	}
@@ -59,15 +45,36 @@ func (r *Registry) Call(ctx context.Context, messageRequest types.CompletionRequ
 	if messageRequest.Model == "" {
 		return nil, fmt.Errorf("model is required")
 	}
+
 	var errs []error
+	var oaiClient *openai.Client
 	for _, client := range r.clients {
 		ok, err := client.Supports(ctx, messageRequest.Model)
 		if err != nil {
+			// If we got an OpenAI invalid auth error back, store the OpenAI client for later.
+			if errors.Is(err, openai.InvalidAuthError{}) {
+				oaiClient = client.(*openai.Client)
+			}
+
 			errs = append(errs, err)
 		} else if ok {
 			return client.Call(ctx, messageRequest, status)
 		}
 	}
+
+	if len(errs) > 0 && oaiClient != nil {
+		// Prompt the user to enter their OpenAI API key and try again.
+		if err := oaiClient.RetrieveAPIKey(); err != nil {
+			return nil, err
+		}
+		ok, err := oaiClient.Supports(ctx, messageRequest.Model)
+		if err != nil {
+			return nil, err
+		} else if ok {
+			return oaiClient.Call(ctx, messageRequest, status)
+		}
+	}
+
 	if len(errs) == 0 {
 		return nil, fmt.Errorf("failed to find a model provider for model [%s]", messageRequest.Model)
 	}

pkg/openai/client.go

@@ -19,7 +19,6 @@ import (
 	"github.com/gptscript-ai/gptscript/pkg/prompt"
 	"github.com/gptscript-ai/gptscript/pkg/system"
 	"github.com/gptscript-ai/gptscript/pkg/types"
-	"github.com/tidwall/gjson"
 )
 
 const (
@@ -168,6 +167,12 @@ func (c *Client) Supports(ctx context.Context, modelName string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
+
+	if len(models) == 0 {
+		// We got no models back, which means our auth is invalid.
+		return false, InvalidAuthError{}
+	}
+
 	return slices.Contains(models, modelName), nil
 }
 
@@ -177,8 +182,9 @@ func (c *Client) ListModels(ctx context.Context, providers ...string) (result []
 		return nil, nil
 	}
 
+	// If auth is invalid, we just want to return nothing.
 	if err := c.ValidAuth(); err != nil {
-		return nil, err
+		return nil, nil
 	}
 
 	models, err := c.c.ListModels(ctx)
@@ -559,38 +565,11 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest,
 }
 
 func (c *Client) RetrieveAPIKey() error {
-	store, err := credentials.NewStore(c.cliCfg, c.credCtx)
-	if err != nil {
-		return err
-	}
-
-	cred, exists, err := store.Get(BuiltinCredName)
+	k, err := prompt.GetModelProviderCredential(BuiltinCredName, "OPENAI_API_KEY", "Please provide your OpenAI API key:", c.credCtx, c.cliCfg)
 	if err != nil {
 		return err
 	}
 
-	var k string
-	if exists {
-		k = cred.Env[key]
-	} else {
-		// SysPrompt doesn't use its first two arguments, so we can safely pass whatever to them
-		result, err := prompt.SysPrompt(context.Background(), nil, `{"message":"Please provide your OpenAI API key:","fields":"key","sensitive":"true"}`)
-		if err != nil {
-			return err
-		}
-
-		k = gjson.Get(result, "key").String()
-		if err := store.Add(credentials.Credential{
-			ToolName: BuiltinCredName,
-			Env: map[string]string{
-				"OPENAI_API_KEY": k,
-			},
-		}); err != nil {
-			return err
-		}
-		log.Infof("Saved API key as credential %s", BuiltinCredName)
-	}
-
 	c.c.SetAPIKey(k)
 	c.invalidAuth = false
 	return nil

pkg/prompt/credential.go

@@ -0,0 +1,47 @@
+package prompt
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/gptscript-ai/gptscript/pkg/config"
+	"github.com/gptscript-ai/gptscript/pkg/credentials"
+	"github.com/tidwall/gjson"
+)
+
+func GetModelProviderCredential(credName, env, message, credCtx string, cliCfg *config.CLIConfig) (string, error) {
+	store, err := credentials.NewStore(cliCfg, credCtx)
+	if err != nil {
+		return "", err
+	}
+
+	cred, exists, err := store.Get(credName)
+	if err != nil {
+		return "", err
+	}
+
+	var k string
+	if exists {
+		k = cred.Env[env]
+	} else {
+		// SysPrompt doesn't use its first two arguments, so we can safely pass whatever to them
+		result, err := SysPrompt(context.Background(), nil, fmt.Sprintf(`{"message":"%s","fields":"key","sensitive":"true"}`, message))
+		if err != nil {
+			return "", err
+		}
+
+		k = gjson.Get(result, "key").String()
+		if err := store.Add(credentials.Credential{
+			ToolName: credName,
+			Type:     credentials.CredentialTypeModelProvider,
+			Env: map[string]string{
+				env: k,
+			},
+		}); err != nil {
+			return "", err
+		}
+		log.Infof("Saved API key as credential %s", credName)
+	}
+
+	return k, nil
+}

pkg/prompt/log.go

@@ -0,0 +1,5 @@
+package prompt
+
+import "github.com/gptscript-ai/gptscript/pkg/mvl"
+
+var log = mvl.Package()

pkg/remote/remote.go

@@ -2,7 +2,6 @@ package remote
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/url"
 	"os"
@@ -16,6 +15,7 @@ import (
 	env2 "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/loader"
 	"github.com/gptscript-ai/gptscript/pkg/openai"
+	"github.com/gptscript-ai/gptscript/pkg/prompt"
 	"github.com/gptscript-ai/gptscript/pkg/runner"
 	"github.com/gptscript-ai/gptscript/pkg/types"
 )
@@ -63,21 +63,7 @@ func (c *Client) ListModels(ctx context.Context, providers ...string) (result []
 		}
 		models, err := client.ListModels(ctx, "")
 		if err != nil {
-			// If we got back an InvalidAuthError, then we know it came from the OpenAI client, and we can
-			// try to get the credential from the cred store.
-			if errors.Is(err, openai.InvalidAuthError{}) {
-				if err := client.RetrieveAPIKey(); err != nil {
-					return nil, err
-				}
-
-				// Now that the API key has been retrieved, try to list models again.
-				models, err = client.ListModels(ctx, "")
-				if err != nil {
-					return nil, err
-				}
-			} else {
-				return nil, err
-			}
+			return nil, err
 		}
 		for _, model := range models {
 			result = append(result, model+" from "+provider)
@@ -121,6 +107,16 @@ func (c *Client) clientFromURL(apiURL string) (*openai.Client, error) {
 		return nil, err
 	}
 	env := "GPTSCRIPT_PROVIDER_" + env2.ToEnvLike(parsed.Hostname()) + "_API_KEY"
+	key := os.Getenv(env)
+
+	if key == "" {
+		var err error
+		key, err = c.retrieveAPIKey(env, apiURL)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return openai.NewClient(c.cliCfg, c.credCtx, openai.Options{
 		BaseURL: apiURL,
 		Cache:   c.cache,
@@ -180,3 +176,7 @@ func (c *Client) load(ctx context.Context, toolName string) (*openai.Client, err
 	c.clients[toolName] = client
 	return client, nil
 }
+
+func (c *Client) retrieveAPIKey(env, url string) (string, error) {
+	return prompt.GetModelProviderCredential(url, env, fmt.Sprintf("Please provide your API key for %s", url), c.credCtx, c.cliCfg)
+}

pkg/runner/runner.go

@@ -860,6 +860,7 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env
 
 			cred = &credentials.Credential{
 				ToolName: credToolName,
+				Type:     credentials.CredentialTypeTool,
 				Env:      envMap.Env,
 			}