diff --git a/README.md b/README.md index 058ff3be13..7520bdf002 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,9 @@ example-runnerdeploy2475h595fr mumoshu/actions-runner-controller-ci Running example-runnerdeploy2475ht2qbr mumoshu/actions-runner-controller-ci Running ``` -#### Autoscaling +### Autoscaling + +#### Repository runners Autoscaling `RunnerDeployment` can scale the number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs. @@ -241,6 +243,8 @@ The scale out performance is controlled via the manager containers startup `--sy Additionally, the autoscaling feature has an anti-flapping option that prevents periodic loop of scaling up and down. By default, it doesn't scale down until the grace period of 10 minutes passes after a scale up. The grace period can be configured by setting `scaleDownDelaySecondsAfterScaleUp`: +Please note that if your `RunnerDeployment` has the `Repository` key set, then do not use the `` notation, only provide the ``. + ```yaml apiVersion: actions.summerwind.dev/v1alpha1 kind: RunnerDeployment @@ -267,6 +271,36 @@ spec: - summerwind/actions-runner-controller ``` +#### Organization runners Autoscaling +To autoscale on an organizational level, you need to remove the `repositoryNames` mapping and leave it empty. The Github Actions API doesn’t offer an endpoint to check the currently queued workflows on an organizational level. The way how the controller tries to get around this is by - after each `sync-period` - select the repositories with the latest `pushed` time and check the Actions workflow queue of those repositories. At the moment, the controller checks the last 10 repositories. + +Please note, in case you want to autoscale your organization runners, that you should modify your Github organization permissions accordingly; for instance, if you are using an organization PAT (Personal Access Token), update the permissions of your PAT to allow the controller to list all the repositories under your organization. + +An example of scaling your organization runners is shown below: + +```yaml +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: example-runnerdeploy +spec: + template: + spec: + organization: "your-organization-name" +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: example-runnerdeploy-autoscaler +spec: + scaleTargetRef: + name: example-runnerdeploy + minReplicas: 1 + maxReplicas: 3 + metrics: + - type: TotalNumberOfQueuedAndInProgressWorkflowRuns +``` + ## Runner with DinD When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace. @@ -321,7 +355,7 @@ spec: requests: cpu: "2.0" memory: "4Gi" - # If set to false, there are no privileged container and you cannot use docker. + # If set to false, there are no privileged container and you cannot use docker. dockerEnabled: false # If set to true, runner pod container only 1 container that's expected to be able to run docker, too. # image summerwind/actions-runner-dind or custom one should be used with true -value diff --git a/controllers/autoscaling.go b/controllers/autoscaling.go index 95e9dfedf4..ea4222cc13 100644 --- a/controllers/autoscaling.go +++ b/controllers/autoscaling.go @@ -2,9 +2,10 @@ package controllers import ( "context" - "errors" "fmt" + "log" "strings" + "time" "github.com/summerwind/actions-runner-controller/api/v1alpha1" ) @@ -18,30 +19,61 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp var repos [][]string - repoID := rd.Spec.Template.Spec.Repository - if repoID == "" { - orgName := rd.Spec.Template.Spec.Organization - if orgName == "" { - return nil, fmt.Errorf("asserting runner deployment spec to detect bug: spec.template.organization should not be empty on this code path") - } + orgName := rd.Spec.Template.Spec.Organization + if orgName == "" { + return nil, fmt.Errorf("asserting runner deployment spec to detect bug: spec.template.organization should not be empty on this code path") + } - metrics := hra.Spec.Metrics + metrics := hra.Spec.Metrics + if len(metrics) == 0 { + return nil, fmt.Errorf("validating autoscaling metrics: one or more metrics is required") + } else if tpe := metrics[0].Type; tpe != v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns { + return nil, fmt.Errorf("validting autoscaling metrics: unsupported metric type %q: only supported value is %s", tpe, v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns) + } + + // Github Enterprise API doesn't offer an endpoint for checking the Actions queue on org level, neither does it offer an endpoint for checking what repos Actions is enabled. This means that, in case of GH Enterprise, you must pass a slice of repositories. + if len(metrics[0].RepositoryNames) < 1 && r.GitHubClient.GithubEnterprise { + return nil, fmt.Errorf("[ERROR] user didn't pass any repository! Please pass a list of repositories the controller has to monitor") + } - if len(metrics) == 0 { - return nil, fmt.Errorf("validating autoscaling metrics: one or more metrics is required") - } else if tpe := metrics[0].Type; tpe != v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns { - return nil, fmt.Errorf("validting autoscaling metrics: unsupported metric type %q: only supported value is %s", tpe, v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns) - } else if len(metrics[0].RepositoryNames) == 0 { - return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment") + // If the above conditionally didn't return an error, we automatically assume, in case on an empty repository slice, that an organization is used. + if len(metrics[0].RepositoryNames) == 0 { + enabledRepos, _, err := r.GitHubClient.Actions.ListEnabledReposInOrg(context.Background(), orgName, nil) + if err != nil { + return nil, fmt.Errorf("[ERROR] 'ListEnabledReposInOrg' failed with error message: %s", err) } - for _, repoName := range metrics[0].RepositoryNames { - repos = append(repos, []string{orgName, repoName}) + if len(enabledRepos.Repositories) < 1 { + return nil, fmt.Errorf("[ERROR] 'ListEnabledReposInOrg' returned an empty slice of repositories, check your permissions. Error message: %s", err) } - } else { - repo := strings.Split(repoID, "/") - repos = append(repos, repo) + for _, v := range enabledRepos.Repositories { + repoName := fmt.Sprint(*v.Name) + + if *v.Archived || *v.Disabled { + continue + } + + lastChange := (int(time.Now().UTC().Sub(v.PushedAt.Time).Minutes())) + // We need a conditional here, since the `ListEnabledReposInOrg(ListOptions)` doesn't allow us to filter on `pushedAt`: https://docs.github.com/en/free-pro-team@latest/rest/reference/actions#list-selected-repositories-enabled-for-github-actions-in-an-organization + if lastChange > int(r.SyncPeriod.Minutes()) { + continue + } else if len(repos) < 20 { + repos = append(repos, []string{orgName, repoName}) + } + } + log.Printf("[INFO] watching the following organizational repositories: %s", repos) + + } else { + repoID := rd.Spec.Template.Spec.Repository + if repoID == "" { + for _, repoName := range metrics[0].RepositoryNames { + repos = append(repos, []string{orgName, repoName}) + } + } else { + repo := strings.Split(repoID, "/") + repos = append(repos, repo) + } } var total, inProgress, queued, completed, unknown int diff --git a/controllers/horizontalrunnerautoscaler_controller.go b/controllers/horizontalrunnerautoscaler_controller.go index 5dcbfdc298..5bc564ece0 100644 --- a/controllers/horizontalrunnerautoscaler_controller.go +++ b/controllers/horizontalrunnerautoscaler_controller.go @@ -46,6 +46,7 @@ type HorizontalRunnerAutoscalerReconciler struct { Log logr.Logger Recorder record.EventRecorder Scheme *runtime.Scheme + SyncPeriod *time.Duration } // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch diff --git a/github/github.go b/github/github.go index 5ab41d177d..cc36da68bc 100644 --- a/github/github.go +++ b/github/github.go @@ -29,16 +29,19 @@ type Client struct { regTokens map[string]*github.RegistrationToken mu sync.Mutex // GithubBaseURL to Github without API suffix. - GithubBaseURL string + GithubBaseURL string + GithubEnterprise bool } // NewClient creates a Github Client func (c *Config) NewClient() (*Client, error) { var ( - httpClient *http.Client - client *github.Client + httpClient *http.Client + client *github.Client + githubEnterprise bool ) githubBaseURL := "https://github.com/" + if len(c.Token) > 0 { httpClient = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource( &oauth2.Token{AccessToken: c.Token}, @@ -60,6 +63,8 @@ func (c *Config) NewClient() (*Client, error) { if len(c.EnterpriseURL) > 0 { var err error + githubEnterprise = true + client, err = github.NewEnterpriseClient(c.EnterpriseURL, c.EnterpriseURL, httpClient) if err != nil { return nil, fmt.Errorf("enterprise client creation failed: %v", err) @@ -67,13 +72,15 @@ func (c *Config) NewClient() (*Client, error) { githubBaseURL = fmt.Sprintf("%s://%s%s", client.BaseURL.Scheme, client.BaseURL.Host, strings.TrimSuffix(client.BaseURL.Path, "api/v3/")) } else { client = github.NewClient(httpClient) + githubEnterprise = false } return &Client{ - Client: client, - regTokens: map[string]*github.RegistrationToken{}, - mu: sync.Mutex{}, - GithubBaseURL: githubBaseURL, + Client: client, + regTokens: map[string]*github.RegistrationToken{}, + mu: sync.Mutex{}, + GithubBaseURL: githubBaseURL, + GithubEnterprise: githubEnterprise, }, nil } diff --git a/go.mod b/go.mod index 29c388a4a6..4c0731a68e 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,6 @@ require ( github.com/bradleyfalzon/ghinstallation v1.1.1 github.com/davecgh/go-spew v1.1.1 github.com/go-logr/logr v0.1.0 - github.com/google/go-github v17.0.0+incompatible // indirect - github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04 github.com/google/go-github/v33 v33.0.0 github.com/google/go-querystring v1.0.0 github.com/gorilla/mux v1.8.0 diff --git a/go.sum b/go.sum index d4c55973bc..f044df7f6f 100644 --- a/go.sum +++ b/go.sum @@ -116,12 +116,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts= github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= -github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04 h1:wEYk2h/GwOhImcVjiTIceP88WxVbXw2F+ARYUQMEsfg= -github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= diff --git a/main.go b/main.go index 63fcff1ffb..1133ade42c 100644 --- a/main.go +++ b/main.go @@ -148,6 +148,7 @@ func main() { Log: ctrl.Log.WithName("controllers").WithName("HorizontalRunnerAutoscaler"), Scheme: mgr.GetScheme(), GitHubClient: ghClient, + SyncPeriod: &syncPeriod, } if err = horizontalRunnerAutoscaler.SetupWithManager(mgr); err != nil {