From b5b6796d20d8e024b541796a5fc7d804a0683e24 Mon Sep 17 00:00:00 2001 From: David Cohen Date: Fri, 29 Jul 2022 13:36:18 -0400 Subject: [PATCH 1/3] feat(ci retry): retry multiple jobs or entire pipeline Expands retry command to retry recent pipelines, if they have failed jobs. Can monitor for failures and retry repeatedly when new pipelines appear, until interrupted. --- commands/ci/retry/retry.go | 103 +++++++++++++++++++++++++++++++++---- 1 file changed, 92 insertions(+), 11 deletions(-) diff --git a/commands/ci/retry/retry.go b/commands/ci/retry/retry.go index 67c276d8c..4fcf96bc1 100644 --- a/commands/ci/retry/retry.go +++ b/commands/ci/retry/retry.go @@ -2,9 +2,11 @@ package retry import ( "fmt" + "time" "github.com/profclems/glab/api" "github.com/profclems/glab/commands/cmdutils" + "github.com/profclems/glab/pkg/git" "github.com/profclems/glab/pkg/utils" "github.com/MakeNowJust/heredoc" @@ -17,10 +19,11 @@ func NewCmdRetry(f *cmdutils.Factory) *cobra.Command { Short: `Retry a CI job`, Aliases: []string{}, Example: heredoc.Doc(` - $ glab ci retry 871528 + $ glab ci retry 871528 # retries a specific job, 871528 + $ glab ci retry # retries most recent pipeline, if retry is necessary + $ glab ci retry --follow # continues to retry most recent pipeline, until interrupted `), Long: ``, - Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { var err error @@ -34,23 +37,101 @@ func NewCmdRetry(f *cmdutils.Factory) *cobra.Command { return err } - jobID := utils.StringToInt(args[0]) + for i := range args { + jobID := utils.StringToInt(args[i]) - if jobID < 1 { - fmt.Fprintln(f.IO.StdErr, "invalid job id:", args[0]) - return cmdutils.SilentError + if jobID < 1 { + fmt.Fprintln(f.IO.StdErr, "invalid job id:", args[0]) + return cmdutils.SilentError + } + + job, err := api.RetryPipelineJob(apiClient, jobID, repo.FullName()) + if err != nil { + return cmdutils.WrapError(err, fmt.Sprintf("Could not retry job with ID: %d", jobID)) + } + fmt.Fprintln(f.IO.StdOut, "Retried job (id:", job.ID, "), status:", job.Status, ", ref:", job.Ref, ", weburl: ", job.WebURL, ")") + } + if len(args) > 0 { + // jobs specified on command line are retried, nothing more to do + return nil } - job, err := api.RetryPipelineJob(apiClient, jobID, repo.FullName()) - if err != nil { - return cmdutils.WrapError(err, fmt.Sprintf("Could not retry job with ID: %d", jobID)) + // retry all failed jobs in pipeline + + follow, _ := cmd.Flags().GetBool("follow") + branch, _ := cmd.Flags().GetString("branch") + if branch == "" { + branch, err = git.CurrentBranch() + if err != nil { + return err + } } - fmt.Fprintln(f.IO.StdOut, "Retried job (id:", job.ID, "), status:", job.Status, ", ref:", job.Ref, ", weburl: ", job.WebURL, ")") - return nil + attempts := map[int]int{} // key is pipeline id, value is how may retries + + for i := 0; i == 0 || follow; i++ { + if i > 0 { + // pause for retries triggered by prior iteration + time.Sleep(30 * time.Minute) + } + + lastPipeline, err := api.GetLastPipeline(apiClient, repo.FullName(), branch) + if err != nil { + if follow { + continue + } + fmt.Fprintf(f.IO.StdOut, "No pipelines running or available on %s branch\n", branch) + return err + } + + switch lastPipeline.Status { + case "canceled", "pending", "success", "skipped": + // nothing to retry + continue + default: // "running", "failed", "created" + // look for any failed jobs + failed := false + jobs, err := api.GetPipelineJobs(apiClient, lastPipeline.ID, repo.FullName()) + if err != nil { + return err + } + for j := range jobs { + if jobs[j].Status == "failed" { + if jobs[j].AllowFailure { + fmt.Fprintf(f.IO.StdErr, "failed job (%s) allows failure, ignoring", jobs[i].WebURL) + continue + } + + failed = true + break + } + } + if !failed { + continue // continue main loop, nothing to retry + } + } + + count := attempts[lastPipeline.ID] + if count >= 3 { + fmt.Fprintf(f.IO.StdErr, "giving up on pipeline (%d), too many retries (%d)", lastPipeline.ID, count) + continue + } + attempts[lastPipeline.ID] = count + 1 + + fmt.Fprintf(f.IO.StdOut, "retrying pipeline (%s)", lastPipeline.WebURL) + _, err = api.RetryPipeline(apiClient, lastPipeline.ID, repo.FullName()) + if err != nil { + fmt.Fprintf(f.IO.StdErr, "failed to retry pipeline (%s): %+v", lastPipeline.WebURL, err) + } + } + + return nil }, } + pipelineRetryCmd.Flags().StringP("branch", "b", "", "Retry latest pipeline associated with branch. (Default is current branch)") + pipelineRetryCmd.Flags().BoolP("follow", "f", false, "Retry when needed, until interrupted.") + return pipelineRetryCmd } From 797dc3fb9a7914385fd9c54ab4a1623161369267 Mon Sep 17 00:00:00 2001 From: dcohen Date: Fri, 5 Aug 2022 14:16:52 -0400 Subject: [PATCH 2/3] produce a helpful error when token expired --- commands/ci/retry/retry.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/commands/ci/retry/retry.go b/commands/ci/retry/retry.go index 4fcf96bc1..599e0ac4b 100644 --- a/commands/ci/retry/retry.go +++ b/commands/ci/retry/retry.go @@ -1,6 +1,7 @@ package retry import ( + "errors" "fmt" "time" @@ -8,6 +9,7 @@ import ( "github.com/profclems/glab/commands/cmdutils" "github.com/profclems/glab/pkg/git" "github.com/profclems/glab/pkg/utils" + "github.com/xanzy/go-gitlab" "github.com/MakeNowJust/heredoc" "github.com/spf13/cobra" @@ -77,10 +79,16 @@ func NewCmdRetry(f *cmdutils.Factory) *cobra.Command { lastPipeline, err := api.GetLastPipeline(apiClient, repo.FullName(), branch) if err != nil { + var response *gitlab.ErrorResponse + if errors.As(err, &response) { + if response.Response.StatusCode == 401 { + return errors.New("unauthorized, try \"glab auth login\"") + } + } + fmt.Fprintf(f.IO.StdOut, "No pipelines running or available on %q branch: %+v\n", branch, err) if follow { continue } - fmt.Fprintf(f.IO.StdOut, "No pipelines running or available on %s branch\n", branch) return err } From 68e39eb6328b3d8c1a5fd17b4f35b6e184dfcb20 Mon Sep 17 00:00:00 2001 From: dcohen Date: Fri, 5 Aug 2022 14:17:27 -0400 Subject: [PATCH 3/3] retry a failed pipeline by status, skipping job inspection --- commands/ci/retry/retry.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/commands/ci/retry/retry.go b/commands/ci/retry/retry.go index 599e0ac4b..d7bb97bf9 100644 --- a/commands/ci/retry/retry.go +++ b/commands/ci/retry/retry.go @@ -98,23 +98,27 @@ func NewCmdRetry(f *cmdutils.Factory) *cobra.Command { continue default: // "running", "failed", "created" - // look for any failed jobs - failed := false - jobs, err := api.GetPipelineJobs(apiClient, lastPipeline.ID, repo.FullName()) - if err != nil { - return err - } - for j := range jobs { - if jobs[j].Status == "failed" { - if jobs[j].AllowFailure { - fmt.Fprintf(f.IO.StdErr, "failed job (%s) allows failure, ignoring", jobs[i].WebURL) - continue - } + failed := lastPipeline.Status == "failed" - failed = true - break + if !failed { + // look for any failed jobs + jobs, err := api.GetPipelineJobs(apiClient, lastPipeline.ID, repo.FullName()) + if err != nil { + return err + } + for j := range jobs { + if jobs[j].Status == "failed" { + if jobs[j].AllowFailure { + fmt.Fprintf(f.IO.StdErr, "failed job (%s) allows failure, ignoring", jobs[i].WebURL) + continue + } + + failed = true + break + } } } + if !failed { continue // continue main loop, nothing to retry } @@ -127,7 +131,7 @@ func NewCmdRetry(f *cmdutils.Factory) *cobra.Command { } attempts[lastPipeline.ID] = count + 1 - fmt.Fprintf(f.IO.StdOut, "retrying pipeline (%s)", lastPipeline.WebURL) + fmt.Fprintf(f.IO.StdOut, "retrying pipeline (%s)\n", lastPipeline.WebURL) _, err = api.RetryPipeline(apiClient, lastPipeline.ID, repo.FullName()) if err != nil { fmt.Fprintf(f.IO.StdErr, "failed to retry pipeline (%s): %+v", lastPipeline.WebURL, err)