diff --git a/commands/ci/retry/retry.go b/commands/ci/retry/retry.go index 67c276d8c..d7bb97bf9 100644 --- a/commands/ci/retry/retry.go +++ b/commands/ci/retry/retry.go @@ -1,11 +1,15 @@ package retry import ( + "errors" "fmt" + "time" "github.com/profclems/glab/api" "github.com/profclems/glab/commands/cmdutils" + "github.com/profclems/glab/pkg/git" "github.com/profclems/glab/pkg/utils" + "github.com/xanzy/go-gitlab" "github.com/MakeNowJust/heredoc" "github.com/spf13/cobra" @@ -17,10 +21,11 @@ func NewCmdRetry(f *cmdutils.Factory) *cobra.Command { Short: `Retry a CI job`, Aliases: []string{}, Example: heredoc.Doc(` - $ glab ci retry 871528 + $ glab ci retry 871528 # retries a specific job, 871528 + $ glab ci retry # retries most recent pipeline, if retry is necessary + $ glab ci retry --follow # continues to retry most recent pipeline, until interrupted `), Long: ``, - Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { var err error @@ -34,23 +39,111 @@ func NewCmdRetry(f *cmdutils.Factory) *cobra.Command { return err } - jobID := utils.StringToInt(args[0]) + for i := range args { + jobID := utils.StringToInt(args[i]) - if jobID < 1 { - fmt.Fprintln(f.IO.StdErr, "invalid job id:", args[0]) - return cmdutils.SilentError + if jobID < 1 { + fmt.Fprintln(f.IO.StdErr, "invalid job id:", args[0]) + return cmdutils.SilentError + } + + job, err := api.RetryPipelineJob(apiClient, jobID, repo.FullName()) + if err != nil { + return cmdutils.WrapError(err, fmt.Sprintf("Could not retry job with ID: %d", jobID)) + } + fmt.Fprintln(f.IO.StdOut, "Retried job (id:", job.ID, "), status:", job.Status, ", ref:", job.Ref, ", weburl: ", job.WebURL, ")") + } + if len(args) > 0 { + // jobs specified on command line are retried, nothing more to do + return nil } - job, err := api.RetryPipelineJob(apiClient, jobID, repo.FullName()) - if err != nil { - return cmdutils.WrapError(err, fmt.Sprintf("Could not retry job with ID: %d", jobID)) + // retry all failed jobs in pipeline + + follow, _ := cmd.Flags().GetBool("follow") + branch, _ := cmd.Flags().GetString("branch") + if branch == "" { + branch, err = git.CurrentBranch() + if err != nil { + return err + } } - fmt.Fprintln(f.IO.StdOut, "Retried job (id:", job.ID, "), status:", job.Status, ", ref:", job.Ref, ", weburl: ", job.WebURL, ")") - return nil + attempts := map[int]int{} // key is pipeline id, value is how may retries + + for i := 0; i == 0 || follow; i++ { + if i > 0 { + // pause for retries triggered by prior iteration + time.Sleep(30 * time.Minute) + } + + lastPipeline, err := api.GetLastPipeline(apiClient, repo.FullName(), branch) + if err != nil { + var response *gitlab.ErrorResponse + if errors.As(err, &response) { + if response.Response.StatusCode == 401 { + return errors.New("unauthorized, try \"glab auth login\"") + } + } + fmt.Fprintf(f.IO.StdOut, "No pipelines running or available on %q branch: %+v\n", branch, err) + if follow { + continue + } + return err + } + + switch lastPipeline.Status { + case "canceled", "pending", "success", "skipped": + // nothing to retry + continue + + default: // "running", "failed", "created" + failed := lastPipeline.Status == "failed" + if !failed { + // look for any failed jobs + jobs, err := api.GetPipelineJobs(apiClient, lastPipeline.ID, repo.FullName()) + if err != nil { + return err + } + for j := range jobs { + if jobs[j].Status == "failed" { + if jobs[j].AllowFailure { + fmt.Fprintf(f.IO.StdErr, "failed job (%s) allows failure, ignoring", jobs[i].WebURL) + continue + } + + failed = true + break + } + } + } + + if !failed { + continue // continue main loop, nothing to retry + } + } + + count := attempts[lastPipeline.ID] + if count >= 3 { + fmt.Fprintf(f.IO.StdErr, "giving up on pipeline (%d), too many retries (%d)", lastPipeline.ID, count) + continue + } + attempts[lastPipeline.ID] = count + 1 + + fmt.Fprintf(f.IO.StdOut, "retrying pipeline (%s)\n", lastPipeline.WebURL) + _, err = api.RetryPipeline(apiClient, lastPipeline.ID, repo.FullName()) + if err != nil { + fmt.Fprintf(f.IO.StdErr, "failed to retry pipeline (%s): %+v", lastPipeline.WebURL, err) + } + } + + return nil }, } + pipelineRetryCmd.Flags().StringP("branch", "b", "", "Retry latest pipeline associated with branch. (Default is current branch)") + pipelineRetryCmd.Flags().BoolP("follow", "f", false, "Retry when needed, until interrupted.") + return pipelineRetryCmd }