DEVPROD-29991 Ensure git_ref_not_found is set properly (evergreen-ci#9968)

Author: malikchaya2

agent/command/git.go

@@ -36,10 +36,6 @@ const (
 	gitGetProjectAttribute = "evergreen.command.git_get_project"
 
 	generatedTokenKey = "EVERGREEN_GENERATED_GITHUB_TOKEN"
-
-	// githubMergeQueueInvalidRefError is the error message returned by Git when it fails to find
-	// a GitHub merge queue reference.
-	githubMergeQueueInvalidRefError = "couldn't find remote ref gh-readonly-queue"
 )
 
 var (
@@ -74,6 +70,8 @@ type gitFetchProject struct {
 
 	CommitterEmail string `mapstructure:"committer_email"`
 
+	refNotFound bool
+
 	base
 }
 
@@ -360,6 +358,22 @@ func (c *gitFetchProject) fetchSource(ctx context.Context, logger client.LoggerP
 	attempt := 0
 	return c.retryFetch(ctx, logger, comm, conf, true, opts, func(opts cloneOpts) error {
 		attempt++
+		// On the second attempt, check if the merge queue ref was deleted before retrying.
+		if attempt == 2 && conf.Task.Requester == evergreen.GithubMergeRequester && conf.GithubMergeData.HeadBranch != "" {
+			ref := "heads/" + conf.GithubMergeData.HeadBranch
+			appToken, tokenErr := comm.CreateInstallationTokenForClone(ctx, conf.TaskData(), opts.owner, opts.repo)
+			if tokenErr == nil && appToken != "" {
+				exists, checkErr := thirdparty.MergeQueueRefExists(ctx, opts.owner, opts.repo, ref, appToken)
+				if checkErr != nil {
+					return errors.Wrap(checkErr, "checking if merge queue ref exists")
+				}
+				if !exists {
+					c.refNotFound = true
+					return errors.New("the GitHub merge SHA is not available most likely because the merge completed or was aborted")
+				}
+			}
+		}
+
 		gitCommands, err := c.buildSourceCloneCommand(conf, opts)
 		if err != nil {
 			return err
@@ -389,9 +403,10 @@ func (c *gitFetchProject) fetchSource(ctx context.Context, logger client.LoggerP
 
 		if err = fetchSourceCmd.Run(ctx); err != nil {
 			span.SetAttributes(attribute.String(cloneErrorAttribute, err.Error()))
+			return err
 		}
 
-		return err
+		return nil
 	})
 }
 
@@ -410,6 +425,9 @@ func (c *gitFetchProject) retryFetch(ctx context.Context, logger client.LoggerPr
 	return utility.Retry(
 		ctx,
 		func() (bool, error) {
+			if isSource {
+				c.refNotFound = false
+			}
 			if attemptNum > 2 {
 				opts.useVerbose = true // use verbose for the last 2 attempts
 				logger.Task().Error(message.Fields{
@@ -423,7 +441,7 @@ func (c *gitFetchProject) retryFetch(ctx context.Context, logger client.LoggerPr
 				if isSource && attemptNum == 1 {
 					logger.Task().Warning("git source clone failed with cached merge SHA; re-requesting merge SHA from GitHub")
 				}
-				if strings.Contains(err.Error(), githubMergeQueueInvalidRefError) {
+				if isSource && c.refNotFound {
 					if markErr := comm.MarkMergeQueueGitRefNotFound(ctx, conf.TaskData()); markErr != nil {
 						logger.Task().Warningf("Failed to mark git ref not found: %s", markErr)
 					}

agent/command/git_test.go

@@ -242,7 +242,8 @@ func (s *GitGetProjectSuite) TestRetryFetchStopsOnInvalidGitHubMergeQueueRef() {
 	attempt := 0
 	err = c.retryFetch(s.ctx, logger, s.comm, conf, true, opts, func(o cloneOpts) error {
 		attempt++
-		return errors.Errorf("fatel: %s", githubMergeQueueInvalidRefError)
+		c.refNotFound = true
+		return errors.New("the GitHub merge SHA is not available most likely because the merge completed or was aborted")
 	})
 
 	s.Equal(1, attempt)

rest/data/patch.go

@@ -177,7 +177,7 @@ func SetMergeQueueGitRefNotFound(ctx context.Context, versionId string) error {
 		},
 	}
 
-	return patch.UpdateOne(ctx, patch.ById(p.Id), update)
+	return patch.UpdateOne(ctx, mgobson.M{patch.IdKey: p.Id}, update)
 }
 
 // FindPatchesByUser finds patches for the input user as ordered by creation time

thirdparty/github.go

@@ -1161,6 +1161,49 @@ func GetTaggedCommitFromGithub(ctx context.Context, owner, repo, tag string) (st
 	return sha, nil
 }
 
+// MergeQueueRefExists checks if a GitHub ref exists. The ref should be "heads/branch-name"
+// (e.g. "heads/gh-readonly-queue/main/pr-515-abc123").
+// If token is non-empty, it will be used for authentication. Otherwise, an installation
+// token is created from Evergreen config (requires server environment).
+func MergeQueueRefExists(ctx context.Context, owner, repo, ref string, token string) (bool, error) {
+	caller := "MergeQueueRefExists"
+	ctx, span := tracer.Start(ctx, caller, trace.WithAttributes(
+		attribute.String(githubOwnerAttribute, owner),
+		attribute.String(githubRepoAttribute, repo),
+		attribute.String(githubRefAttribute, ref),
+	))
+	defer span.End()
+
+	var authToken string
+	if token != "" {
+		authToken = token
+	} else {
+		var err error
+		authToken, err = getInstallationToken(ctx, owner, repo, nil)
+		if err != nil {
+			return false, errors.Wrap(err, "getting installation token")
+		}
+	}
+
+	githubClient := getGithubClient(authToken, caller, retryConfig{retry: true})
+	defer githubClient.Close()
+
+	_, resp, err := githubClient.Git.GetRef(ctx, owner, repo, ref)
+	if resp != nil {
+		defer resp.Body.Close()
+		if resp.StatusCode == http.StatusNotFound {
+			return false, nil
+		}
+		if err != nil {
+			return false, parseGithubErrorResponse(resp)
+		}
+	}
+	if err != nil {
+		return false, errors.Wrapf(err, "checking if ref '%s' exists", ref)
+	}
+	return true, nil
+}
+
 func getObjectTag(ctx context.Context, owner, repo, sha string) (*github.Tag, error) {
 	caller := "getObjectTag"
 	ctx, span := tracer.Start(ctx, caller, trace.WithAttributes(

thirdparty/github_test.go

@@ -240,6 +240,20 @@ func (s *githubSuite) TestGetTaggedCommitFromGithub() {
 	})
 }
 
+func (s *githubSuite) TestMergeQueueRefExists() {
+	s.Run("RefDoesNotExist", func() {
+		exists, err := MergeQueueRefExists(s.ctx, "evergreen-ci", "evergreen", "heads/gh-readonly-queue/main/pr-999999999-nonexistent", "")
+		s.NoError(err)
+		s.False(exists)
+	})
+
+	s.Run("RefExists", func() {
+		exists, err := MergeQueueRefExists(s.ctx, "evergreen-ci", "evergreen", "heads/main", "")
+		s.NoError(err)
+		s.True(exists)
+	})
+}
+
 func (s *githubSuite) TestGetBranchEvent() {
 	branch, err := GetBranchEvent(s.ctx, "evergreen-ci", "evergreen", "main")
 	s.NoError(err)