Cloud Task sporadically responds with 403

When scheduling Cloud Tasks in GCP from within a Cloud Run Function I sometimes (~40% of the time) get a 403 error once the Cloud Tasks runs (the second internal Cloud Run Function). I generate an OIDC token inside the scheduling Cloud Run Function that creates the task.

I'm puzzled why it works sometimes. Here is the code for invoking:

func (q *Queue) ScheduleTask(body UpgradeRequest, execTime time.Time) (*cloudtaskspb.Task, error) { if body.TaskId == "" { return nil, fmt.Errorf("cannot create task with empty task ID") } location := os.Getenv("LOCATION") queue := os.Getenv("QUEUE_NAME") updateURL := os.Getenv("UPDATE_URL") serviceAccount := os.Getenv("TASKS_SERVICE_ACCOUNT") projectID := os.Getenv("PROJECT_ID") payload, err := json.Marshal(body) if err != nil { return nil, fmt.Errorf("could not parse update request payload") } taskResponse, err := q.client.CreateTask(q.ctx, &cloudtaskspb.CreateTaskRequest{ Parent: fmt.Sprintf("projects/%s/locations/%s/queues/%s", projectID, location, queue), Task: &cloudtaskspb.Task{ MessageType: &cloudtaskspb.Task_HttpRequest{ HttpRequest: &cloudtaskspb.HttpRequest{ Url: updateURL, Body: payload, HttpMethod: cloudtaskspb.HttpMethod_PUT, AuthorizationHeader: &cloudtaskspb.HttpRequest_OidcToken{ OidcToken: &cloudtaskspb.OidcToken{ ServiceAccountEmail: serviceAccount, Audience: updateURL, }, }, }, }, ScheduleTime: timestamppb.New(execTime), }, ResponseView: 0, }) if err != nil { return nil, fmt.Errorf("failed to schedule upgrade") } return taskResponse, nil }

Here is the terraform to set the envs:

environment_variables = { PROJECT_ID = var.project_id LOCATION = var.region QUEUE_NAME = google_cloud_tasks_queue.queue.name # The SA of the second Cloud Run Function that is internal, triggered by the task TASKS_SERVICE_ACCOUNT = google_service_account.internal_function_sa.email UPDATE_URL = google_cloudfunctions2_function.update.url }