How to overcome Cloud Run's 32MB request limit

from typing import Optional
from datetime import timedelta

from google import auth
from google.auth.transport import requests
from google.cloud.storage import Client


def make_signed_upload_url(
    bucket: str,
    blob: str,
    *,
    exp: Optional[timedelta] = None,
    content_type="application/octet-stream",
    min_size=1,
    max_size=int(1e6)
):
    """
    Compute a GCS signed upload URL without needing a private key file.
    Can only be called when a service account is used as the application
    default credentials, and when that service account has the proper IAM
    roles, like `roles/storage.objectCreator` for the bucket, and
    `roles/iam.serviceAccountTokenCreator`.
    Source: https://stackoverflow.com/a/64245028

    Parameters
    ----------
    bucket : str
        Name of the GCS bucket the signed URL will reference.
    blob : str
        Name of the GCS blob (in `bucket`) the signed URL will reference.
    exp : timedelta, optional
        Time from now when the signed url will expire.
    content_type : str, optional
        The required mime type of the data that is uploaded to the generated
        signed url.
    min_size : int, optional
        The minimum size the uploaded file can be, in bytes (inclusive).
        If the file is smaller than this, GCS will return a 400 code on upload.
    max_size : int, optional
        The maximum size the uploaded file can be, in bytes (inclusive).
        If the file is larger than this, GCS will return a 400 code on upload.
    """
    if exp is None:
        exp = timedelta(hours=1)
    credentials, project_id = auth.default()
    if credentials.token is None:
        # Perform a refresh request to populate the access token of the
        # current credentials.
        credentials.refresh(requests.Request())
    client = Client()
    bucket = client.get_bucket(bucket)
    blob = bucket.blob(blob)
    return blob.generate_signed_url(
        version="v4",
        expiration=exp,
        service_account_email=credentials.service_account_email,
        access_token=credentials.token,
        method="PUT",
        content_type=content_type,
        headers={"X-Goog-Content-Length-Range": f"{min_size},{max_size}"}
    )

# Resources to handle big data files (>32 Mb)
# These files are uploaded to a special bucket with notifications

provider "google-beta" {
  project = <your GCP project name>
}

data "google_project" "default" {
  provider = google-beta
}

resource "google_storage_bucket" "bigframes_bucket" {
  project  = <your GCP project name>
  name     = "upload-big-files"
  location = "EU"

  cors {
    origin = ["*"]
    method = ["*"]
    response_header = [
      "Content-Type",
      "Access-Control-Allow-Origin",
      "X-Goog-Content-Length-Range"
    ]
    max_age_seconds = 3600
  }
}

resource "google_service_account" "default" {
  provider     = google-beta
  account_id   = "sa-webservice"
}

resource "google_storage_bucket_iam_member" "bigframes_admin" {
  bucket = google_storage_bucket.bigframes_bucket.name
  role   = "roles/storage.admin"
  member = "serviceAccount:${google_service_account.default.email}"
}

# required to generate a signed url
resource "google_service_account_iam_member" "tokencreator" {
  provider           = google-beta
  service_account_id = google_service_account.default.name
  role               = "roles/iam.serviceAccountTokenCreator"
  member             = "serviceAccount:${google_service_account.default.email}"
}

# upload topic for notifications
resource "google_pubsub_topic" "bigframes_topic" {
  provider = google-beta
  name     = "topic-bigframes"
}

# upload deadletter topic for failed notifications
resource "google_pubsub_topic" "bigframes_topic_deadletter" {
  provider = google-beta
  name     = "topic-bigframesdeadletter"
}

# add frame upload notifications on the bucket
resource "google_storage_notification" "bigframes_notification" {
  provider       = google-beta
  bucket         = google_storage_bucket.bigframes_bucket.name
  payload_format = "JSON_API_V1"
  topic          = google_pubsub_topic.bigframes_topic.id
  event_types    = ["OBJECT_FINALIZE"]
  depends_on = [google_pubsub_topic_iam_binding.bigframes_binding]
}

# required for storage notifications
# seriously, Google, this should be by default !
resource "google_pubsub_topic_iam_binding" "bigframes_binding" {
  topic   = google_pubsub_topic.bigframes_topic.id
  role    = "roles/pubsub.publisher"
  members = ["serviceAccount:service-${data.google_project.default.number}@gs-project-accounts.iam.gserviceaccount.com"]
}

# frame upload main sub
resource "google_pubsub_subscription" "bigframes_sub" {
  provider = google-beta
  name     = "sub-bigframes"
  topic    = google_pubsub_topic.bigframes_topic.id

  push_config {
    push_endpoint = <URL where pushed notification are POST-ed>
  }
  dead_letter_policy {
    dead_letter_topic = google_pubsub_topic.bigframes_topic_deadletter.id
  }
}

# frame upload deadletter subscription
resource "google_pubsub_subscription" "bigframes_sub_deadletter" {
  provider             = google-beta
  name                 = "sub-bigframesdeadletter"
  topic                = google_pubsub_topic.bigframes_topic_deadletter.id
  ack_deadline_seconds = 600

  push_config {
    push_endpoint = <URL where pushed notification are POST-ed>
  }
}