Add Terraform Code for Deploying Infrastructure for Node Pack Extract…

…ion Logic (#96)

* add node-pack-extract

* add node-pack-extract

* add node-pack-extract

* Modify input variable correctly

---------

Co-authored-by: James Kwon <[email protected]>

.gitignore

@@ -10,4 +10,8 @@ tmp/
 # Lazy way to make atlas usable on windows
 atlas.exe
 
-.env
+
+# terraform
+**/.terraform/*
+terraform.tfstate
+terraform.tfstate.*

infrastructure/examples/simple-node-pack-extract/main.tf

@@ -0,0 +1,57 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.14.1"
+    }
+  }
+}
+
+variable "prefix" {
+  type = string
+}
+
+variable "region" {
+  type    = string
+  default = "us-central1"
+}
+
+provider "google" {
+  region = var.region
+}
+
+resource "google_storage_bucket" "bucket" {
+  name     = "${var.prefix}-comfy-registry-bucket"
+  location = var.region
+}
+
+resource "google_service_account" "service_account" {
+  account_id = "${var.prefix}-comfy-registry-sa"
+}
+
+module "node_pack_extract_trigger" {
+  depends_on = [google_service_account.service_account, google_storage_bucket.bucket]
+  source     = "../../modules/node-pack-extract-trigger"
+  providers = {
+    google = google
+  }
+  region                      = var.region
+  bucket_name                 = google_storage_bucket.bucket.name
+  cloud_build_service_account = google_service_account.service_account.email
+  topic_name                  = "${var.prefix}-comfy-registry-event"
+  trigger_name                = "${var.prefix}-comfy-registry-event"
+  registry_backend_url        = "https://stagingapi.comfy.org"
+}
+
+output "trigger_id" {
+  value = module.node_pack_extract_trigger.trigger_id
+}
+output "topic_id" {
+  value = module.node_pack_extract_trigger.topic_id
+}
+output "bucket_notification_id" {
+  value = module.node_pack_extract_trigger.bucket_notification_id
+}
+output "bucket_name" {
+  value = google_storage_bucket.bucket.name
+}

infrastructure/modules/node-pack-extract-trigger/README.md

@@ -0,0 +1,10 @@
+# Trigger for node-pack-extract
+
+Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/)
+
+## Requirements
+
+- Google Cloud Account
+- Existing Google Cloud Storage public bucket where the Registry backend store the comfy node packs.
+- Existing Service Account that is whitelisted in [service_account_auth](../../../server/middleware/authentication/service_account_auth.go#65) middleware and with `Service Account Token Creator` Role.
+- [Connected repositories](https://cloud.google.com/build/docs/repositories) contains the [node-pack-extract](../../../node-pack-extract/) folder

infrastructure/modules/node-pack-extract-trigger/main.tf

@@ -0,0 +1,67 @@
+# get the existing GCS bucket
+data "google_storage_bucket" "bucket" {
+  name = var.bucket_name
+}
+
+# create a Pub/Sub topic
+resource "google_pubsub_topic" "topic" {
+  name = var.topic_name
+}
+
+# get the default GCS service account
+data "google_storage_project_service_account" "gcs_account" {
+}
+
+# Grant the GCS service account permission to publish to the Pub/Sub topic
+resource "google_pubsub_topic_iam_binding" "gcs_publisher" {
+  topic   = google_pubsub_topic.topic.name
+  role    = "roles/pubsub.publisher"
+  members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"]
+}
+
+# enable GCS Bucket Notification to Pub/Sub
+resource "google_storage_notification" "notification" {
+  bucket         = data.google_storage_bucket.bucket.name
+  topic          = google_pubsub_topic.topic.id
+  payload_format = "JSON_API_V1"
+  depends_on     = [google_pubsub_topic_iam_binding.gcs_publisher]
+  event_types = [
+    "OBJECT_FINALIZE", # Triggered when an object is successfully created or overwritten
+  ]
+}
+
+
+# Get the existing cloudbuild service account
+data "google_service_account" "cloudbuild_service_account" {
+  account_id = var.cloud_build_service_account
+}
+
+# Create the cloud build trigger
+resource "google_cloudbuild_trigger" "trigger" {
+  name            = var.trigger_name
+  location        = var.region
+  service_account = data.google_service_account.cloudbuild_service_account.id
+
+  pubsub_config {
+    topic = google_pubsub_topic.topic.id
+  }
+
+  source_to_build {
+    uri       = var.git_repo_uri
+    ref       = "refs/heads/${var.git_repo_branch}"
+    repo_type = "GITHUB"
+  }
+
+  git_file_source {
+    uri       = var.git_repo_uri
+    revision  = "refs/heads/${var.git_repo_branch}"
+    repo_type = "GITHUB"
+    path      = "node-pack-extract/cloudbuild.yaml"
+  }
+
+  substitutions = {
+    _CUSTOM_NODE_NAME     = "custom-node"
+    _CUSTOM_NODE_URL      = "https://storage.googleapis.com/$(body.message.data.bucket)/$(body.message.data.name)"
+    _REGISTRY_BACKEND_URL = var.registry_backend_url
+  }
+}

infrastructure/modules/node-pack-extract-trigger/output.tf

@@ -0,0 +1,11 @@
+output "topic_id" {
+  value = google_pubsub_topic.topic.id
+}
+
+output "bucket_notification_id" {
+  value = google_storage_notification.notification.id
+}
+
+output "trigger_id" {
+  value = google_cloudbuild_trigger.trigger.id
+}

infrastructure/modules/node-pack-extract-trigger/variable.tf

@@ -0,0 +1,48 @@
+# REQUIRED VARIABLE
+variable "bucket_name" {
+  type        = string
+  description = "Existing public bucket that store the comfy node-packs."
+}
+
+variable "cloud_build_service_account" {
+  type        = string
+  description = "Existing service account used to run the cloud build and used to access registry backend, e.g. [email protected]. Note that this service account needs to have 'Service Account Token Creator' role."
+}
+
+variable "registry_backend_url" {
+  type        = string
+  description = "The base url where registry backend can be reached"
+}
+
+# OPTIONAL VARIABLE
+variable "region" {
+  type        = string
+  description = "Google Cloud region"
+  default     = "us-central1"
+}
+
+variable "topic_name" {
+  type        = string
+  description = "Google Cloudpub/sub topic to be created"
+  default     = "comfy-registry-event"
+}
+
+variable "trigger_name" {
+  type        = string
+  description = "Cloud build trigger name"
+  default     = "comfy-registry-nodepack"
+
+}
+
+variable "git_repo_uri" {
+  type        = string
+  description = "Connected git repo containing the cloud build pipeline. See https://cloud.google.com/build/docs/repositories"
+  default     = "https://github.com/Comfy-Org/registry-backend"
+}
+
+variable "git_repo_branch" {
+  type        = string
+  description = "Git repo branch."
+  default     = "main"
+}
+

infrastructure/modules/node-pack-extract-trigger/version.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.14.1"
+    }
+  }
+}

infrastructure/prod/README.md

@@ -0,0 +1,23 @@
+# Trigger for node-pack-extract production
+
+Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/)
+
+## Requirements
+
+- Google Cloud Account
+
+## Configuration
+
+This use the following configuration value:
+
+- bucket_name: "comfy-registry "
+- service account: "<[email protected]>"
+- topic_name: "comfy-registry-event"
+
+## Apply
+
+```bash
+terraform apply 
+    -var project_id=dreamboothy-dev
+    -var region=us-central1
+```

infrastructure/prod/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.14.1"
+    }
+  }
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+module "node_pack_extract_trigger" {
+  source = "../module/node-pack-extract-trigger"
+  providers = {
+    google = google
+  }
+  region                      = var.region
+  bucket_name                 = "comfy-registry"
+  cloud_build_service_account = "[email protected]"
+  topic_name                  = "comfy-registry-event"
+  registry_backend_url        = "https://api.comfy.org"
+}

infrastructure/prod/variable.tf

@@ -0,0 +1,10 @@
+variable "project_id" {
+  type        = string
+  description = "google cloud project id"
+}
+
+variable "region" {
+  type        = string
+  default     = "us-central1"
+  description = "google cloud region"
+}

infrastructure/staging/README.md

@@ -0,0 +1,23 @@
+# Trigger for node-pack-extract staging
+
+Terraform modules to setup trigger for cloud build that will run [node-pack-extract](../../../node-pack-extract/)
+
+## Requirements
+
+- Google Cloud Account
+
+## Configuration
+
+This use the following configuration value:
+
+- bucket_name: "comfy-registry "
+- service account: "<[email protected]>"
+- topic_name: "comfy-registry-event-staging"
+
+## Apply
+
+```bash
+terraform apply 
+    -var project_id=dreamboothy-dev
+    -var region=us-central1
+```

infrastructure/staging/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.14.1"
+    }
+  }
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+module "node_pack_extract_trigger" {
+  source = "../module/node-pack-extract-trigger"
+  providers = {
+    google = google
+  }
+  region                      = var.region
+  bucket_name                 = "comfy-registry"
+  cloud_build_service_account = "[email protected]"
+  topic_name                  = "comfy-registry-event-staging"
+  registry_backend_url        = "https://stagingapi.comfy.org"
+}