feat(TPG>=6.39)!: Fleet app operator permissions custom roles (#2377)

Co-authored-by: Andrew Peabody <andrewpeabody@google.com>

Author: tonybayvas

examples/simple_fleet_app_operator_permissions/main.tf

@@ -15,9 +15,11 @@
  */
 
 locals {
-  app_operator_id   = "app-operator-id"
-  app_operator_team = "app-operator-team"
-  app_operator_role = "VIEW"
+  app_operator_id          = "app-operator-id"
+  app_operator_team        = "app-operator-team"
+  app_operator_role        = "VIEW"
+  custom_app_operator_id   = "custom-app-operator-id"
+  custom_app_operator_role = "my-custom-role"
 }
 
 # Create a Service Account, which can be used as an app operator.
@@ -27,12 +29,31 @@ resource "google_service_account" "service_account" {
   display_name = "Test App Operator Service Account"
 }
 
+# Create another Service Account, which can be used as a custom role app operator.
+resource "google_service_account" "custom_service_account" {
+  project      = var.fleet_project_id
+  account_id   = local.custom_app_operator_id
+  display_name = "Test App Operator Custom Role Service Account"
+}
+
 # Create a Fleet Scope for the app operator's team.
 resource "google_gke_hub_scope" "scope" {
   project  = var.fleet_project_id
   scope_id = local.app_operator_team
 }
 
+# Allowlist custom roles for usage in Scope RBAC
+resource "google_gke_hub_feature" "rbacrolebindingactuation" {
+  name     = "rbacrolebindingactuation"
+  location = "global"
+  spec {
+    rbacrolebindingactuation {
+      allowed_custom_roles = [local.custom_app_operator_role]
+    }
+  }
+  project = var.fleet_project_id
+}
+
 # Grant permissions to the app operator to work with the Fleet Scope.
 module "permissions" {
   source  = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions"
@@ -48,3 +69,19 @@ module "permissions" {
   ]
 }
 
+# Grant custom role permissions to the app operator to work with the Fleet Scope.
+module "custom_permissions" {
+  source  = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions"
+  version = "~> 37.0"
+
+  fleet_project_id = var.fleet_project_id
+  scope_id         = google_gke_hub_scope.scope.scope_id
+  users            = ["${local.custom_app_operator_id}@${var.fleet_project_id}.iam.gserviceaccount.com"]
+  custom_role      = local.custom_app_operator_role
+
+  depends_on = [
+    google_service_account.custom_service_account,
+    google_gke_hub_feature.rbacrolebindingactuation,
+  ]
+}
+

examples/simple_fleet_app_operator_permissions/versions.tf

@@ -20,11 +20,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.81.0"
+      version = ">= 6.39.0"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.81.0"
+      version = ">= 6.39.0"
     }
   }
 }

modules/fleet-app-operator-permissions/README.md

@@ -1,4 +1,4 @@
-# Terrafrom Module for Fleet App Operator Permissions
+# Terraform Module for Fleet App Operator Permissions
 
 This module bundles different permissions (IAM and RBAC Role Bindings) required for [Fleet team management](https://cloud.google.com/kubernetes-engine/fleet-management/docs/team-management). A platform admin can use this module to set up permissions for an app operator (user or group) in a team--including usage of Fleet Scopes, Connect Gateway, logging, and metrics--based on predefined roles (VIEW, EDIT, ADMIN).
 
@@ -14,6 +14,17 @@ module "fleet_app_operator_permissions" {
   groups           = ["people@company.com"]
   role             = "EDIT"
 }
+
+Example:
+module "fleet_app_operator_permissions" {
+  source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions"
+
+  fleet_project_id = "my-project-id"
+  scope_id         = "frontend-team"
+  users            = ["person1@company.com", "person2@company.com"]
+  groups           = ["people@company.com"]
+  custom_role      = "my-custom-role"
+}
 ```
 
 To deploy this config, run:
@@ -28,9 +39,10 @@ To deploy this config, run:
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| custom\_role | The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set | `string` | `null` | no |
 | fleet\_project\_id | The project to which the Fleet belongs. | `string` | n/a | yes |
 | groups | The list of app operator group principals, e.g., `people@google.com`, `principalSet://iam.googleapis.com/locations/global/workforcePools/my-pool/group/people`. | `list(string)` | `[]` | no |
-| role | The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). | `string` | n/a | yes |
+| role | The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set | `string` | `null` | no |
 | scope\_id | The scope for which IAM and RBAC role bindings are created. | `string` | n/a | yes |
 | users | The list of app operator user principals, e.g., `person@google.com`, `principal://iam.googleapis.com/locations/global/workforcePools/my-pool/subject/person`, `serviceAccount:my-service-account@my-project.iam.gserviceaccount.com`. | `list(string)` | `[]` | no |
 

modules/fleet-app-operator-permissions/main.tf

@@ -27,15 +27,17 @@ locals {
   ))]
 
   project_level_scope_role = {
-    "VIEW"  = "roles/gkehub.scopeViewerProjectLevel"
-    "EDIT"  = "roles/gkehub.scopeEditorProjectLevel"
-    "ADMIN" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT
+    "VIEW"   = "roles/gkehub.scopeViewerProjectLevel"
+    "EDIT"   = "roles/gkehub.scopeEditorProjectLevel"
+    "ADMIN"  = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT
+    "CUSTOM" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT
   }
 
   resource_level_scope_role = {
-    "VIEW"  = "roles/gkehub.scopeViewer"
-    "EDIT"  = "roles/gkehub.scopeEditor"
-    "ADMIN" = "roles/gkehub.scopeAdmin"
+    "VIEW"   = "roles/gkehub.scopeViewer"
+    "EDIT"   = "roles/gkehub.scopeEditor"
+    "ADMIN"  = "roles/gkehub.scopeAdmin"
+    "CUSTOM" = "roles/gkehub.scopeViewer" # Same as VIEW
   }
 }
 
@@ -54,15 +56,16 @@ resource "google_project_iam_member" "log_view_permissions" {
 resource "google_project_iam_member" "project_level_scope_permissions" {
   project  = var.fleet_project_id
   for_each = toset(concat(local.user_principals, local.group_principals))
-  role     = local.project_level_scope_role[var.role]
+  role     = (var.custom_role != null ? local.project_level_scope_role["CUSTOM"] : local.project_level_scope_role[var.role])
   member   = each.value
 }
 
-resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" {
+resource "google_gke_hub_scope_iam_member" "resource_level_scope_permissions" {
   project  = var.fleet_project_id
+  for_each = toset(concat(local.user_principals, local.group_principals))
   scope_id = var.scope_id
-  role     = local.resource_level_scope_role[var.role]
-  members  = concat(local.user_principals, local.group_principals)
+  role     = (var.custom_role != null ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role])
+  member   = each.value
 }
 
 resource "random_id" "user_rand_suffix" {
@@ -77,6 +80,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings
   scope_id                   = var.scope_id
   user                       = each.key
   role {
+    # Setting both types of roles will return an error when creating the resource.
+    custom_role     = var.custom_role
     predefined_role = var.role
   }
 }
@@ -93,6 +98,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding
   scope_id                   = var.scope_id
   group                      = each.key
   role {
+    # Setting both types of roles will return an error when creating the resource.
+    custom_role     = var.custom_role
     predefined_role = var.role
   }
 }

modules/fleet-app-operator-permissions/metadata.display.yaml

@@ -20,14 +20,17 @@ metadata:
     config.kubernetes.io/local-config: "true"
 spec:
   info:
-    title: Terrafrom Module for Fleet App Operator Permissions
+    title: Terraform Module for Fleet App Operator Permissions
     source:
       repo: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git
       sourceType: git
       dir: /modules/fleet-app-operator-permissions
   ui:
     input:
       variables:
+        custom_role:
+          name: custom_role
+          title: Custom Role
         fleet_project_id:
           name: fleet_project_id
           title: Fleet Project Id

modules/fleet-app-operator-permissions/metadata.yaml

@@ -20,7 +20,7 @@ metadata:
     config.kubernetes.io/local-config: "true"
 spec:
   info:
-    title: Terrafrom Module for Fleet App Operator Permissions
+    title: Terraform Module for Fleet App Operator Permissions
     source:
       repo: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git
       sourceType: git
@@ -139,9 +139,11 @@ spec:
         varType: list(string)
         defaultValue: []
       - name: role
-        description: The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`).
+        description: The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set
+        varType: string
+      - name: custom_role
+        description: The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set
         varType: string
-        required: true
     outputs:
       - name: fleet_project_id
         description: The project to which the Fleet belongs.

modules/fleet-app-operator-permissions/variables.tf

@@ -37,11 +37,18 @@ variable "groups" {
 }
 
 variable "role" {
-  description = "The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`)."
+  description = "The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set"
   type        = string
   validation {
-    condition     = contains(["VIEW", "EDIT", "ADMIN"], var.role)
-    error_message = "Allowed values for role are VIEW, EDIT, or ADMIN."
+    condition     = var.role == null || contains(["VIEW", "EDIT", "ADMIN"], var.role)
+    error_message = "Allowed values for role are VIEW, EDIT, ADMIN, or null."
   }
+  default = null
+}
+
+variable "custom_role" {
+  description = "The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set"
+  type        = string
+  default     = null
 }
 

modules/fleet-app-operator-permissions/versions.tf

@@ -20,11 +20,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.81.0"
+      version = ">= 6.39.0"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.81.0"
+      version = ">= 6.39.0"
     }
     random = {
       source  = "hashicorp/random"

test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go

@@ -39,23 +39,37 @@ func TestSimpleFleetAppOperatorPermissions(t *testing.T) {
 		appOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail)
 		scopeLevelRole := "roles/gkehub.scopeViewer"
 		projectLevelRole := "roles/gkehub.scopeViewerProjectLevel"
+		customAppOperatorEmail := fmt.Sprintf("custom-app-operator-id@%s.iam.gserviceaccount.com", projectId)
+		customAppOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", customAppOperatorEmail)
+		customScopeLevelRole := "roles/gkehub.scopeViewer"
+		customProjectLevelRole := "roles/gkehub.scopeEditorProjectLevel"
 		logViewRole := "roles/logging.viewAccessor"
 		logViewContainerBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_container", projectId, scopeId, scopeId)
 		logViewPodBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_pod", projectId, scopeId, scopeId)
+		filterFormat := "\"bindings.members:%s\""
+		flattenOpt := "bindings[].members"
 
 		scopeRrbList := gcloud.Runf(t, "container fleet scopes rbacrolebindings list --scope %s --project %s", scopeId, projectId).String()
 		assert.Equal(strings.Contains(scopeRrbList, appOperatorEmail), true, "app operator email should be in the list of Scope RBAC Role Bindings")
+		assert.Equal(strings.Contains(scopeRrbList, customAppOperatorEmail), true, "custom app operator email should be in the list of Scope RBAC Role Bindings")
 
-		scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s", scopeId, projectId).String()
-		assert.Equal(strings.Contains(scopeIam, appOperatorPrincipal), true, "app operator principal should be in the Scope IAM policy")
+		scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s --filter %s", scopeId, projectId, fmt.Sprintf(filterFormat, appOperatorPrincipal)).String()
 		assert.Equal(strings.Contains(scopeIam, scopeLevelRole), true, "app operator Scope role should be in the Scope IAM policy")
 
-		projectIam := gcloud.Runf(t, "projects get-iam-policy %s", projectId).String()
-		assert.Equal(strings.Contains(projectIam, appOperatorPrincipal), true, "app operator principal should be in the project IAM policy")
+		customScopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s --filter %s", scopeId, projectId, fmt.Sprintf(filterFormat, customAppOperatorPrincipal)).String()
+		assert.Equal(strings.Contains(customScopeIam, customScopeLevelRole), true, "custom app operator Scope role should be in the Scope IAM policy")
+
+		projectIam := gcloud.Runf(t, "projects get-iam-policy %s --filter %s --flatten %s", projectId, fmt.Sprintf(filterFormat, appOperatorPrincipal), flattenOpt).String()
 		assert.Equal(strings.Contains(projectIam, projectLevelRole), true, "app operator Scope role should be in the project IAM policy")
 		assert.Equal(strings.Contains(projectIam, logViewRole), true, "app operator log view role should be in the project IAM policy")
 		assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "app operator log view container bucket should be in the project IAM policy")
 		assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "app operator log view pod bucket should be in the project IAM policy")
+
+		customProjectIam := gcloud.Runf(t, "projects get-iam-policy %s --filter %s --flatten %s", projectId, fmt.Sprintf(filterFormat, customAppOperatorPrincipal), flattenOpt).String()
+		assert.Equal(strings.Contains(customProjectIam, customProjectLevelRole), true, "custom app operator Scope role should be in the project IAM policy")
+		assert.Equal(strings.Contains(customProjectIam, logViewRole), true, "custom app operator log view role should be in the project IAM policy")
+		assert.Equal(strings.Contains(customProjectIam, logViewContainerBucket), true, "custom app operator log view container bucket should be in the project IAM policy")
+		assert.Equal(strings.Contains(customProjectIam, logViewPodBucket), true, "custom app operator log view pod bucket should be in the project IAM policy")
 	})
 
 	appOppT.Test()