From 2d9c5d6fdf20082b4bcd67517f2296d735bf54f1 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 1 Jul 2025 02:26:33 -0700 Subject: [PATCH 01/32] Add custom fields to mappings --- modules/fleet-app-operator-permissions/main.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index e7bd846ba1..be16b5caea 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,12 +30,14 @@ locals { "VIEW" = "roles/gkehub.scopeViewerProjectLevel" "EDIT" = "roles/gkehub.scopeEditorProjectLevel" "ADMIN" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT + "CUSTOM" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT } resource_level_scope_role = { "VIEW" = "roles/gkehub.scopeViewer" "EDIT" = "roles/gkehub.scopeEditor" "ADMIN" = "roles/gkehub.scopeAdmin" + "CUSTOM" = "roles/gkehub.scopeViewer" # Same as VIEW } } From 4a9ccbda429669ad40676837c856af17830a41e6 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 1 Jul 2025 02:27:48 -0700 Subject: [PATCH 02/32] Update variables.tf and add custom_role --- modules/fleet-app-operator-permissions/variables.tf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/fleet-app-operator-permissions/variables.tf b/modules/fleet-app-operator-permissions/variables.tf index 400ed239af..5dd97a1b97 100644 --- a/modules/fleet-app-operator-permissions/variables.tf +++ b/modules/fleet-app-operator-permissions/variables.tf @@ -37,7 +37,7 @@ variable "groups" { } variable "role" { - description = "The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`)." + description = "The principal's role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`)." type = string validation { condition = contains(["VIEW", "EDIT", "ADMIN"], var.role) @@ -45,3 +45,8 @@ variable "role" { } } +variable "custom_role" { + description = "The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole." + type = string +} + From d8ceac2a920860ef71726cdd3656ef0ad96821ca Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 1 Jul 2025 13:59:54 -0700 Subject: [PATCH 03/32] Add defaults for role and custom role to get rid of required fields --- modules/fleet-app-operator-permissions/variables.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/fleet-app-operator-permissions/variables.tf b/modules/fleet-app-operator-permissions/variables.tf index 5dd97a1b97..c7247a8628 100644 --- a/modules/fleet-app-operator-permissions/variables.tf +++ b/modules/fleet-app-operator-permissions/variables.tf @@ -43,10 +43,12 @@ variable "role" { condition = contains(["VIEW", "EDIT", "ADMIN"], var.role) error_message = "Allowed values for role are VIEW, EDIT, or ADMIN." } + default = "" } variable "custom_role" { description = "The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole." type = string + default = "" } From 417c359797b3caf20a6ff88c0895fc9cdce817da Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 2 Jul 2025 04:59:06 -0700 Subject: [PATCH 04/32] If custom role is not empty, set it --- modules/fleet-app-operator-permissions/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index be16b5caea..693ae27d73 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -79,7 +79,11 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings scope_id = var.scope_id user = each.key role { + {% if var.custom_role != "" %} + custom_role = var.custom_role + {% else %} predefined_role = var.role + {% endif %} } } @@ -95,7 +99,11 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding scope_id = var.scope_id group = each.key role { + {% if var.custom_role != "" %} + custom_role = var.custom_role + {% else %} predefined_role = var.role + {% endif %} } } From 46ddc31442ca029cb670cd3cae8cf23e8a3f5bd0 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 2 Jul 2025 05:52:15 -0700 Subject: [PATCH 05/32] Update main.tf --- modules/fleet-app-operator-permissions/main.tf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index 693ae27d73..d9ac2c7c32 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -56,14 +56,22 @@ resource "google_project_iam_member" "log_view_permissions" { resource "google_project_iam_member" "project_level_scope_permissions" { project = var.fleet_project_id for_each = toset(concat(local.user_principals, local.group_principals)) - role = local.project_level_scope_role[var.role] + {% if var.custom_role != "" %} + role = local.resource_level_scope_role["CUSTOM"] + {% else %} + role = local.resource_level_scope_role[var.role] + {% endif %} member = each.value } resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" { project = var.fleet_project_id scope_id = var.scope_id - role = local.resource_level_scope_role[var.role] + {% if var.custom_role != "" %} + role = local.resource_level_scope_role["CUSTOM"] + {% else %} + role = local.resource_level_scope_role[var.role] + {% endif %} members = concat(local.user_principals, local.group_principals) } From b82892355791d1c2b5af06e7683e3c58f4e9026f Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 2 Jul 2025 07:05:13 -0700 Subject: [PATCH 06/32] Update main.tf to include custom role examples --- .../main.tf | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 4a1d16b2d4..8523aa1558 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -18,6 +18,7 @@ locals { app_operator_id = "app-operator-id" app_operator_team = "app-operator-team" app_operator_role = "VIEW" + app_operator_custom_role = "my-custom-role" } # Create a Service Account, which can be used as an app operator. @@ -33,6 +34,17 @@ resource "google_gke_hub_scope" "scope" { scope_id = local.app_operator_team } +# Allowlist custom roles for usage in Scope RBAC +resource "google_gke_hub_feature" "rbacrolebindingactuation" { + name = "rbacrolebindingactuation" + location = "global" + spec { + rbacrolebindingactuation { + allowed_custom_roles = [local.app_operator_custom_role] + } + } +} + # Grant permissions to the app operator to work with the Fleet Scope. module "permissions" { source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions" @@ -48,3 +60,19 @@ module "permissions" { ] } +# Grant custom role permissions to the app operator to work with the Fleet Scope. +module "permissions" { + source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions" + version = "~> 37.0" + + fleet_project_id = var.fleet_project_id + scope_id = google_gke_hub_scope.scope.scope_id + users = ["${local.app_operator_id}@${var.fleet_project_id}.iam.gserviceaccount.com"] + custom_role = local.app_operator_custom_role + + depends_on = [ + google_service_account.service_account, + google_gke_hub_feature.rbacrolebindingactuation, + ] +} + From f84ff9936a1bf2e487b542e98e141a7b5530ff1e Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 2 Jul 2025 07:14:08 -0700 Subject: [PATCH 07/32] Update main.tf --- examples/simple_fleet_app_operator_permissions/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 8523aa1558..2b485d9d26 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -61,7 +61,7 @@ module "permissions" { } # Grant custom role permissions to the app operator to work with the Fleet Scope. -module "permissions" { +module "custom_permissions" { source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions" version = "~> 37.0" From 7c588b674b26e3a7c4461ccddec1465e02dc5e6f Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Mon, 7 Jul 2025 12:09:19 +0000 Subject: [PATCH 08/32] Update main.tf and variables.tf --- examples/simple_fleet_app_operator_permissions/main.tf | 2 +- modules/fleet-app-operator-permissions/variables.tf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 2b485d9d26..97d9227893 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/fleet-app-operator-permissions/variables.tf b/modules/fleet-app-operator-permissions/variables.tf index c7247a8628..fd2906f903 100644 --- a/modules/fleet-app-operator-permissions/variables.tf +++ b/modules/fleet-app-operator-permissions/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ variable "groups" { } variable "role" { - description = "The principal's role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`)." + description = "The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set" type = string validation { condition = contains(["VIEW", "EDIT", "ADMIN"], var.role) @@ -47,7 +47,7 @@ variable "role" { } variable "custom_role" { - description = "The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole." + description = "The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set" type = string default = "" } From a6e44c16190407593de0638d912c51892d09e36b Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Mon, 7 Jul 2025 13:27:57 +0000 Subject: [PATCH 09/32] Update the testing logic --- .../main.tf | 9 +++-- ...ple_fleet_app_operator_permissions_test.go | 37 ++++++++++++++++++- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 97d9227893..90a71d0a82 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -18,7 +18,8 @@ locals { app_operator_id = "app-operator-id" app_operator_team = "app-operator-team" app_operator_role = "VIEW" - app_operator_custom_role = "my-custom-role" + custom_app_operator_id = "custom-app-operator-id" + custom_app_operator_role = "my-custom-role" } # Create a Service Account, which can be used as an app operator. @@ -40,7 +41,7 @@ resource "google_gke_hub_feature" "rbacrolebindingactuation" { location = "global" spec { rbacrolebindingactuation { - allowed_custom_roles = [local.app_operator_custom_role] + allowed_custom_roles = [local.custom_app_operator_role] } } } @@ -67,8 +68,8 @@ module "custom_permissions" { fleet_project_id = var.fleet_project_id scope_id = google_gke_hub_scope.scope.scope_id - users = ["${local.app_operator_id}@${var.fleet_project_id}.iam.gserviceaccount.com"] - custom_role = local.app_operator_custom_role + users = ["${local.custom_app_operator_id}@${var.fleet_project_id}.iam.gserviceaccount.com"] + custom_role = local.custom_app_operator_role depends_on = [ google_service_account.service_account, diff --git a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go index eb180e1965..1875fc2114 100644 --- a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go +++ b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -61,3 +61,38 @@ func TestSimpleFleetAppOperatorPermissions(t *testing.T) { appOppT.Test() } +func TestCustomFleetAppOperatorPermissions(t *testing.T) { + appOppT := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + appOppT.DefineVerify(func(assert *assert.Assertions) { + appOppT.DefaultVerify(assert) + + projectId := appOppT.GetStringOutput("project_id") + scopeId := "app-operator-team" + appOperatorEmail := fmt.Sprintf("custom-app-operator-id@%s.iam.gserviceaccount.com", projectId) + appOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail) + scopeLevelRole := "roles/gkehub.scopeViewer" + projectLevelRole := "roles/gkehub.scopeEditorProjectLevel" + logViewRole := "roles/logging.viewAccessor" + logViewContainerBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_container", projectId, scopeId, scopeId) + logViewPodBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_pod", projectId, scopeId, scopeId) + + scopeRrbList := gcloud.Runf(t, "container fleet scopes rbacrolebindings list --scope %s --project %s", scopeId, projectId).String() + assert.Equal(strings.Contains(scopeRrbList, appOperatorEmail), true, "app operator email should be in the list of Scope RBAC Role Bindings") + + scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s", scopeId, projectId).String() + assert.Equal(strings.Contains(scopeIam, appOperatorPrincipal), true, "app operator principal should be in the Scope IAM policy") + assert.Equal(strings.Contains(scopeIam, scopeLevelRole), true, "app operator Scope role should be in the Scope IAM policy") + + projectIam := gcloud.Runf(t, "projects get-iam-policy %s", projectId).String() + assert.Equal(strings.Contains(projectIam, appOperatorPrincipal), true, "app operator principal should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, projectLevelRole), true, "app operator Scope role should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewRole), true, "app operator log view role should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "app operator log view container bucket should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "app operator log view pod bucket should be in the project IAM policy") + }) + + appOppT.Test() +} + From d412d114c1934281f18ed6db340e2d00205942b1 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Mon, 7 Jul 2025 13:56:20 +0000 Subject: [PATCH 10/32] Update custom vs predefined logic in main.tf --- .../fleet-app-operator-permissions/main.tf | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index d9ac2c7c32..4d253e79da 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -56,22 +56,14 @@ resource "google_project_iam_member" "log_view_permissions" { resource "google_project_iam_member" "project_level_scope_permissions" { project = var.fleet_project_id for_each = toset(concat(local.user_principals, local.group_principals)) - {% if var.custom_role != "" %} - role = local.resource_level_scope_role["CUSTOM"] - {% else %} - role = local.resource_level_scope_role[var.role] - {% endif %} + role = (var.custom_role != "" ? local.project_level_scope_role["CUSTOM"] : local.project_level_scope_role[var.role]) member = each.value } resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" { project = var.fleet_project_id scope_id = var.scope_id - {% if var.custom_role != "" %} - role = local.resource_level_scope_role["CUSTOM"] - {% else %} - role = local.resource_level_scope_role[var.role] - {% endif %} + role = (var.custom_role != "" ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role]) members = concat(local.user_principals, local.group_principals) } @@ -87,11 +79,7 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings scope_id = var.scope_id user = each.key role { - {% if var.custom_role != "" %} - custom_role = var.custom_role - {% else %} - predefined_role = var.role - {% endif %} + var.custom_role != "" ? custom_role = var.custom_role : predefined_role = var.role } } @@ -107,11 +95,7 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding scope_id = var.scope_id group = each.key role { - {% if var.custom_role != "" %} - custom_role = var.custom_role - {% else %} - predefined_role = var.role - {% endif %} + var.custom_role != "" ? custom_role = var.custom_role : predefined_role = var.role } } From 02a08a8dc07b19bfb4757616ebf434157225fbf7 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 8 Jul 2025 10:29:41 +0000 Subject: [PATCH 11/32] Address comments on PR --- examples/simple_fleet_app_operator_permissions/main.tf | 2 +- modules/fleet-app-operator-permissions/main.tf | 8 +++++--- modules/fleet-app-operator-permissions/variables.tf | 6 +++--- .../simple_fleet_app_operator_permissions_test.go | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 90a71d0a82..050d9336bf 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2025 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index 4d253e79da..d1351c04df 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2025 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -79,7 +79,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings scope_id = var.scope_id user = each.key role { - var.custom_role != "" ? custom_role = var.custom_role : predefined_role = var.role + custom_role = (var.custom_role != "" ? var.custom_role : null) + predefined_role = (var.custom_role != "" ? null : var.role) } } @@ -95,7 +96,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding scope_id = var.scope_id group = each.key role { - var.custom_role != "" ? custom_role = var.custom_role : predefined_role = var.role + custom_role = (var.custom_role != "" ? var.custom_role : null) + predefined_role = (var.custom_role != "" ? null : var.role) } } diff --git a/modules/fleet-app-operator-permissions/variables.tf b/modules/fleet-app-operator-permissions/variables.tf index fd2906f903..2a7250d9ea 100644 --- a/modules/fleet-app-operator-permissions/variables.tf +++ b/modules/fleet-app-operator-permissions/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2025 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,12 +43,12 @@ variable "role" { condition = contains(["VIEW", "EDIT", "ADMIN"], var.role) error_message = "Allowed values for role are VIEW, EDIT, or ADMIN." } - default = "" + default = null } variable "custom_role" { description = "The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set" type = string - default = "" + default = null } diff --git a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go index 1875fc2114..f3602109c9 100644 --- a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go +++ b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 Google LLC +// Copyright 2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From a3f432240deee2a373807548c7b38aeea4feac22 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 8 Jul 2025 04:29:02 -0700 Subject: [PATCH 12/32] Format main.tf --- .../fleet-app-operator-permissions/main.tf | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index d1351c04df..01cf2e01b9 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -27,16 +27,16 @@ locals { ))] project_level_scope_role = { - "VIEW" = "roles/gkehub.scopeViewerProjectLevel" - "EDIT" = "roles/gkehub.scopeEditorProjectLevel" - "ADMIN" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT + "VIEW" = "roles/gkehub.scopeViewerProjectLevel" + "EDIT" = "roles/gkehub.scopeEditorProjectLevel" + "ADMIN" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT "CUSTOM" = "roles/gkehub.scopeEditorProjectLevel" # Same as EDIT } resource_level_scope_role = { - "VIEW" = "roles/gkehub.scopeViewer" - "EDIT" = "roles/gkehub.scopeEditor" - "ADMIN" = "roles/gkehub.scopeAdmin" + "VIEW" = "roles/gkehub.scopeViewer" + "EDIT" = "roles/gkehub.scopeEditor" + "ADMIN" = "roles/gkehub.scopeAdmin" "CUSTOM" = "roles/gkehub.scopeViewer" # Same as VIEW } } @@ -56,14 +56,14 @@ resource "google_project_iam_member" "log_view_permissions" { resource "google_project_iam_member" "project_level_scope_permissions" { project = var.fleet_project_id for_each = toset(concat(local.user_principals, local.group_principals)) - role = (var.custom_role != "" ? local.project_level_scope_role["CUSTOM"] : local.project_level_scope_role[var.role]) + role = (var.custom_role != "" ? local.project_level_scope_role["CUSTOM"] : local.project_level_scope_role[var.role]) member = each.value } resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" { project = var.fleet_project_id scope_id = var.scope_id - role = (var.custom_role != "" ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role]) + role = (var.custom_role != "" ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role]) members = concat(local.user_principals, local.group_principals) } @@ -79,7 +79,7 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings scope_id = var.scope_id user = each.key role { - custom_role = (var.custom_role != "" ? var.custom_role : null) + custom_role = (var.custom_role != "" ? var.custom_role : null) predefined_role = (var.custom_role != "" ? null : var.role) } } @@ -96,7 +96,7 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding scope_id = var.scope_id group = each.key role { - custom_role = (var.custom_role != "" ? var.custom_role : null) + custom_role = (var.custom_role != "" ? var.custom_role : null) predefined_role = (var.custom_role != "" ? null : var.role) } } From b2ac55d072a9d81d8f57000fa5eb459db1687e03 Mon Sep 17 00:00:00 2001 From: Tony Bayvas Date: Tue, 8 Jul 2025 11:41:34 +0000 Subject: [PATCH 13/32] Update generated files --- examples/simple_fleet_app_operator_permissions/main.tf | 10 +++++----- modules/fleet-app-operator-permissions/README.md | 3 ++- .../metadata.display.yaml | 3 +++ modules/fleet-app-operator-permissions/metadata.yaml | 6 ++++-- modules/fleet-app-operator-permissions/variables.tf | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 050d9336bf..57d2dbd5fd 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -15,10 +15,10 @@ */ locals { - app_operator_id = "app-operator-id" - app_operator_team = "app-operator-team" - app_operator_role = "VIEW" - custom_app_operator_id = "custom-app-operator-id" + app_operator_id = "app-operator-id" + app_operator_team = "app-operator-team" + app_operator_role = "VIEW" + custom_app_operator_id = "custom-app-operator-id" custom_app_operator_role = "my-custom-role" } @@ -37,7 +37,7 @@ resource "google_gke_hub_scope" "scope" { # Allowlist custom roles for usage in Scope RBAC resource "google_gke_hub_feature" "rbacrolebindingactuation" { - name = "rbacrolebindingactuation" + name = "rbacrolebindingactuation" location = "global" spec { rbacrolebindingactuation { diff --git a/modules/fleet-app-operator-permissions/README.md b/modules/fleet-app-operator-permissions/README.md index 960c2fe410..5a9967c405 100644 --- a/modules/fleet-app-operator-permissions/README.md +++ b/modules/fleet-app-operator-permissions/README.md @@ -28,9 +28,10 @@ To deploy this config, run: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| custom\_role | The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set | `string` | `null` | no | | fleet\_project\_id | The project to which the Fleet belongs. | `string` | n/a | yes | | groups | The list of app operator group principals, e.g., `people@google.com`, `principalSet://iam.googleapis.com/locations/global/workforcePools/my-pool/group/people`. | `list(string)` | `[]` | no | -| role | The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). | `string` | n/a | yes | +| role | The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set | `string` | `null` | no | | scope\_id | The scope for which IAM and RBAC role bindings are created. | `string` | n/a | yes | | users | The list of app operator user principals, e.g., `person@google.com`, `principal://iam.googleapis.com/locations/global/workforcePools/my-pool/subject/person`, `serviceAccount:my-service-account@my-project.iam.gserviceaccount.com`. | `list(string)` | `[]` | no | diff --git a/modules/fleet-app-operator-permissions/metadata.display.yaml b/modules/fleet-app-operator-permissions/metadata.display.yaml index 854efd839d..d7d5461da4 100644 --- a/modules/fleet-app-operator-permissions/metadata.display.yaml +++ b/modules/fleet-app-operator-permissions/metadata.display.yaml @@ -28,6 +28,9 @@ spec: ui: input: variables: + custom_role: + name: custom_role + title: Custom Role fleet_project_id: name: fleet_project_id title: Fleet Project Id diff --git a/modules/fleet-app-operator-permissions/metadata.yaml b/modules/fleet-app-operator-permissions/metadata.yaml index 7cd1b93b68..588ce53544 100644 --- a/modules/fleet-app-operator-permissions/metadata.yaml +++ b/modules/fleet-app-operator-permissions/metadata.yaml @@ -139,9 +139,11 @@ spec: varType: list(string) defaultValue: [] - name: role - description: The principals role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). + description: The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set + varType: string + - name: custom_role + description: The principal's role for the Fleet Scope which is a custom Kubernetes ClusterRole. Either a predefined role or a custom role should be set varType: string - required: true outputs: - name: fleet_project_id description: The project to which the Fleet belongs. diff --git a/modules/fleet-app-operator-permissions/variables.tf b/modules/fleet-app-operator-permissions/variables.tf index 2a7250d9ea..14f1f5f837 100644 --- a/modules/fleet-app-operator-permissions/variables.tf +++ b/modules/fleet-app-operator-permissions/variables.tf @@ -43,7 +43,7 @@ variable "role" { condition = contains(["VIEW", "EDIT", "ADMIN"], var.role) error_message = "Allowed values for role are VIEW, EDIT, or ADMIN." } - default = null + default = null } variable "custom_role" { From ceb1197a0fa1e2b95da61ae0f2113c6bb503e36d Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 8 Jul 2025 05:45:34 -0700 Subject: [PATCH 14/32] Update variables.tf to include null in predef role --- modules/fleet-app-operator-permissions/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/fleet-app-operator-permissions/variables.tf b/modules/fleet-app-operator-permissions/variables.tf index 14f1f5f837..540d71633c 100644 --- a/modules/fleet-app-operator-permissions/variables.tf +++ b/modules/fleet-app-operator-permissions/variables.tf @@ -40,8 +40,8 @@ variable "role" { description = "The principal's predefined role for the Fleet Scope (`VIEW`/`EDIT`/`ADMIN`). Either a predefined role or a custom role should be set" type = string validation { - condition = contains(["VIEW", "EDIT", "ADMIN"], var.role) - error_message = "Allowed values for role are VIEW, EDIT, or ADMIN." + condition = var.role == null || contains(["VIEW", "EDIT", "ADMIN"], var.role) + error_message = "Allowed values for role are VIEW, EDIT, ADMIN, or null." } default = null } From 823e845e2f964990d075228c00571babcd02e3fa Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 8 Jul 2025 10:38:41 -0700 Subject: [PATCH 15/32] Replace empty strings with null in main.tf --- modules/fleet-app-operator-permissions/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index 01cf2e01b9..d0c157b570 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -56,14 +56,14 @@ resource "google_project_iam_member" "log_view_permissions" { resource "google_project_iam_member" "project_level_scope_permissions" { project = var.fleet_project_id for_each = toset(concat(local.user_principals, local.group_principals)) - role = (var.custom_role != "" ? local.project_level_scope_role["CUSTOM"] : local.project_level_scope_role[var.role]) + role = (var.custom_role != null ? local.project_level_scope_role["CUSTOM"] : local.project_level_scope_role[var.role]) member = each.value } resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" { project = var.fleet_project_id scope_id = var.scope_id - role = (var.custom_role != "" ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role]) + role = (var.custom_role != null ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role]) members = concat(local.user_principals, local.group_principals) } @@ -79,8 +79,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings scope_id = var.scope_id user = each.key role { - custom_role = (var.custom_role != "" ? var.custom_role : null) - predefined_role = (var.custom_role != "" ? null : var.role) + custom_role = (var.custom_role != null ? var.custom_role : null) + predefined_role = (var.custom_role != null ? null : var.role) } } @@ -96,8 +96,8 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding scope_id = var.scope_id group = each.key role { - custom_role = (var.custom_role != "" ? var.custom_role : null) - predefined_role = (var.custom_role != "" ? null : var.role) + custom_role = (var.custom_role != null ? var.custom_role : null) + predefined_role = (var.custom_role != null ? null : var.role) } } From 81313b2858ceb5731205489c64542402cbde24b8 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 8 Jul 2025 11:29:15 -0700 Subject: [PATCH 16/32] Update main.tf to include feature project --- examples/simple_fleet_app_operator_permissions/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 57d2dbd5fd..a595d40fff 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -44,6 +44,7 @@ resource "google_gke_hub_feature" "rbacrolebindingactuation" { allowed_custom_roles = [local.custom_app_operator_role] } } + project = var.fleet_project_id } # Grant permissions to the app operator to work with the Fleet Scope. From b532929ecceedc62c74124baee181a8d390095b7 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 9 Jul 2025 02:15:21 -0700 Subject: [PATCH 17/32] Update main.tf --- examples/simple_fleet_app_operator_permissions/main.tf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index a595d40fff..40da5ab933 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -29,6 +29,13 @@ resource "google_service_account" "service_account" { display_name = "Test App Operator Service Account" } +# Create another Service Account, which can be used as a custom role app operator. +resource "google_service_account" "custom_service_account" { + project = var.fleet_project_id + account_id = local.custom_app_operator_id + display_name = "Test App Operator Custom Role Service Account" +} + # Create a Fleet Scope for the app operator's team. resource "google_gke_hub_scope" "scope" { project = var.fleet_project_id @@ -73,7 +80,7 @@ module "custom_permissions" { custom_role = local.custom_app_operator_role depends_on = [ - google_service_account.service_account, + google_service_account.custom_service_account, google_gke_hub_feature.rbacrolebindingactuation, ] } From c8a60ae7d5a7a4f7bf744fe274260da110cbe42c Mon Sep 17 00:00:00 2001 From: Tony Bayvas Date: Wed, 9 Jul 2025 09:49:58 +0000 Subject: [PATCH 18/32] Format main.tf --- examples/simple_fleet_app_operator_permissions/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_fleet_app_operator_permissions/main.tf b/examples/simple_fleet_app_operator_permissions/main.tf index 40da5ab933..e371d05c4c 100644 --- a/examples/simple_fleet_app_operator_permissions/main.tf +++ b/examples/simple_fleet_app_operator_permissions/main.tf @@ -51,7 +51,7 @@ resource "google_gke_hub_feature" "rbacrolebindingactuation" { allowed_custom_roles = [local.custom_app_operator_role] } } - project = var.fleet_project_id + project = var.fleet_project_id } # Grant permissions to the app operator to work with the Fleet Scope. From 8d81900b64ebab6726ea1072e8a2dd9a5b7ca3d8 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 9 Jul 2025 03:20:45 -0700 Subject: [PATCH 19/32] Update README.md to include custom role example --- modules/fleet-app-operator-permissions/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/fleet-app-operator-permissions/README.md b/modules/fleet-app-operator-permissions/README.md index 5a9967c405..c1e99a380e 100644 --- a/modules/fleet-app-operator-permissions/README.md +++ b/modules/fleet-app-operator-permissions/README.md @@ -14,6 +14,17 @@ module "fleet_app_operator_permissions" { groups = ["people@company.com"] role = "EDIT" } + +Example: +module "fleet_app_operator_permissions" { + source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-app-operator-permissions" + + fleet_project_id = "my-project-id" + scope_id = "frontend-team" + users = ["person1@company.com", "person2@company.com"] + groups = ["people@company.com"] + custom_role = "my-custom-role" +} ``` To deploy this config, run: From 948f25b88bad8787bf26e205b8cf28c7380ea84f Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 9 Jul 2025 03:33:36 -0700 Subject: [PATCH 20/32] Update int.cloudbuild.yaml --- build/int.cloudbuild.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 23c3fe20b8..1eca4ed650 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -466,6 +466,21 @@ steps: - verify simple-fleet-app-operator-permissions name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage teardown --verbose'] +- id: apply custom-fleet-app-operator-permissions + waitFor: + - init-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage apply --verbose'] +- id: verify custom-fleet-app-operator-permissions + waitFor: + - apply custom-fleet-app-operator-permissions + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage verify --verbose'] +- id: teardown custom-fleet-app-operator-permissions + waitFor: + - verify custom-fleet-app-operator-permissions + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage teardown --verbose'] - id: apply test-confidential-safer-cluster waitFor: - init-all From 1b6d915e4227824486b9e156995783af64485fa3 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 9 Jul 2025 05:27:22 -0700 Subject: [PATCH 21/32] Update int.cloudbuild.yaml --- build/int.cloudbuild.yaml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 1eca4ed650..47b7073505 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -461,19 +461,9 @@ steps: - apply simple-fleet-app-operator-permissions name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage verify --verbose'] -- id: teardown simple-fleet-app-operator-permissions - waitFor: - - verify simple-fleet-app-operator-permissions - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage teardown --verbose'] -- id: apply custom-fleet-app-operator-permissions - waitFor: - - init-all - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage apply --verbose'] - id: verify custom-fleet-app-operator-permissions waitFor: - - apply custom-fleet-app-operator-permissions + - apply simple-fleet-app-operator-permissions name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage verify --verbose'] - id: teardown custom-fleet-app-operator-permissions @@ -481,6 +471,11 @@ steps: - verify custom-fleet-app-operator-permissions name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage teardown --verbose'] +- id: teardown simple-fleet-app-operator-permissions + waitFor: + - verify simple-fleet-app-operator-permissions + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage teardown --verbose'] - id: apply test-confidential-safer-cluster waitFor: - init-all From 3e2fe07dc3e456adde245dc1459af8f294fabf01 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Mon, 14 Jul 2025 10:20:55 -0700 Subject: [PATCH 22/32] Update main.tf --- modules/fleet-app-operator-permissions/main.tf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index d0c157b570..aff9781e69 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -79,8 +79,9 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_user_role_bindings scope_id = var.scope_id user = each.key role { - custom_role = (var.custom_role != null ? var.custom_role : null) - predefined_role = (var.custom_role != null ? null : var.role) + # Setting both types of roles will return an error when creating the resource. + custom_role = var.custom_role + predefined_role = var.role } } @@ -96,8 +97,9 @@ resource "google_gke_hub_scope_rbac_role_binding" "scope_rbac_group_role_binding scope_id = var.scope_id group = each.key role { - custom_role = (var.custom_role != null ? var.custom_role : null) - predefined_role = (var.custom_role != null ? null : var.role) + # Setting both types of roles will return an error when creating the resource. + custom_role = var.custom_role + predefined_role = var.role } } From c9a79593444ba2f88792064e6d7ff70837b511ae Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Mon, 14 Jul 2025 13:47:05 -0700 Subject: [PATCH 23/32] Update int.cloudbuild.yaml --- build/int.cloudbuild.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 47b7073505..23c3fe20b8 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -461,16 +461,6 @@ steps: - apply simple-fleet-app-operator-permissions name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleFleetAppOperatorPermissions --stage verify --verbose'] -- id: verify custom-fleet-app-operator-permissions - waitFor: - - apply simple-fleet-app-operator-permissions - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage verify --verbose'] -- id: teardown custom-fleet-app-operator-permissions - waitFor: - - verify custom-fleet-app-operator-permissions - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'cft test run TestCustomFleetAppOperatorPermissions --stage teardown --verbose'] - id: teardown simple-fleet-app-operator-permissions waitFor: - verify simple-fleet-app-operator-permissions From c88315aed8c7e534460fa3c61eee668f3c2da391 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Mon, 14 Jul 2025 15:32:20 -0700 Subject: [PATCH 24/32] Update simple_fleet_app_operator_permissions_test.go --- ...ple_fleet_app_operator_permissions_test.go | 55 ++++++------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go index f3602109c9..a43df63948 100644 --- a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go +++ b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go @@ -39,58 +39,37 @@ func TestSimpleFleetAppOperatorPermissions(t *testing.T) { appOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail) scopeLevelRole := "roles/gkehub.scopeViewer" projectLevelRole := "roles/gkehub.scopeViewerProjectLevel" + customAppOperatorEmail := fmt.Sprintf("custom-app-operator-id@%s.iam.gserviceaccount.com", projectId) + customAppOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail) + customScopeLevelRole := "roles/gkehub.scopeViewer" + customProjectLevelRole := "roles/gkehub.scopeEditorProjectLevel" logViewRole := "roles/logging.viewAccessor" logViewContainerBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_container", projectId, scopeId, scopeId) logViewPodBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_pod", projectId, scopeId, scopeId) + filterFormat := "\"bindings.members:%s\"" + flattenOpt := "bindings[].members" scopeRrbList := gcloud.Runf(t, "container fleet scopes rbacrolebindings list --scope %s --project %s", scopeId, projectId).String() assert.Equal(strings.Contains(scopeRrbList, appOperatorEmail), true, "app operator email should be in the list of Scope RBAC Role Bindings") + assert.Equal(strings.Contains(scopeRrbList, customAppOperatorEmail), true, "custom app operator email should be in the list of Scope RBAC Role Bindings") - scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s", scopeId, projectId).String() - assert.Equal(strings.Contains(scopeIam, appOperatorPrincipal), true, "app operator principal should be in the Scope IAM policy") + scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s --filter %s", scopeId, projectId, fmt.Sprintf(filterFormat, appOperatorPrincipal)).String() assert.Equal(strings.Contains(scopeIam, scopeLevelRole), true, "app operator Scope role should be in the Scope IAM policy") - projectIam := gcloud.Runf(t, "projects get-iam-policy %s", projectId).String() - assert.Equal(strings.Contains(projectIam, appOperatorPrincipal), true, "app operator principal should be in the project IAM policy") - assert.Equal(strings.Contains(projectIam, projectLevelRole), true, "app operator Scope role should be in the project IAM policy") - assert.Equal(strings.Contains(projectIam, logViewRole), true, "app operator log view role should be in the project IAM policy") - assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "app operator log view container bucket should be in the project IAM policy") - assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "app operator log view pod bucket should be in the project IAM policy") - }) - - appOppT.Test() -} - -func TestCustomFleetAppOperatorPermissions(t *testing.T) { - appOppT := tft.NewTFBlueprintTest(t, - tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), - ) - appOppT.DefineVerify(func(assert *assert.Assertions) { - appOppT.DefaultVerify(assert) + customScopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s --filter %s", scopeId, projectId, fmt.Sprintf(filterFormat, customAppOperatorPrincipal)).String() + assert.Equal(strings.Contains(customScopeIam, customScopeLevelRole), true, "custom app operator Scope role should be in the Scope IAM policy") - projectId := appOppT.GetStringOutput("project_id") - scopeId := "app-operator-team" - appOperatorEmail := fmt.Sprintf("custom-app-operator-id@%s.iam.gserviceaccount.com", projectId) - appOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail) - scopeLevelRole := "roles/gkehub.scopeViewer" - projectLevelRole := "roles/gkehub.scopeEditorProjectLevel" - logViewRole := "roles/logging.viewAccessor" - logViewContainerBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_container", projectId, scopeId, scopeId) - logViewPodBucket := fmt.Sprintf("projects/%s/locations/global/buckets/fleet-o11y-scope-%s/views/fleet-o11y-scope-%s-k8s_pod", projectId, scopeId, scopeId) - - scopeRrbList := gcloud.Runf(t, "container fleet scopes rbacrolebindings list --scope %s --project %s", scopeId, projectId).String() - assert.Equal(strings.Contains(scopeRrbList, appOperatorEmail), true, "app operator email should be in the list of Scope RBAC Role Bindings") - - scopeIam := gcloud.Runf(t, "container fleet scopes get-iam-policy %s --project %s", scopeId, projectId).String() - assert.Equal(strings.Contains(scopeIam, appOperatorPrincipal), true, "app operator principal should be in the Scope IAM policy") - assert.Equal(strings.Contains(scopeIam, scopeLevelRole), true, "app operator Scope role should be in the Scope IAM policy") - - projectIam := gcloud.Runf(t, "projects get-iam-policy %s", projectId).String() - assert.Equal(strings.Contains(projectIam, appOperatorPrincipal), true, "app operator principal should be in the project IAM policy") + projectIam := gcloud.Runf(t, "projects get-iam-policy %s --filter %s --flatten %s", projectId, fmt.Sprintf(filterFormat, appOperatorPrincipal), flattenOpt).String() assert.Equal(strings.Contains(projectIam, projectLevelRole), true, "app operator Scope role should be in the project IAM policy") assert.Equal(strings.Contains(projectIam, logViewRole), true, "app operator log view role should be in the project IAM policy") assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "app operator log view container bucket should be in the project IAM policy") assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "app operator log view pod bucket should be in the project IAM policy") + + customProjectIam := gcloud.Runf(t, "projects get-iam-policy %s --filter %s --flatten %s", projectId, fmt.Sprintf(filterFormat, customAppOperatorPrincipal), flattenOpt).String() + assert.Equal(strings.Contains(projectIam, customProjectLevelRole), true, "custom app operator Scope role should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewRole), true, "custom app operator log view role should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "custom app operator log view container bucket should be in the project IAM policy") + assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "custom app operator log view pod bucket should be in the project IAM policy") }) appOppT.Test() From 45943c02f1dd32cd1fd4424a26438523fc3035b5 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 15 Jul 2025 08:31:45 -0700 Subject: [PATCH 25/32] Update versions.tf --- modules/fleet-app-operator-permissions/versions.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/fleet-app-operator-permissions/versions.tf b/modules/fleet-app-operator-permissions/versions.tf index 07d1259104..38e45138c7 100644 --- a/modules/fleet-app-operator-permissions/versions.tf +++ b/modules/fleet-app-operator-permissions/versions.tf @@ -20,11 +20,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.81.0" + version = ">= 6.39.0" } google-beta = { source = "hashicorp/google-beta" - version = ">= 4.81.0" + version = ">= 6.39.0" } random = { source = "hashicorp/random" From 12385f6c7ec70f9a7afb7b3537fbc0577e2c7e90 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 15 Jul 2025 08:32:18 -0700 Subject: [PATCH 26/32] Update versions.tf --- examples/simple_fleet_app_operator_permissions/versions.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple_fleet_app_operator_permissions/versions.tf b/examples/simple_fleet_app_operator_permissions/versions.tf index fe82a4e387..3d80f15374 100644 --- a/examples/simple_fleet_app_operator_permissions/versions.tf +++ b/examples/simple_fleet_app_operator_permissions/versions.tf @@ -20,11 +20,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.81.0" + version = ">= 6.39.0" } google-beta = { source = "hashicorp/google-beta" - version = ">= 4.81.0" + version = ">= 6.39.0" } } } From 767f30c3c3386a3b6407b19edb5331b891de66cc Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 15 Jul 2025 10:08:03 -0700 Subject: [PATCH 27/32] Update simple_fleet_app_operator_permissions_test.go --- .../simple_fleet_app_operator_permissions_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go index a43df63948..91686fbd67 100644 --- a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go +++ b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go @@ -66,10 +66,10 @@ func TestSimpleFleetAppOperatorPermissions(t *testing.T) { assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "app operator log view pod bucket should be in the project IAM policy") customProjectIam := gcloud.Runf(t, "projects get-iam-policy %s --filter %s --flatten %s", projectId, fmt.Sprintf(filterFormat, customAppOperatorPrincipal), flattenOpt).String() - assert.Equal(strings.Contains(projectIam, customProjectLevelRole), true, "custom app operator Scope role should be in the project IAM policy") - assert.Equal(strings.Contains(projectIam, logViewRole), true, "custom app operator log view role should be in the project IAM policy") - assert.Equal(strings.Contains(projectIam, logViewContainerBucket), true, "custom app operator log view container bucket should be in the project IAM policy") - assert.Equal(strings.Contains(projectIam, logViewPodBucket), true, "custom app operator log view pod bucket should be in the project IAM policy") + assert.Equal(strings.Contains(customProjectIam, customProjectLevelRole), true, "custom app operator Scope role should be in the project IAM policy") + assert.Equal(strings.Contains(customProjectIam, logViewRole), true, "custom app operator log view role should be in the project IAM policy") + assert.Equal(strings.Contains(customProjectIam, logViewContainerBucket), true, "custom app operator log view container bucket should be in the project IAM policy") + assert.Equal(strings.Contains(customProjectIam, logViewPodBucket), true, "custom app operator log view pod bucket should be in the project IAM policy") }) appOppT.Test() From 28f64f5a919123094af55257ed9d9a2405043c12 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Tue, 15 Jul 2025 14:50:49 -0700 Subject: [PATCH 28/32] Update simple_fleet_app_operator_permissions_test.go --- .../simple_fleet_app_operator_permissions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go index 91686fbd67..462e7f34b1 100644 --- a/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go +++ b/test/integration/simple_fleet_app_operator_permissions/simple_fleet_app_operator_permissions_test.go @@ -40,7 +40,7 @@ func TestSimpleFleetAppOperatorPermissions(t *testing.T) { scopeLevelRole := "roles/gkehub.scopeViewer" projectLevelRole := "roles/gkehub.scopeViewerProjectLevel" customAppOperatorEmail := fmt.Sprintf("custom-app-operator-id@%s.iam.gserviceaccount.com", projectId) - customAppOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", appOperatorEmail) + customAppOperatorPrincipal := fmt.Sprintf("serviceAccount:%s", customAppOperatorEmail) customScopeLevelRole := "roles/gkehub.scopeViewer" customProjectLevelRole := "roles/gkehub.scopeEditorProjectLevel" logViewRole := "roles/logging.viewAccessor" From ae915e2a7d4202f96429e905d89eb0d440cd09c1 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 16 Jul 2025 13:10:19 -0700 Subject: [PATCH 29/32] Update main.tf --- modules/fleet-app-operator-permissions/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index aff9781e69..6695b72fa8 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -60,7 +60,7 @@ resource "google_project_iam_member" "project_level_scope_permissions" { member = each.value } -resource "google_gke_hub_scope_iam_binding" "resource_level_scope_permissions" { +resource "google_gke_hub_scope_iam_member" "resource_level_scope_permissions" { project = var.fleet_project_id scope_id = var.scope_id role = (var.custom_role != null ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role]) From afcccea4fdf6e266fb8b8e3f480fc141301924bc Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 16 Jul 2025 14:39:07 -0700 Subject: [PATCH 30/32] Update README.md --- modules/fleet-app-operator-permissions/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/fleet-app-operator-permissions/README.md b/modules/fleet-app-operator-permissions/README.md index c1e99a380e..a742788735 100644 --- a/modules/fleet-app-operator-permissions/README.md +++ b/modules/fleet-app-operator-permissions/README.md @@ -1,4 +1,4 @@ -# Terrafrom Module for Fleet App Operator Permissions +# Terraform Module for Fleet App Operator Permissions This module bundles different permissions (IAM and RBAC Role Bindings) required for [Fleet team management](https://cloud.google.com/kubernetes-engine/fleet-management/docs/team-management). A platform admin can use this module to set up permissions for an app operator (user or group) in a team--including usage of Fleet Scopes, Connect Gateway, logging, and metrics--based on predefined roles (VIEW, EDIT, ADMIN). From 6a548a38b80706bbfa106dddd42d7267228188d6 Mon Sep 17 00:00:00 2001 From: tonybayvas Date: Wed, 16 Jul 2025 15:00:47 -0700 Subject: [PATCH 31/32] Update main.tf --- modules/fleet-app-operator-permissions/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/fleet-app-operator-permissions/main.tf b/modules/fleet-app-operator-permissions/main.tf index 6695b72fa8..d6432246a0 100644 --- a/modules/fleet-app-operator-permissions/main.tf +++ b/modules/fleet-app-operator-permissions/main.tf @@ -62,9 +62,10 @@ resource "google_project_iam_member" "project_level_scope_permissions" { resource "google_gke_hub_scope_iam_member" "resource_level_scope_permissions" { project = var.fleet_project_id + for_each = toset(concat(local.user_principals, local.group_principals)) scope_id = var.scope_id role = (var.custom_role != null ? local.resource_level_scope_role["CUSTOM"] : local.resource_level_scope_role[var.role]) - members = concat(local.user_principals, local.group_principals) + member = each.value } resource "random_id" "user_rand_suffix" { From 492a939e909129c665f63bc693690089d810cffd Mon Sep 17 00:00:00 2001 From: Tony Bayvas Date: Wed, 16 Jul 2025 22:17:32 +0000 Subject: [PATCH 32/32] Regenerate files --- modules/fleet-app-operator-permissions/metadata.display.yaml | 2 +- modules/fleet-app-operator-permissions/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/fleet-app-operator-permissions/metadata.display.yaml b/modules/fleet-app-operator-permissions/metadata.display.yaml index d7d5461da4..f49194cb3e 100644 --- a/modules/fleet-app-operator-permissions/metadata.display.yaml +++ b/modules/fleet-app-operator-permissions/metadata.display.yaml @@ -20,7 +20,7 @@ metadata: config.kubernetes.io/local-config: "true" spec: info: - title: Terrafrom Module for Fleet App Operator Permissions + title: Terraform Module for Fleet App Operator Permissions source: repo: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git sourceType: git diff --git a/modules/fleet-app-operator-permissions/metadata.yaml b/modules/fleet-app-operator-permissions/metadata.yaml index 588ce53544..a88d511f51 100644 --- a/modules/fleet-app-operator-permissions/metadata.yaml +++ b/modules/fleet-app-operator-permissions/metadata.yaml @@ -20,7 +20,7 @@ metadata: config.kubernetes.io/local-config: "true" spec: info: - title: Terrafrom Module for Fleet App Operator Permissions + title: Terraform Module for Fleet App Operator Permissions source: repo: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine.git sourceType: git