Deploy ToolHive Operator into OpenShift (#1063)

cmd/thv-operator/controllers/mcpserver_controller.go

@@ -552,6 +552,8 @@ func (r *MCPServerReconciler) deploymentForMCPServer(m *mcpv1alpha1.MCPServer) *
 		}
 	}
 
+	env = ensureRequiredEnvVars(env)
+
 	dep := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        m.Name,
@@ -621,6 +623,36 @@ func (r *MCPServerReconciler) deploymentForMCPServer(m *mcpv1alpha1.MCPServer) *
 	return dep
 }
 
+func ensureRequiredEnvVars(env []corev1.EnvVar) []corev1.EnvVar {
+	// Check for the existence of the XDG_CONFIG_HOME and HOME environment variables
+	// and set them to /tmp if they don't exist
+	xdgConfigHomeFound := false
+	homeFound := false
+	for _, envVar := range env {
+		if envVar.Name == "XDG_CONFIG_HOME" {
+			xdgConfigHomeFound = true
+		}
+		if envVar.Name == "HOME" {
+			homeFound = true
+		}
+	}
+	if !xdgConfigHomeFound {
+		logger.Debugf("XDG_CONFIG_HOME not found, setting to /tmp")
+		env = append(env, corev1.EnvVar{
+			Name:  "XDG_CONFIG_HOME",
+			Value: "/tmp",
+		})
+	}
+	if !homeFound {
+		logger.Debugf("HOME not found, setting to /tmp")
+		env = append(env, corev1.EnvVar{
+			Name:  "HOME",
+			Value: "/tmp",
+		})
+	}
+	return env
+}
+
 func createServiceName(mcpServerName string) string {
 	return fmt.Sprintf("mcp-%s-proxy", mcpServerName)
 }

deploy/charts/operator/values.yaml

@@ -42,13 +42,14 @@ operator:
   # -- Pod security context settings
   podSecurityContext:
     runAsNonRoot: true
+    seccompProfile:
+      type: RuntimeDefault
 
   # -- Container security context settings for the operator
   containerSecurityContext:
     allowPrivilegeEscalation: false
     readOnlyRootFilesystem: true
     runAsNonRoot: true
-    runAsUser: 1000
     capabilities:
       drop:
       - ALL
@@ -85,10 +86,10 @@ operator:
   resources:
     limits:
       cpu: 500m
-      memory: 128Mi
+      memory: 384Mi
     requests:
       cpu: 10m
-      memory: 64Mi
+      memory: 192Mi
 
   # -- RBAC configuration for the operator
   rbac:

pkg/container/kubernetes/client.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/client-go/tools/watch"
+	clientconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
 
 	"github.com/stacklok/toolhive/pkg/container/runtime"
 	"github.com/stacklok/toolhive/pkg/logger"
@@ -246,17 +247,30 @@ func (c *Client) DeployWorkload(ctx context.Context,
 	}
 
 	// Ensure the pod template has required configuration (labels, etc.)
-	podTemplateSpec = ensurePodTemplateConfig(podTemplateSpec, containerLabels)
+	isOpenShift := false
+	// Get a config to talk to the apiserver
+	cfg, err := clientconfig.GetConfig()
+	if err != nil {
+		return 0, fmt.Errorf("error getting config for APIServer: %w", err)
+	}
+
+	isOpenShift, err = DetectOpenShiftWith(cfg)
+	if err != nil {
+		return 0, fmt.Errorf("can't determine api server type: %w", err)
+	}
+
+	podTemplateSpec = ensurePodTemplateConfig(podTemplateSpec, containerLabels, isOpenShift)
 
 	// Configure the MCP container
-	err := configureMCPContainer(
+	err = configureMCPContainer(
 		podTemplateSpec,
 		image,
 		command,
 		attachStdio,
 		envVarList,
 		transportType,
 		options,
+		isOpenShift,
 	)
 	if err != nil {
 		return 0, err
@@ -891,6 +905,7 @@ func createPodTemplateFromPatch(patchJSON string) (*corev1apply.PodTemplateSpecA
 func ensurePodTemplateConfig(
 	podTemplateSpec *corev1apply.PodTemplateSpecApplyConfiguration,
 	containerLabels map[string]string,
+	isOpenShift bool,
 ) *corev1apply.PodTemplateSpecApplyConfiguration {
 	podTemplateSpec = ensureObjectMetaApplyConfigurationExists(podTemplateSpec)
 	// Ensure the pod template has labels
@@ -940,6 +955,31 @@ func ensurePodTemplateConfig(
 			podTemplateSpec.Spec.SecurityContext = podTemplateSpec.Spec.SecurityContext.WithRunAsGroup(int64(1000))
 		}
 	}
+
+	if isOpenShift {
+		if podTemplateSpec.Spec.SecurityContext.RunAsUser != nil {
+			podTemplateSpec.Spec.SecurityContext.RunAsUser = nil
+		}
+
+		if podTemplateSpec.Spec.SecurityContext.RunAsGroup != nil {
+			podTemplateSpec.Spec.SecurityContext.RunAsGroup = nil
+		}
+
+		if podTemplateSpec.Spec.SecurityContext.FSGroup != nil {
+			podTemplateSpec.Spec.SecurityContext.FSGroup = nil
+		}
+
+		if podTemplateSpec.Spec.SecurityContext.SeccompProfile == nil {
+			podTemplateSpec.Spec.SecurityContext.SeccompProfile =
+				corev1apply.SeccompProfile().WithType(
+					corev1.SeccompProfileTypeRuntimeDefault)
+		} else {
+			podTemplateSpec.Spec.SecurityContext.SeccompProfile =
+				podTemplateSpec.Spec.SecurityContext.SeccompProfile.WithType(
+					corev1.SeccompProfileTypeRuntimeDefault)
+		}
+	}
+
 	return podTemplateSpec
 }
 
@@ -985,7 +1025,18 @@ func configureContainer(
 	command []string,
 	attachStdio bool,
 	envVars []*corev1apply.EnvVarApplyConfiguration,
+	isOpenShift bool,
 ) {
+	logger.Infof("Configuring container %s with image %s", *container.Name, image)
+	logger.Infof("Command: ")
+	for _, arg := range command {
+		logger.Infof("Arg: %s", arg)
+	}
+	logger.Infof("AttachStdio: %v", attachStdio)
+	for _, envVar := range envVars {
+		logger.Infof("EnvVar: %s=%s", *envVar.Name, *envVar.Value)
+	}
+
 	container.WithImage(image).
 		WithArgs(command...).
 		WithStdin(attachStdio).
@@ -1029,6 +1080,34 @@ func configureContainer(
 			container.SecurityContext = container.SecurityContext.WithAllowPrivilegeEscalation(false)
 		}
 	}
+
+	if isOpenShift {
+		logger.Infof("Setting OpenShift security context requirements to container %s", *container.Name)
+
+		if container.SecurityContext.RunAsUser != nil {
+			container.SecurityContext.RunAsUser = nil
+		}
+
+		if container.SecurityContext.RunAsGroup != nil {
+			container.SecurityContext.RunAsGroup = nil
+		}
+
+		if container.SecurityContext.SeccompProfile == nil {
+			container.SecurityContext.SeccompProfile =
+				corev1apply.SeccompProfile().WithType(
+					corev1.SeccompProfileTypeRuntimeDefault)
+		} else {
+			container.SecurityContext.SeccompProfile =
+				container.SecurityContext.SeccompProfile.WithType(
+					corev1.SeccompProfileTypeRuntimeDefault)
+		}
+
+		if container.SecurityContext.Capabilities == nil {
+			container.SecurityContext.Capabilities = &corev1apply.CapabilitiesApplyConfiguration{
+				Drop: []corev1.Capability{"ALL"},
+			}
+		}
+	}
 }
 
 // configureMCPContainer configures the MCP container in the pod template
@@ -1040,6 +1119,7 @@ func configureMCPContainer(
 	envVarList []*corev1apply.EnvVarApplyConfiguration,
 	transportType string,
 	options *runtime.DeployWorkloadOptions,
+	isOpenShift bool,
 ) error {
 	// Get the "mcp" container if it exists
 	mcpContainer := getMCPContainer(podTemplateSpec)
@@ -1049,7 +1129,7 @@ func configureMCPContainer(
 		mcpContainer = corev1apply.Container().WithName("mcp")
 
 		// Configure the container
-		configureContainer(mcpContainer, image, command, attachStdio, envVarList)
+		configureContainer(mcpContainer, image, command, attachStdio, envVarList, isOpenShift)
 
 		// Configure ports if needed
 		if options != nil && transportType == string(transtypes.TransportTypeSSE) {
@@ -1064,7 +1144,7 @@ func configureMCPContainer(
 		podTemplateSpec.Spec.WithContainers(mcpContainer)
 	} else {
 		// Configure the existing container
-		configureContainer(mcpContainer, image, command, attachStdio, envVarList)
+		configureContainer(mcpContainer, image, command, attachStdio, envVarList, isOpenShift)
 
 		// Configure ports if needed
 		if options != nil && transportType == string(transtypes.TransportTypeSSE) {

pkg/container/kubernetes/client_test.go

@@ -346,7 +346,7 @@ func TestEnsurePodTemplateConfig(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 			// Call the function
-			result := ensurePodTemplateConfig(tc.podTemplateSpec, tc.containerLabels)
+			result := ensurePodTemplateConfig(tc.podTemplateSpec, tc.containerLabels, false)
 
 			// Check the result
 			assert.NotNil(t, result)
@@ -521,6 +521,7 @@ func TestConfigureMCPContainer(t *testing.T) {
 				tc.envVars,
 				tc.transportType,
 				tc.options,
+				false,
 			)
 
 			// Check that there was no error

pkg/container/kubernetes/common.go

@@ -0,0 +1,73 @@
+package kubernetes
+
+import (
+	"os"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
+
+	"github.com/stacklok/toolhive/pkg/logger"
+)
+
+// extra kinds
+const (
+	// defaultRetries is the number of times a resource discovery is retried
+	defaultRetries = 10
+
+	//defaultRetryInterval is the interval to wait before retring a resource discovery
+	defaultRetryInterval = 3 * time.Second
+)
+
+var isOpenShift *bool = nil
+
+// DetectOpenShiftWith determines whether the current cluster is an OpenShift
+// cluster by attempting to discover the Route resource (route.openshift.io/v1).
+// It returns true when the resource is present. The provided REST config is used
+// by the discovery client to query the API server.
+func DetectOpenShiftWith(config *rest.Config) (bool, error) {
+	// If first time, check if we are running on OpenShift
+	if isOpenShift == nil {
+		value, ok := os.LookupEnv("OPERATOR_OPENSHIFT")
+		if ok {
+			//ctrl.Log.V(1).Info("Set by env-var 'OPERATOR_OPENSHIFT': " + value)
+			logger.Infof("OpenShift set by env var 'OPERATOR_OPENSHIFT': " + value)
+			return strings.ToLower(value) == "true", nil
+		}
+
+		discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
+		if err != nil {
+			return false, err
+		}
+
+		isOpenShiftResourcePresent := false
+		for i := 0; i < defaultRetries; i++ {
+			isOpenShiftResourcePresent, err = discovery.IsResourceEnabled(discoveryClient,
+				schema.GroupVersionResource{
+					Group:    "route.openshift.io",
+					Version:  "v1",
+					Resource: "routes",
+				})
+
+			if err == nil {
+				break
+			}
+
+			time.Sleep(defaultRetryInterval)
+		}
+
+		if err != nil {
+			return false, err
+		}
+
+		isOpenShift = &isOpenShiftResourcePresent
+		if isOpenShiftResourcePresent {
+			logger.Infof("OpenShift detected by route resource check.")
+		}
+	}
+
+	// Otherwise, return the cached value
+	return *isOpenShift, nil
+}