Verify cluster configuration in metatdata.yml

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 
 ### Changed
+* Enforce prod config in metadata.yml ([#1222](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1222))
 * Helm parity with Tailor ([#1219](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1219))
 
 

src/org/ods/orchestration/DeployStage.groovy

@@ -22,7 +22,6 @@ class DeployStage extends Stage {
 
     @SuppressWarnings(['ParameterName', 'AbcMetric', 'MethodSize', 'LineLength'])
     def run() {
-        def steps = ServiceRegistry.instance.get(IPipelineSteps)
         def levaDocScheduler = ServiceRegistry.instance.get(LeVADocumentScheduler)
         def os = ServiceRegistry.instance.get(OpenShiftService)
         def util = ServiceRegistry.instance.get(MROPipelineUtil)
@@ -85,43 +84,18 @@ class DeployStage extends Stage {
 
         runOnAgentPod(agentPodCondition) {
             if (project.isPromotionMode) {
-                def targetEnvironment = project.buildParams.targetEnvironment
-                def targetProject = project.targetProject
-                def installableRepos = this.project.repositories.findAll { repo ->
-                    // We only manage the installable repositories in OpenShift if they are included in the release
-                    if (repo.include
-                        && repo.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_INFRA) {
-                        MROPipelineUtil.PipelineConfig.INSTALLABLE_REPO_TYPES.contains(repo.type)
-                    }
-                }
-                logger.info("Deploying project '${project.key}' into environment '${targetEnvironment}'" +
-                    " installable repos? ${installableRepos.size()}")
+                def installableRepos = util.getInstallableRepos()
 
-                if (installableRepos?.size() > 0) {
-                    if (project.targetClusterIsExternal) {
-                        logger.info("Target cluster is external, logging into ${project.openShiftTargetApiUrl}")
-                        script.withCredentials([
-                            script.usernamePassword(
-                                credentialsId: project.environmentConfig.credentialsId,
-                                usernameVariable: 'EXTERNAL_OCP_API_SA',
-                                passwordVariable: 'EXTERNAL_OCP_API_TOKEN'
-                            )
-                        ]) {
-                            OpenShiftService.loginToExternalCluster(
-                                steps,
-                                project.openShiftTargetApiUrl,
-                                script.EXTERNAL_OCP_API_TOKEN
-                            )
-                        }
-                    }
+                logger.info("Verify project deployment '${project.key}' into environment " +
+                    "'${project.buildParams.targetEnvironment}' installable repos? ${installableRepos?.size()}")
 
-                    // Check if the target environment exists in OpenShift
-                    if (!os.envExists(targetProject)) {
-                        throw new RuntimeException(
-                            "Error: target environment '${targetProject}' does not exist " +
-                            "in ${project.openShiftTargetApiUrl}."
-                        )
-                    }
+                if (installableRepos?.size() > 0) {
+                    util.verifyEnvLoginAndExistence(script,
+                        os,
+                        project.targetProject,
+                        project.data.openshift?.sessionApiUrl,
+                        project.data.openshift?.targetApiUrl,
+                        project.environmentConfig?.credentialsId)
                 }
             }
 

src/org/ods/orchestration/InitStage.groovy

@@ -135,12 +135,8 @@ class InitStage extends Stage {
         String stageToStartAgent = findBestPlaceToStartAgent(repos, logger)
 
         // Compute target project. For now, the existance of DEV on the same cluster is verified.
-        def concreteEnv = Project.getConcreteEnvironment(
-            project.buildParams.targetEnvironment,
-            project.buildParams.version.toString(),
-            project.versionedDevEnvsEnabled
-        )
-        def targetProject = "${project.key}-${concreteEnv}"
+        def targetProject = Project.getTargetProjectForEnv(project, project.buildParams.targetEnvironment)
+
         def os = registry.get(OpenShiftService)
         if (project.buildParams.targetEnvironment == 'dev' && !os.envExists(targetProject)) {
             throw new RuntimeException(
@@ -150,13 +146,78 @@ class InitStage extends Stage {
         }
         project.setTargetProject(targetProject)
 
+        if (util.getInstallableRepos()?.size() > 0) {
+            validateEnvConfig(logger, registry, util)
+        } else {
+            logger.debug("No installable repos found, skipping env existence check.")
+        }
+
         logger.debug 'Compute groups of repository configs for convenient parallelization'
         repos = util.computeRepoGroups(repos)
         registry.get(LeVADocumentScheduler).run(phase, PipelinePhaseLifecycleStage.PRE_END)
 
         return [project: project, repos: repos, startAgent: stageToStartAgent]
     }
 
+    protected void validateEnvConfig(Logger logger, ServiceRegistry registry, MROPipelineUtil util) {
+        logger.debug("Validate environment metadata.yml config started")
+        def os = registry.get(OpenShiftService)
+        Map envs = project.getEnvironments()
+        def wronglyConfiguredEnvs = []
+        boolean reloginRequired = false
+        try {
+            for (MROPipelineUtil.PipelineEnv env : MROPipelineUtil.PipelineEnv.values()) {
+                def targetProjectForEnv = Project.getTargetProjectForEnv(project, env.value)
+                logger.debug("Check cluster config for env ${env.value} and project ${targetProjectForEnv}")
+                try {
+                    String openshiftClusterApiUrl = envs."$env.value"?.apiUrl
+                        ?: envs."$env.value"?.openshiftClusterApiUrl
+                    String openshiftClusterCredentialsId = envs."$env.value"?.credentialsId
+                        ?: envs."$env.value"?.openshiftClusterCredentialsId
+                    if (!openshiftClusterApiUrl && !openshiftClusterCredentialsId) {
+                        if (!os.envExists(targetProjectForEnv)) {
+                            wronglyConfiguredEnvs.add(env.value)
+                        }
+                    } else {
+                        reloginRequired = true
+                        util.verifyEnvLoginAndExistence(script,
+                            os,
+                            targetProjectForEnv,
+                            project.data.openshift.sessionApiUrl,
+                            openshiftClusterApiUrl,
+                            openshiftClusterCredentialsId
+                        )
+                    }
+                } catch (Exception e) {
+                    wronglyConfiguredEnvs.add(env.value)
+                }
+            }
+            if (wronglyConfiguredEnvs.size() > 0) {
+                String message = "The Release Manager configuration for environment(s) " +
+                    "${wronglyConfiguredEnvs.join(', ')} is incorrect in the metadata.yml. " +
+                    "Please verify the openshift cluster api url and credentials for " +
+                    "each environment mentioned."
+                if (project.isWorkInProgress) { // Warn build pipeline in this case
+                    project.addCommentInReleaseStatus(message)
+                    util.warnBuild(message)
+                } else {                        // Error
+                    util.failBuild(message)
+                    throw new RuntimeException(message) // This also add a comment in the release status issue
+                }
+            }
+        } finally {
+            if (reloginRequired) {
+                logger.debug("Try to relogin to current cluster")
+                try {
+                    os.reloginToCurrentClusterIfNeeded()
+                } catch (ex) {
+                    logger.error("Error logging back to current cluster ${ex.getMessage()}")
+                }
+                logger.debug("Success relogging in to current cluster.")
+            }
+        }
+    }
+
     private String findBestPlaceToStartAgent(List<Map> repos, ILogger logger) {
         String stageToStartAgent
         repos.each { repo ->

src/org/ods/orchestration/phases/DeployOdsComponent.groovy

@@ -271,7 +271,7 @@ class DeployOdsComponent {
             def imageInfo = imageParts.last().split('@')
             def imageName = imageInfo.first()
             def imageSha = imageInfo.last()
-            if (project.targetClusterIsExternal) {
+            if (project.targetClusterExternal) {
                 os.importImageShaFromSourceRegistry(
                     project.targetProject,
                     imageName,

src/org/ods/orchestration/usecase/JiraUseCase.groovy

@@ -427,6 +427,10 @@ class JiraUseCase {
     }
 
     void addCommentInReleaseStatus(String message) {
+        if (!this.jira) {
+            logger.debug("Jira not connected, cannot append comment: ${message}")
+            return
+        }
         def changeId = this.project.buildParams.changeId
         if (message) {
             def projectKey = this.project.jiraProjectKey

src/org/ods/orchestration/util/MROPipelineUtil.groovy

@@ -46,10 +46,21 @@ class MROPipelineUtil extends PipelineUtil {
         ]
     }
 
-    class PipelineEnvs {
-        static final String DEV = "dev"
-        static final String QA = "qa"
-        static final String PROD = "prod"
+    enum PipelineEnv {
+
+        DEV("dev"),
+        QA("qa"),
+        PROD("prod")
+
+        private String value
+
+        String getValue() {
+            return value
+        }
+
+        private PipelineEnv(String value) {
+            this.value = value
+        }
     }
 
     class PipelinePhases {

src/org/ods/orchestration/util/PipelineUtil.groovy

@@ -7,7 +7,7 @@ import com.cloudbees.groovy.cps.NonCPS
 
 import net.lingala.zip4j.ZipFile
 import net.lingala.zip4j.model.ZipParameters
-
+import org.ods.services.OpenShiftService
 import org.ods.util.IPipelineSteps
 import org.ods.util.ILogger
 import org.ods.services.GitService
@@ -184,4 +184,44 @@ class PipelineUtil {
         return this.steps.load(path)
     }
 
+    List getInstallableRepos() {
+        return this.project.repositories.findAll { repo ->
+            // We only manage the installable repositories in OpenShift if they are included in the release
+            if (repo.include
+                && repo.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_INFRA) {
+                MROPipelineUtil.PipelineConfig.INSTALLABLE_REPO_TYPES.contains(repo.type)
+            }
+        }
+    }
+
+    @SuppressWarnings('ParameterCount')
+    def verifyEnvLoginAndExistence(def script, OpenShiftService os, def targetProject, def sessionApiUrl,
+                                   def targetApiUrl, def credentialsId) {
+
+        if (Project.isTargetClusterExternal(sessionApiUrl, targetApiUrl)) {
+            logger.info("Target cluster is external, logging into ${targetApiUrl}")
+            script.withCredentials([
+                script.usernamePassword(
+                    credentialsId: credentialsId,
+                    usernameVariable: 'EXTERNAL_OCP_API_SA',
+                    passwordVariable: 'EXTERNAL_OCP_API_TOKEN'
+                )
+            ]) {
+                OpenShiftService.loginToExternalCluster(
+                    steps,
+                    targetApiUrl,
+                    script.EXTERNAL_OCP_API_TOKEN
+                )
+            }
+        }
+
+        // Check if the target environment exists in OpenShift
+        if (!os.envExists(targetProject)) {
+            throw new RuntimeException(
+                "Error: target environment '${targetProject}' does not exist " +
+                    "in ${targetApiUrl}."
+            )
+        }
+    }
+
 }

src/org/ods/orchestration/util/Project.groovy

@@ -299,7 +299,6 @@ class Project {
     protected Boolean isVersioningEnabled = false
     private String _gitReleaseBranch
 
-
     private TestResults aggregatedTestResults;
 
     protected Map data = [:]
@@ -720,7 +719,7 @@ class Project {
 
     void setOpenShiftData(String sessionApiUrl) {
         def envConfig = getEnvironmentConfig()
-        def targetApiUrl = envConfig?.apiUrl
+        def targetApiUrl = envConfig?.apiUrl ?: envConfig?.openshiftClusterApiUrl
         if (!targetApiUrl) {
             targetApiUrl = sessionApiUrl
         }
@@ -730,10 +729,13 @@ class Project {
     }
 
     @NonCPS
-    boolean getTargetClusterIsExternal() {
+    boolean isTargetClusterExternal() {
+        return isTargetClusterExternal(this.data.openshift.sessionApiUrl, this.data.openshift.targetApiUrl)
+    }
+
+    @NonCPS
+    static boolean isTargetClusterExternal(def sessionApiUrl, def targetApiUrl) {
         def isExternal = false
-        def sessionApiUrl = this.data.openshift.sessionApiUrl
-        def targetApiUrl = this.data.openshift.targetApiUrl
         def targetApiUrlMatcher = targetApiUrl =~ /:[0-9]+$/
         if (targetApiUrlMatcher.find()) {
             isExternal = sessionApiUrl != targetApiUrl
@@ -779,6 +781,15 @@ class Project {
         environment
     }
 
+    static String getTargetProjectForEnv(Project project, String env) {
+        def concreteEnv = Project.getConcreteEnvironment(
+            env,
+            project.buildParams.version.toString(),
+            project.versionedDevEnvsEnabled
+        )
+        return "${project.key}-${concreteEnv}"
+    }
+
     static List<String> getBuildEnvironment(IPipelineSteps steps, boolean debug = false, boolean versionedDevEnvsEnabled = false) {
         def params = loadBuildParams(steps)
 

src/org/ods/services/OpenShiftService.groovy

@@ -1058,7 +1058,7 @@ class OpenShiftService {
         }
         // if we have a resourceName only return the items matching that
         if (resourceName != null) {
-            def filteredPods= pods.findAll { it.podName.startsWith(resourceName) }
+            def filteredPods = pods.findAll { it.podName.startsWith(resourceName) }
             return filteredPods
         }
         return pods
@@ -1228,24 +1228,6 @@ class OpenShiftService {
         )
     }
 
-    private void reloginToCurrentClusterIfNeeded() {
-        def kubeUrl = steps.env.KUBERNETES_MASTER ?: 'https://kubernetes.default:443'
-        def success = steps.sh(
-            script: """
-               ${logger.shellScriptDebugFlag}
-                oc login ${kubeUrl} --insecure-skip-tls-verify=true \
-                --token=\$(cat /run/secrets/kubernetes.io/serviceaccount/token) &> /dev/null
-            """,
-            returnStatus: true,
-            label: 'Check if OCP session exists'
-        ) == 0
-        if (!success) {
-            throw new RuntimeException(
-                'Could not (re)login to cluster, this is a systemic failure'
-            )
-        }
-    }
-
     private void importImageFromProject(
         String project,
         String sourceProject,
@@ -1469,4 +1451,22 @@ class OpenShiftService {
             returnStdout: true
         ).toString().trim()
     }
+
+    void reloginToCurrentClusterIfNeeded() {
+        def kubeUrl = steps.env.KUBERNETES_MASTER ?: 'https://kubernetes.default:443'
+        def success = steps.sh(
+            script: """
+               ${logger.shellScriptDebugFlag}
+                oc login ${kubeUrl} --insecure-skip-tls-verify=true \
+                --token=\$(cat /run/secrets/kubernetes.io/serviceaccount/token) &> /dev/null
+            """,
+            returnStatus: true,
+            label: 'Check if OCP session exists'
+        ) == 0
+        if (!success) {
+            throw new RuntimeException(
+                'Could not (re)login to cluster, this is a systemic failure'
+            )
+        }
+    }
 }