4.x Partial rebuild fix

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Fixed
+- allowPartialRebuild is broken ([#977](https://github.com/opendevstack/ods-jenkins-shared-library/pull/977))
 - Memory leak fixes for component pipeline ([#857](https://github.com/opendevstack/ods-jenkins-shared-library/issues/857))
 
 ## [4.2.0] - 2023-02-21

src/org/ods/component/Pipeline.groovy

@@ -341,6 +341,12 @@ class Pipeline implements Serializable {
         if (!this.bitbucketNotificationEnabled) {
             return
         }
+        // in some nasty nullpointer cases - jenkins suddenly nullifies this bitbucket service?
+        // which in case a previous error, will push the nullp up the stack, and hide the "real" error
+        if (!bitbucketService) {
+            logger.warn "Cannot set Bitbucket build status to '${state}' as bitbucket service is null!"
+            return
+        }
         if (!context.buildUrl || !context.gitCommit) {
             logger.info "Cannot set Bitbucket build status to '${state}' because required data is missing!"
             return

src/org/ods/orchestration/InitStage.groovy

@@ -82,6 +82,26 @@ class InitStage extends Stage {
 
         def repos = project.repositories
         MROPipelineUtil util = registry.get(MROPipelineUtil)
+
+        // Compute target project. For now, the existance of DEV on the same cluster is verified.
+        // This must be done here - very early, otherwise the resurrection code will later fail
+        // as it depends on the target environment to check the artifacts there
+        def concreteEnv = Project.getConcreteEnvironment(
+            project.buildParams.targetEnvironment,
+            project.buildParams.version.toString(),
+            project.versionedDevEnvsEnabled
+        )
+        def targetProject = "${project.key}-${concreteEnv}"
+        def os = registry.get(OpenShiftService)
+        if (project.buildParams.targetEnvironment == 'dev' && !os.envExists(targetProject)) {
+            throw new RuntimeException(
+                "Target project ${targetProject} does not exist " +
+                    "(versionedDevEnvsEnabled=${project.versionedDevEnvsEnabled})."
+            )
+        }
+        logger.debug "Computed targetProject: ${targetProject}"
+        project.setTargetProject(targetProject)
+
         Closure checkoutClosure = buildCheckOutClousure(repos, logger, envState, util)
         Closure<String> loadClosure = buildLoadClousure(logger, registry, buildParams, git, steps)
         try {
@@ -104,22 +124,6 @@ class InitStage extends Stage {
 
         String stageToStartAgent = findBestPlaceToStartAgent(repos, logger)
 
-        // Compute target project. For now, the existance of DEV on the same cluster is verified.
-        def concreteEnv = Project.getConcreteEnvironment(
-            project.buildParams.targetEnvironment,
-            project.buildParams.version.toString(),
-            project.versionedDevEnvsEnabled
-        )
-        def targetProject = "${project.key}-${concreteEnv}"
-        def os = registry.get(OpenShiftService)
-        if (project.buildParams.targetEnvironment == 'dev' && !os.envExists(targetProject)) {
-            throw new RuntimeException(
-                "Target project ${targetProject} does not exist " +
-                    "(versionedDevEnvsEnabled=${project.versionedDevEnvsEnabled})."
-            )
-        }
-        project.setTargetProject(targetProject)
-
         logger.debug 'Compute groups of repository configs for convenient parallelization'
         repos = util.computeRepoGroups(repos)
         registry.get(LeVADocumentScheduler).run(phase, PipelinePhaseLifecycleStage.PRE_END)

src/org/ods/orchestration/phases/DeployOdsComponent.groovy

@@ -263,4 +263,5 @@ class DeployOdsComponent {
             }
         }
     }
+
 }

src/org/ods/orchestration/phases/FinalizeNonOdsComponent.groovy

@@ -39,4 +39,5 @@ class FinalizeNonOdsComponent {
     private String commitBuildReference() {
         "${steps.currentBuild.description}\r${steps.env.BUILD_URL}"
     }
+
 }

src/org/ods/orchestration/phases/FinalizeOdsComponent.groovy

@@ -31,7 +31,12 @@ class FinalizeOdsComponent {
     @TypeChecked(TypeCheckingMode.SKIP)
     public void run(Map repo, String baseDir) {
         this.os = ServiceRegistry.instance.get(OpenShiftService)
-
+        logger.debug("Checking ${repo} for resurrection ... ")
+        // check if resurrected
+        if (repo.data?.openshift?.resurrectedBuild) {
+            logger.info("Skipping ${repo.id} as it was resurrected")
+            return
+        }
         steps.dir(baseDir) {
             Map deploymentMean = verifyDeploymentsBuiltByODS(repo).values().get(0) as Map
             logger.debug("DeploymentMean: ${deploymentMean}")

src/org/ods/orchestration/usecase/DocGenUseCase.groovy

@@ -187,6 +187,12 @@ abstract class DocGenUseCase {
         def basename = getDocumentBasename(
             documentType, oldBuildVersion, buildVersionKey[1], repo)
         def path = "${this.steps.env.WORKSPACE}/reports/${repo.id}"
+        // hack - classically it does not work, code returns false, dir not there
+        // def mkdirresult = new File(path).mkdir() -> returns false!
+        // so we have to go the jenkins version
+        this.steps.dir (path) {
+            this.steps.writeFile(file: 'dummy', text: 'dummy')
+        }
 
         def fileExtensions = getFiletypeForDocumentType(documentType)
         String storageType = fileExtensions.storage ?: 'zip'

src/org/ods/orchestration/util/MROPipelineUtil.groovy

@@ -269,16 +269,14 @@ class MROPipelineUtil extends PipelineUtil {
         ]
         def repoPath = "${this.steps.env.WORKSPACE}/${REPOS_BASE_DIR}/${repo.id}"
         loadPipelineConfig(repoPath, repo)
-        if (this.project.isAssembleMode) {
-            if (this.project.forceGlobalRebuild) {
-                this.logger.debug('Project forces global rebuild ...')
-            } else {
-                this.steps.dir(repoPath) {
-                    this.logger.startClocked("${repo.id}-resurrect-data")
-                    this.logger.debug('Checking if repo can be resurrected from previous build ...')
-                    amendRepoForResurrectionIfEligible(repo)
-                    this.logger.debugClocked("${repo.id}-resurrect-data")
-                }
+        if (this.project.forceGlobalRebuild) {
+            this.logger.debug("Global force-rebuild configured for '${repo.id}'")
+        } else if (this.project.isAssembleMode || this.project.isWorkInProgress) {
+            this.steps.dir(repoPath) {
+                this.logger.startClocked("${repo.id}-resurrect-data")
+                this.logger.debug("Checking if repo '${repo.id}' can be resurrected from previous build(s) ...")
+                amendRepoForResurrectionIfEligible(repo)
+                this.logger.debugClocked("${repo.id}-resurrect-data")
             }
         }
     }
@@ -394,15 +392,17 @@ class MROPipelineUtil extends PipelineUtil {
                         } else if (this.project.isPromotionMode && name == PipelinePhases.BUILD) {
                             executeODSComponent(repo, baseDir)
                         } else if (this.project.isAssembleMode && name == PipelinePhases.FINALIZE) {
-                            new FinalizeNonOdsComponent(project, steps, git, logger).run(repo, baseDir)
+                            this.logger.warn("Repo '${repo.id}' is of type ODS Infrastructure Component - SKIPPING FINALIZE!")
+                            // new FinalizeNonOdsComponent(project, steps, git, logger).run(repo, baseDir)
                         } else {
                             this.logger.debug("Repo '${repo.id}' is of type ODS Infrastructure as Code Component/Configuration Management. Nothing to do in phase '${name}' for target environment'${targetEnvToken}'.")
                         }
                     } else if (repo.type?.toLowerCase() == PipelineConfig.REPO_TYPE_ODS_LIB) {
                         if (this.project.isAssembleMode && name == PipelinePhases.BUILD) {
                             executeODSComponent(repo, baseDir)
                         } else if (this.project.isAssembleMode && name == PipelinePhases.FINALIZE) {
-                            new FinalizeNonOdsComponent(project, steps, git, logger).run(repo, baseDir)
+                            this.logger.warn("Repo '${repo.id}' is of type ODS library Component - SKIPPING FINALIZE!")
+                            // new FinalizeNonOdsComponent(project, steps, git, logger).run(repo, baseDir)
                         } else {
                             this.logger.debug("Repo '${repo.id}' is of type ODS library. Nothing to do in phase '${name}' for target environment'${targetEnvToken}'.")
                         }
@@ -431,7 +431,8 @@ class MROPipelineUtil extends PipelineUtil {
                         } else if (name == PipelinePhases.TEST) {
                             executeODSComponent(repo, baseDir)
                         } else if (this.project.isAssembleMode && name == PipelinePhases.FINALIZE) {
-                            new FinalizeNonOdsComponent(project, steps, git, logger).run(repo, baseDir)
+                            this.logger.warn("Repo '${repo.id}' is of type ODS Test Component - SKIPPING FINALIZE!")
+                            // new FinalizeNonOdsComponent(project, steps, git, logger).run(repo, baseDir)
                         } else {
                             this.logger.debug("Repo '${repo.id}' is of type ODS Test Component. Nothing to do in phase '${name}' for target environment '${targetEnvToken}'.")
                         }
@@ -513,7 +514,8 @@ class MROPipelineUtil extends PipelineUtil {
     private void amendRepoForResurrectionIfEligible(Map repo) {
         def os = ServiceRegistry.instance.get(OpenShiftService)
 
-        if (repo.type?.toLowerCase() != PipelineConfig.REPO_TYPE_ODS_CODE) {
+        if (!repo.type?.toLowerCase() == PipelineConfig.REPO_TYPE_ODS_CODE &&
+            !repo.type?.toLowerCase() == PipelineConfig.REPO_TYPE_ODS_SERVICE) {
             logger.info(
                 "Resurrection of previous build for '${repo.id}' not possible as " +
                 "type '${repo.type}' is not eligible."
@@ -524,7 +526,7 @@ class MROPipelineUtil extends PipelineUtil {
         if (repo.forceRebuild) {
             logger.info(
                 "Resurrection of previous build for '${repo.id}' not possible as " +
-                "repo '${repo.id}' is forced to rebuild."
+                "repo '${repo.id}' is forced to rebuild (configuration)."
             )
             return
         }

src/org/ods/services/GitService.groovy

@@ -207,6 +207,10 @@ class GitService {
     }
 
     def commit(List files, String msg, boolean allowEmpty = true) {
+        if (allowEmpty && !files) {
+            logger.debug("Nothing to commit - skipping")
+            return
+        }
         def allowEmptyFlag = allowEmpty ? '--allow-empty' : ''
         def filesToAddCommand = "git add ${files.join(' ')}"
         if (files.empty) {
@@ -215,6 +219,16 @@ class GitService {
         script.sh(
             script: """
                 ${filesToAddCommand}
+            """,
+            label: 'Stash'
+        )
+        // check if the commit is dirty, so real changes happened - if not - skip
+        if (!isCommitDirty()) {
+            logger.debug("Commit stash is Not dirty - skipping")
+            return
+        }
+        script.sh(
+            script: """
                 git commit -m "${msg}" ${allowEmptyFlag}
             """,
             label: 'Commit'
@@ -337,6 +351,16 @@ class GitService {
         script.sh("git checkout ${branchToMerge}")
     }
 
+    def isCommitDirty () {
+        def content = script.sh(
+            script: "git status --porcelain -uno",
+            label: "get current commit stash",
+            returnStdout: true
+        ).trim()
+        logger.debug("Check current commit stash: |${content}|")
+        return (content.length() > 0)
+    }
+
     def checkoutNewLocalBranch(String name) {
         // Local state might have a branch from previous, failed pipeline runs.
         // If so, we'd rather start from a clean state.

src/org/ods/services/NexusService.groovy

@@ -151,6 +151,7 @@ class NexusService {
         if (artifactExists) {
             artifactExists.delete()
         }
+        // NOTE: unnirest expects the directory tree to be present! MUST be created from outside
         def response = restCall.asFile("${extractionPath}/${name}")
 
         response.ifFailure {
@@ -161,14 +162,18 @@ class NexusService {
             if (response.getStatus() == 404) {
                 message = "Error: unable to get artifact. Nexus could not be found at: '${urlToDownload}'."
             }
-            // very weird, we get a 200 as failure with a good artifact, wtf.
-            if (response.getStatus() != 200) {
+
+            // if we get a 200 as failure with a good artifact, wtf. - parsing error?!
+            if (response.getStatus() == 200) {
+                throw new RuntimeException("Could not parse response from: ${urlToDownload}",
+                    response.getParsingError().get())
+            } else {
                 throw new RuntimeException(message)
             }
         }
 
         return [
-            uri: this.baseURL.resolve("/repository/${nexusRepository}/${nexusDirectory}/${name}"),
+            uri: new URI(urlToDownload),
             content: response.getBody(),
         ]
     }

test/groovy/org/ods/services/NexusServiceSpec.groovy

@@ -210,9 +210,12 @@ class NexusServiceSpec extends SpecHelper {
 
         def server = createServer(WireMock.&get, request, response)
         def service = createService(server.port(), request.username, request.password)
+        // this has changed appearently, nio file permissions?
+        def extractionPath =  System.getProperty("java.io.tmpdir")
+        new File(extractionPath).mkdir()
 
         when:
-        Map result = service.retrieveArtifact(request.data.repository, request.data.directory, request.data.name, "abc")
+        Map result = service.retrieveArtifact(request.data.repository, request.data.directory, request.data.name, extractionPath)
 
         then:
         result.uri == new URI("http://localhost:${server.port()}${request.path}")