Feature/test stage (#21)

* Adding a test stage to the pipeline, cleaning up CodeBuild environment properties

* Renaming stages for more clarity

* Documentation updates

Co-authored-by: Srijit Mitra <srijit@amazon.com>

Author: srijitm

README.md

@@ -4,6 +4,12 @@ This project provides a serverless CI/CD platform leveraging native AWS services
 
 This CI/CD platform is in use at a variety of AWS clients where the development teams are leveraging CloudFormation, Serverless Framework, AWS CDK and Terraform.
 
+## Getting started
+
+- [Pre-Requisites](docs/prereq.md)
+- [Administrator Guide](docs/admin.md)
+- [Developer Guide](docs/developer.md)
+
 ## Goals
 
 - Pipelines as code.
@@ -39,9 +45,3 @@ The number of stages and their function is fully customizable e.g. adding a stag
 - AWS CloudWatch
 - AWS Systems Manager: Parameter Store
 - AWS CloudFormation
-
-## Getting started
-
-- [Pre-Requisites](docs/prereq.md)
-- [Administrator Guide](docs/admin.md)
-- [Developer Guide](docs/developer.md)

docs/developer.md

@@ -7,18 +7,20 @@ As the developer you are responsible for maintaining a scripts folder with three
 ```text
 scripts/build.sh
 
+scripts/test.sh
+
 scripts/deploy.sh
 ```
 
-The build and deploy scripts must contain all the commands required to build your application, including the installation of any dependencies. The pipeline will execute the build script once. The deploy script will be executed once per target environment i.e dev, test and prod.
+The build,test and deploy scripts must contain all the commands required to build your application, including the installation of any dependencies. The pipeline will execute the build and test scripts once per run. The deploy script will be executed once per target environment i.e dev, test and prod.
 
 This enforces the *"build once, deploy many"* paradigm and also ensures that your deployment script is environment agnostic.
 
-Finally, you as the developer are now in complete control of how your applications gets deployed to your AWS accounts.
+Finally, you as the developer are now in complete control of how your applications get deployed to your AWS accounts.
 
 ## Detailed Instructions
 
-Create a directory called scripts in the root of the application source code repository called ***scripts*** with 2 files:
+Create a directory called scripts in the root of the application source code repository called ***scripts*** with 3 files:
 
 1. build.sh
 
@@ -58,6 +60,20 @@ Create a directory called scripts in the root of the application source code rep
       echo "Nothing to build!"
     ```
 
+1. test.sh
+
+    Sample build script for a project which doesn't require a build
+
+    ```bash
+      #! /bin/bash
+
+      set -e
+      set -u
+      set -o pipefail
+
+      echo "Nothing to test. Tests executed as part of build."
+    ```
+
 1. deploy.sh - A bash script which deploys the application. The shell script can trigger Terraform/Cloudformation/AWS CDK etc. Since the shell script will run in the AWS CodeBuild docker environment, as long as the binaries are available CodeBuild will be able to execute it.
 
     Sample deployment script which installs AWS CDK and triggers the deployment

lib/pipelines/codecommit-pipeline.ts

@@ -34,7 +34,7 @@ import { Role } from '@aws-cdk/aws-iam'
 import { IFunction } from '@aws-cdk/aws-lambda'
 import { Repository } from '@aws-cdk/aws-codecommit'
 import { BuildProject } from '../projects/build-project'
-import { BuildEnvironmentVariableType } from '@aws-cdk/aws-codebuild'
+import { TestProject } from '../projects/test-project'
 import { Rule, Schedule } from '@aws-cdk/aws-events'
 import ssm = require('@aws-cdk/aws-ssm');
 import sns = require('@aws-cdk/aws-sns');
@@ -88,22 +88,19 @@ export class CodeCommitPipeline extends Pipeline {
       stringValue: notificationTopic.topicArn
     })
 
+    // Source Control Stage (CodeCommit)
     const sourceOutputArtifact = new Artifact('SourceArtifact')
-    const buildOutputArtifact = new Artifact('BuildArtifact')
-    
     const codeCommitRepo = Repository.fromRepositoryName(
       scope,
       `${repoName}${repoBranch}CodeCommitRepo`,
       repoName
     )
 
-    // CodeCommit Repo Target
     codeCommitRepo.onCommit('OnCommit', {
       target: new targets.CodePipeline(this),
       branches: [`${repoBranch}`]
     })
 
-    // SourceAction
     const sourceAction = new CodeCommitSourceAction({
       repository: codeCommitRepo,
       branch: repoBranch,
@@ -116,29 +113,18 @@ export class CodeCommitPipeline extends Pipeline {
       actions: [sourceAction]
     })
 
-    // Build
-    const sourceCodeBuildRole = new CodeBuildRole(this, 'sourceCodeBuildRole')
-
-    const buildProject = new BuildProject(this, 'sourceProject', {
-      projectName: `${prefix}-${repoName}-${repoBranch}-build`,
-      role: sourceCodeBuildRole,
-      environmentVariables: {
-        ARTIFACTS_BUCKET_NAME: {
-          type: BuildEnvironmentVariableType.PLAINTEXT,
-          value: artifactsBucket.bucketName
-        },
-        ARTIFACTS_BUCKET_ARN: {
-          type: BuildEnvironmentVariableType.PLAINTEXT,
-          value: artifactsBucket.bucketArn
-        },
-        REPO_NAME: {
-          type: BuildEnvironmentVariableType.PLAINTEXT,
-          value: repoName
-        }
-      }
+    // Building Stage
+    const buildOutputArtifact = new Artifact('BuildArtifact')
+    const buildRole = new CodeBuildRole(this, 'buildRole')
+
+    const buildProject = new BuildProject(this, `${prefix}-${repoName}-${repoBranch}-build`, {
+        repoName: repoName,
+        role: buildRole,
+        bucketArn: artifactsBucket.bucketArn,
+        bucketName: artifactsBucket.bucketName
     })
 
-    buildProject.onStateChange('deploymentStatus', {
+    buildProject.onStateChange('buildStatus', {
       target: new targets.LambdaFunction(emailHandler)
     })
 
@@ -164,7 +150,34 @@ export class CodeCommitPipeline extends Pipeline {
       actions: [buildAction, semverAction]
     })
 
-    // Deploy
+    // Testing Stage
+    const testOutputArtifact = new Artifact('TestArtifact')
+    const testRole = new CodeBuildRole(this, 'testRole')
+
+    const testProject = new TestProject(this, `${prefix}-${repoName}-${repoBranch}-test`, {
+        repoName: repoName,
+        role: testRole,
+        bucketArn: artifactsBucket.bucketArn,
+        bucketName: artifactsBucket.bucketName
+    })
+
+    testProject.onStateChange('testStatus', {
+      target: new targets.LambdaFunction(emailHandler)
+    })
+
+    const testAction = new CodeBuildAction ({
+      actionName: 'Test',
+      outputs: [testOutputArtifact],
+      input: buildOutputArtifact,
+      project: testProject
+    })
+
+    this.addStage({
+      stageName: 'Test',
+      actions: [testAction]
+    })
+
+    // Deploying Stage (One stage per target environment)
     ;[StageName.dev, StageName.test, StageName.prod].forEach((stageName: StageName) => {
 
       if (config.accountIds[stageName]) {
@@ -196,13 +209,13 @@ export class CodeCommitPipeline extends Pipeline {
         const moduleDeployOutputArtifact = new Artifact()
         const moduleDeployAction = new CodeBuildAction({
           actionName: 'Deploy',
-          input: buildOutputArtifact,
+          input: testOutputArtifact,
           outputs: [moduleDeployOutputArtifact],
           project: deployProject,
           role: modulePipelineRole
         })
         this.addStage({
-          stageName: `${stageName}-deploy`,
+          stageName: `Deploy-to-${stageName}-environment`,
           actions: [moduleDeployAction]
         })
       }

lib/projects/build-project-environment.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: MIT-0
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the "Software"), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+import { BuildEnvironmentVariableType, ComputeType, 
+  LinuxBuildImage  } from '@aws-cdk/aws-codebuild'
+
+export const defaultEnvironment = {
+  buildImage: LinuxBuildImage.STANDARD_4_0,
+  computeType: ComputeType.SMALL,
+  privileged: true
+}
+
+export function projectEnvironmentVars(props: {
+  repoName: string
+  bucketName: string
+  bucketArn: string
+}) {
+  const siteVars: any = {}
+
+  return {
+    ARTIFACTS_BUCKET_NAME: {
+      type: BuildEnvironmentVariableType.PLAINTEXT,
+      value: props.bucketName
+    },
+    ARTIFACTS_BUCKET_ARN: {
+      type: BuildEnvironmentVariableType.PLAINTEXT,
+      value: props.bucketArn
+    },
+    REPO_NAME: {
+      type: BuildEnvironmentVariableType.PLAINTEXT,
+      value: props.repoName
+    },
+    ...siteVars
+  }
+}

lib/projects/build-project.ts

@@ -19,15 +19,25 @@
 
 import { Construct } from '@aws-cdk/core'
 import { PipelineProject, ProjectProps, BuildSpec} from '@aws-cdk/aws-codebuild'
-import { defaultEnvironment } from './project-environment'
+import { defaultEnvironment } from './build-project-environment'
+import { projectEnvironmentVars } from './build-project-environment'
+import { Role } from '@aws-cdk/aws-iam'
 
 export interface BuildProjectProps extends ProjectProps {
+  repoName: string
+  bucketName: string
+  bucketArn: string
+  role: Role
 }
 
 export class BuildProject extends PipelineProject {
   constructor(scope: Construct, id: string, props: BuildProjectProps) {
-    const { ...rest } = props
+    const { repoName, bucketName, bucketArn } = props
     super(scope, id, {
+      projectName: id,
+      role: props.role,
+      environment: defaultEnvironment,
+      environmentVariables: projectEnvironmentVars({ repoName, bucketName, bucketArn }),
       buildSpec: BuildSpec.fromObject({
         version: '0.2',
         phases: {
@@ -40,18 +50,12 @@ export class BuildProject extends PipelineProject {
             commands: [
               'bash ${CODEBUILD_SRC_DIR}/scripts/build.sh'
             ]
-          },
-          post_build: {
-            commands: [
-            ]
           }
         },
         artifacts: {
           files: '**/*'
         }
-      }),
-      environment: defaultEnvironment,
-      ...rest
+      })
     })
   }
 }

lib/projects/project-environment.ts renamed to lib/projects/deploy-project-environment.ts

@@ -19,8 +19,8 @@
 
 import { PipelineProject, BuildSpec } from '@aws-cdk/aws-codebuild'
 import { Construct } from '@aws-cdk/core'
-import { defaultEnvironment } from './project-environment'
-import { projectEnvironmentVars } from './project-environment'
+import { defaultEnvironment } from './deploy-project-environment'
+import { projectEnvironmentVars } from './deploy-project-environment'
 import { Role } from '@aws-cdk/aws-iam'
 import { StageName } from '../../config/config';
 

lib/projects/deploy-project.ts

@@ -0,0 +1,61 @@
+/**
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: MIT-0
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the "Software"), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+
+import { Construct } from '@aws-cdk/core'
+import { PipelineProject, ProjectProps, BuildSpec} from '@aws-cdk/aws-codebuild'
+import { defaultEnvironment } from './build-project-environment'
+import { projectEnvironmentVars } from './build-project-environment'
+import { Role } from '@aws-cdk/aws-iam'
+
+export interface BuildProjectProps extends ProjectProps {
+  repoName: string
+  bucketName: string
+  bucketArn: string
+  role: Role
+}
+
+export class TestProject extends PipelineProject {
+  constructor(scope: Construct, id: string, props: BuildProjectProps) {
+    const { repoName, bucketName, bucketArn } = props
+    super(scope, id, {
+      projectName: id,
+      role: props.role,
+      environment: defaultEnvironment,
+      environmentVariables: projectEnvironmentVars({ repoName, bucketName, bucketArn }),
+      buildSpec: BuildSpec.fromObject({
+        version: '0.2',
+        phases: {
+          install: {
+            'runtime-versions': {
+              nodejs: '10'
+            }
+          },
+          build: {
+            commands: [
+              'bash ${CODEBUILD_SRC_DIR}/scripts/test.sh'
+            ]
+          }
+        },
+        artifacts: {
+          files: '**/*'
+        }
+      })
+    })
+  }
+}