From e17983e4f71f756e5d9a5af735eb8aee4581e316 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 10:40:26 +0100 Subject: [PATCH 01/13] add official workflow to tutorial --- .gitignore | 5 +++ workflow/Snakefile | 56 ++++++++++++++++++++++++++++++++++ workflow/scripts/plot-quals.py | 9 ++++++ 3 files changed, 70 insertions(+) create mode 100644 .gitignore create mode 100644 workflow/Snakefile create mode 100644 workflow/scripts/plot-quals.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..15f5c82 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +calls +plots +sorted_reads +mapped_reads +.snakemake \ No newline at end of file diff --git a/workflow/Snakefile b/workflow/Snakefile new file mode 100644 index 0000000..3cba6f1 --- /dev/null +++ b/workflow/Snakefile @@ -0,0 +1,56 @@ +SAMPLES = ["A", "B"] + + +rule all: + input: + "plots/quals.svg" + + +rule bwa_map: + input: + "data/genome.fa", + "data/samples/{sample}.fastq" + output: + "mapped_reads/{sample}.bam" + shell: + "bwa mem {input} | samtools view -Sb - > {output}" + + +rule samtools_sort: + input: + "mapped_reads/{sample}.bam" + output: + "sorted_reads/{sample}.bam" + shell: + "samtools sort -T sorted_reads/{wildcards.sample} " + "-O bam {input} > {output}" + + +rule samtools_index: + input: + "sorted_reads/{sample}.bam" + output: + "sorted_reads/{sample}.bam.bai" + shell: + "samtools index {input}" + + +rule bcftools_call: + input: + fa="data/genome.fa", + bam=expand("sorted_reads/{sample}.bam", sample=SAMPLES), + bai=expand("sorted_reads/{sample}.bam.bai", sample=SAMPLES) + output: + "calls/all.vcf" + shell: + "bcftools mpileup -f {input.fa} {input.bam} | " + "bcftools call -mv - > {output}" + + +rule plot_quals: + input: + "calls/all.vcf" + output: + "plots/quals.svg" + script: + "scripts/plot-quals.py" \ No newline at end of file diff --git a/workflow/scripts/plot-quals.py b/workflow/scripts/plot-quals.py new file mode 100644 index 0000000..8483275 --- /dev/null +++ b/workflow/scripts/plot-quals.py @@ -0,0 +1,9 @@ +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from pysam import VariantFile + +quals = [record.qual for record in VariantFile(snakemake.input[0])] +plt.hist(quals) + +plt.savefig(snakemake.output[0]) \ No newline at end of file From b74021001cbb6c8cb7a0b6f9f1a2dd37abc77d73 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 10:41:57 +0100 Subject: [PATCH 02/13] snakefmt --- workflow/Snakefile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3cba6f1..dcb194d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -3,24 +3,24 @@ SAMPLES = ["A", "B"] rule all: input: - "plots/quals.svg" + "plots/quals.svg", rule bwa_map: input: "data/genome.fa", - "data/samples/{sample}.fastq" + "data/samples/{sample}.fastq", output: - "mapped_reads/{sample}.bam" + "mapped_reads/{sample}.bam", shell: "bwa mem {input} | samtools view -Sb - > {output}" rule samtools_sort: input: - "mapped_reads/{sample}.bam" + "mapped_reads/{sample}.bam", output: - "sorted_reads/{sample}.bam" + "sorted_reads/{sample}.bam", shell: "samtools sort -T sorted_reads/{wildcards.sample} " "-O bam {input} > {output}" @@ -28,9 +28,9 @@ rule samtools_sort: rule samtools_index: input: - "sorted_reads/{sample}.bam" + "sorted_reads/{sample}.bam", output: - "sorted_reads/{sample}.bam.bai" + "sorted_reads/{sample}.bam.bai", shell: "samtools index {input}" @@ -39,9 +39,9 @@ rule bcftools_call: input: fa="data/genome.fa", bam=expand("sorted_reads/{sample}.bam", sample=SAMPLES), - bai=expand("sorted_reads/{sample}.bam.bai", sample=SAMPLES) + bai=expand("sorted_reads/{sample}.bam.bai", sample=SAMPLES), output: - "calls/all.vcf" + "calls/all.vcf", shell: "bcftools mpileup -f {input.fa} {input.bam} | " "bcftools call -mv - > {output}" @@ -49,8 +49,8 @@ rule bcftools_call: rule plot_quals: input: - "calls/all.vcf" + "calls/all.vcf", output: - "plots/quals.svg" + "plots/quals.svg", script: - "scripts/plot-quals.py" \ No newline at end of file + "scripts/plot-quals.py" From 9a3ba8f90cc14cfe96001470733ee11556a9ef5e Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 11:04:17 +0100 Subject: [PATCH 03/13] update ci --- .github/workflows/main.yml | 45 +++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e1381bd..e5fbc5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,11 +5,50 @@ on: branches: - master pull_request: - branches_ignore: [] + branches-ignore: [] jobs: + docker-build: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Lint Dockerfile + uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: Dockerfile + + - name: Build Docker Image + run: docker build -t snakemake-tutorial . + + lint: + runs-on: ubuntu-latest + needs: docker-build + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Lint Snakefile + uses: github/super-linter@v3 + env: + VALIDATE_ALL_CODEBASE: false + DEFAULT_BRANCH: master + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + VALIDATE_SNAKEMAKE_SNAKEFMT: true + + - name: DryRun Snakefile + run: docker run snakemake-tutorial snakemake --cores 1 --dryrun + test: runs-on: ubuntu-latest + needs: docker-build steps: - - uses: actions/checkout@master - - run: docker build . \ No newline at end of file + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Run Tests + run: docker run snakemake-tutorial snakemake --cores 1 \ No newline at end of file From 93cf633a85cd5d9badc674d78a35b84c6b9229e5 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 11:09:39 +0100 Subject: [PATCH 04/13] update ci --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5fbc5c..0bbc4c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: Test container build +name: Build, Lint, and Test on: push: @@ -18,6 +18,7 @@ jobs: uses: hadolint/hadolint-action@v3.1.0 with: dockerfile: Dockerfile + no-fail: true - name: Build Docker Image run: docker build -t snakemake-tutorial . From d1f0481599d0404a8c88dad6aeb0762615092ce6 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 11:24:29 +0100 Subject: [PATCH 05/13] bump superlinter --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0bbc4c9..082a0ee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,7 @@ jobs: fetch-depth: 0 - name: Lint Snakefile - uses: github/super-linter@v3 + uses: github/super-linter@v7 env: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: master @@ -49,7 +49,7 @@ jobs: needs: docker-build steps: - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run Tests run: docker run snakemake-tutorial snakemake --cores 1 \ No newline at end of file From 9e97cb053f062345008ddf82b6fddf5a00d01d0d Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 16:05:53 +0100 Subject: [PATCH 06/13] cache docker artifact --- .github/workflows/main.yml | 46 +++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 082a0ee..a0ff3cf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,6 +14,9 @@ jobs: - name: Checkout Code uses: actions/checkout@v4 + - name: Setup Buildx + uses: docker/setup-buildx-action@v3 + - name: Lint Dockerfile uses: hadolint/hadolint-action@v3.1.0 with: @@ -21,7 +24,18 @@ jobs: no-fail: true - name: Build Docker Image - run: docker build -t snakemake-tutorial . + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile + tags: snakemake-tutorial + outputs: type=docker, dest=/tmp/snakemake-tutorial.tar + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: snakemake-tutorial + path: /tmp/snakemake-tutorial.tar lint: runs-on: ubuntu-latest @@ -29,8 +43,9 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v4 - with: - fetch-depth: 0 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v3 - name: Lint Snakefile uses: github/super-linter@v7 @@ -41,8 +56,17 @@ jobs: VALIDATE_SNAKEMAKE_SNAKEFMT: true + - name: Download Artifact + uses: actions/download-artifact@v4 + with: + name: snakemake-tutorial + path: /tmp/snakemake-tutorial.tar + + - name: Load Docker Image + run: docker load -i /tmp/snakemake-tutorial.tar + - name: DryRun Snakefile - run: docker run snakemake-tutorial snakemake --cores 1 --dryrun + run: docker run snakemake/snakemake-tutorial snakemake --cores 1 --dryrun test: runs-on: ubuntu-latest @@ -51,5 +75,17 @@ jobs: - name: Checkout Code uses: actions/checkout@v4 + - name: Setup Buildx + uses: docker/setup-buildx-action@v3 + + - name: Download Artifact + uses: actions/download-artifact@v4 + with: + name: snakemake-tutorial + path: /tmp/snakemake-tutorial.tar + + - name: Load Docker Image + run: docker load -i /tmp/snakemake-tutorial.tar + - name: Run Tests - run: docker run snakemake-tutorial snakemake --cores 1 \ No newline at end of file + run: docker run snakemake-tutorial snakemake --cores 1 From ee7d2cf048cad51ee4de447e796930d2d04bd673 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 16:46:45 +0100 Subject: [PATCH 07/13] udpate ci --- .github/workflows/main.yml | 69 ++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a0ff3cf..51abb1d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,35 +7,38 @@ on: pull_request: branches-ignore: [] +env: + DOCKER_TAG_NAME: snakemake-tutorial + jobs: docker-build: runs-on: ubuntu-latest steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Buildx - uses: docker/setup-buildx-action@v3 - - - name: Lint Dockerfile - uses: hadolint/hadolint-action@v3.1.0 - with: - dockerfile: Dockerfile - no-fail: true - - - name: Build Docker Image - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile - tags: snakemake-tutorial - outputs: type=docker, dest=/tmp/snakemake-tutorial.tar - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: snakemake-tutorial - path: /tmp/snakemake-tutorial.tar + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v3 + + - name: Lint Dockerfile + uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: Dockerfile + no-fail: true + + - name: Build Docker Image + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile + tags: ${{ env.DOCKER_TAG_NAME }} + outputs: type=docker, dest=/tmp/${{ env.DOCKER_TAG_NAME }}.tar + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DOCKER_TAG_NAME }} + path: /tmp/${{ env.DOCKER_TAG_NAME }}.tar lint: runs-on: ubuntu-latest @@ -43,6 +46,8 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Setup Buildx uses: docker/setup-buildx-action@v3 @@ -60,13 +65,13 @@ jobs: uses: actions/download-artifact@v4 with: name: snakemake-tutorial - path: /tmp/snakemake-tutorial.tar + path: /tmp - name: Load Docker Image - run: docker load -i /tmp/snakemake-tutorial.tar + run: docker load -i /tmp/${{ env.DOCKER_TAG_NAME }}.tar - name: DryRun Snakefile - run: docker run snakemake/snakemake-tutorial snakemake --cores 1 --dryrun + run: docker run ${{ env.DOCKER_TAG_NAME }} snakemake --cores 1 --dryrun test: runs-on: ubuntu-latest @@ -81,11 +86,11 @@ jobs: - name: Download Artifact uses: actions/download-artifact@v4 with: - name: snakemake-tutorial - path: /tmp/snakemake-tutorial.tar + name: ${{ env.DOCKER_TAG_NAME }} + path: /tmp - name: Load Docker Image - run: docker load -i /tmp/snakemake-tutorial.tar + run: docker load -i /tmp/${{env.DOCKER_TAG_NAME}}.tar - name: Run Tests - run: docker run snakemake-tutorial snakemake --cores 1 + run: docker run ${{ env.DOCKER_TAG_NAME }} snakemake --cores 1 From 4daa61e0f50d73187aa76c75f259a05ba543c2eb Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 16:50:56 +0100 Subject: [PATCH 08/13] address warning --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c472fed..6a6b116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,4 @@ ADD environment.yaml . RUN conda create -n snakemake-tutorial --clone snakemake; \ conda env update -n snakemake-tutorial -f environment.yaml; RUN mkdir -p /tmp/conda -ENV CONDA_PKGS_DIRS /tmp/conda \ No newline at end of file +ENV CONDA_PKGS_DIRS=/tmp/conda \ No newline at end of file From 36dd2f94a5e1daded706c2fc5d050a95c04966e4 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 17:17:42 +0100 Subject: [PATCH 09/13] update docker run commmand --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51abb1d..7e9579d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,4 +93,6 @@ jobs: run: docker load -i /tmp/${{env.DOCKER_TAG_NAME}}.tar - name: Run Tests - run: docker run ${{ env.DOCKER_TAG_NAME }} snakemake --cores 1 + run: docker run -v $(pwd):/work -w /work snakemake/snakemake-tutorial:stable \ + conda run -n snakemake-tutorial snakemake --cores 1 + From 76eafb7ada8977204885e7613acf9fa50d19c254 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 17:19:03 +0100 Subject: [PATCH 10/13] update docker run command --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7e9579d..6b7783f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,6 +93,6 @@ jobs: run: docker load -i /tmp/${{env.DOCKER_TAG_NAME}}.tar - name: Run Tests - run: docker run -v $(pwd):/work -w /work snakemake/snakemake-tutorial:stable \ - conda run -n snakemake-tutorial snakemake --cores 1 + run: docker run -v $(pwd):/work -w /work ${{env.DOCKER_TAG_NAME}} \ + conda run -n ${{env.DOCKER_TAG_NAME}} -tutorial snakemake --cores 1 From 1150d82a27540e8e20af72d69e5435ddc41832e0 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 17:36:10 +0100 Subject: [PATCH 11/13] fix docker run command --- .github/workflows/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b7783f..78bb826 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,7 +71,9 @@ jobs: run: docker load -i /tmp/${{ env.DOCKER_TAG_NAME }}.tar - name: DryRun Snakefile - run: docker run ${{ env.DOCKER_TAG_NAME }} snakemake --cores 1 --dryrun + run: | + docker run -v $(pwd):/work -w /work ${{env.DOCKER_TAG_NAME}} \ + conda run -n ${{env.DOCKER_TAG_NAME}} snakemake --cores 1 test: runs-on: ubuntu-latest @@ -93,6 +95,7 @@ jobs: run: docker load -i /tmp/${{env.DOCKER_TAG_NAME}}.tar - name: Run Tests - run: docker run -v $(pwd):/work -w /work ${{env.DOCKER_TAG_NAME}} \ + run: | + docker run -v $(pwd):/work -w /work ${{env.DOCKER_TAG_NAME}} \ conda run -n ${{env.DOCKER_TAG_NAME}} -tutorial snakemake --cores 1 From 9da520df865e5cca5070fa3751c59f6f4cec95ce Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 17:51:07 +0100 Subject: [PATCH 12/13] fix --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 78bb826..bc908bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,7 +73,7 @@ jobs: - name: DryRun Snakefile run: | docker run -v $(pwd):/work -w /work ${{env.DOCKER_TAG_NAME}} \ - conda run -n ${{env.DOCKER_TAG_NAME}} snakemake --cores 1 + conda run -n ${{env.DOCKER_TAG_NAME}} snakemake -n test: runs-on: ubuntu-latest @@ -97,5 +97,5 @@ jobs: - name: Run Tests run: | docker run -v $(pwd):/work -w /work ${{env.DOCKER_TAG_NAME}} \ - conda run -n ${{env.DOCKER_TAG_NAME}} -tutorial snakemake --cores 1 + conda run -n ${{env.DOCKER_TAG_NAME}} snakemake --cores 1 From 9db203d3ba40c30bca559c85dd9d937960724c16 Mon Sep 17 00:00:00 2001 From: jakevc Date: Thu, 13 Mar 2025 18:02:50 +0100 Subject: [PATCH 13/13] address warning --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a6b116..534d5ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM snakemake/snakemake:stable -ADD environment.yaml . +COPY environment.yaml . RUN conda create -n snakemake-tutorial --clone snakemake; \ conda env update -n snakemake-tutorial -f environment.yaml; RUN mkdir -p /tmp/conda