From 16e9da49cc8593e7d93c9e0cdea65c44c42dacee Mon Sep 17 00:00:00 2001
From: Alex Petty <pettyalex@gmail.com>
Date: Thu, 15 Feb 2024 16:08:29 -0600
Subject: [PATCH 1/5] Add the support for configurable protocol, ftp, http, or
 https to ensembl reference data download and vep cache download.

---
 bio/reference/ensembl-annotation/wrapper.py | 4 +++-
 bio/reference/ensembl-sequence/wrapper.py   | 3 ++-
 bio/reference/ensembl-variation/wrapper.py  | 4 +++-
 bio/vep/cache/wrapper.py                    | 3 ++-
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/bio/reference/ensembl-annotation/wrapper.py b/bio/reference/ensembl-annotation/wrapper.py
index 2f1b78c2a6a..5a5b6729c73 100644
--- a/bio/reference/ensembl-annotation/wrapper.py
+++ b/bio/reference/ensembl-annotation/wrapper.py
@@ -19,6 +19,7 @@
 out_fmt = Path(snakemake.output[0]).suffixes
 out_gz = (out_fmt.pop() and True) if out_fmt[-1] == ".gz" else False
 out_fmt = out_fmt.pop().lstrip(".")
+protocol = snakemake.params.get("protocol", "ftp")
 
 
 branch = ""
@@ -48,7 +49,7 @@
     )
 
 
-url = "ftp://ftp.ensembl.org/pub/{branch}release-{release}/{out_fmt}/{species}/{species_cap}.{build}.{gtf_release}.{flavor}{suffix}".format(
+url = "{protocol}://ftp.ensembl.org/pub/{branch}release-{release}/{out_fmt}/{species}/{species_cap}.{build}.{gtf_release}.{flavor}{suffix}".format(
     release=release,
     gtf_release=gtf_release,
     build=build,
@@ -58,6 +59,7 @@
     suffix=suffix,
     flavor=flavor,
     branch=branch,
+    protocol=protocol,
 )
 
 
diff --git a/bio/reference/ensembl-sequence/wrapper.py b/bio/reference/ensembl-sequence/wrapper.py
index df9a6eef693..a13ccc612f6 100644
--- a/bio/reference/ensembl-sequence/wrapper.py
+++ b/bio/reference/ensembl-sequence/wrapper.py
@@ -11,6 +11,7 @@
 species = snakemake.params.species.lower()
 release = int(snakemake.params.release)
 build = snakemake.params.build
+protocol = snakemake.params.get("protocol", "ftp")
 
 branch = ""
 if release >= 81 and build == "GRCh37":
@@ -51,7 +52,7 @@
         )
 
 spec = spec.format(build=build, release=release)
-url_prefix = f"ftp://ftp.ensembl.org/pub/{branch}release-{release}/fasta/{species}/{datatype}/{species.capitalize()}.{spec}"
+url_prefix = f"{protocol}://ftp.ensembl.org/pub/{branch}release-{release}/fasta/{species}/{datatype}/{species.capitalize()}.{spec}"
 
 success = False
 for suffix in suffixes:
diff --git a/bio/reference/ensembl-variation/wrapper.py b/bio/reference/ensembl-variation/wrapper.py
index f42f7aa5287..952386e5fa8 100644
--- a/bio/reference/ensembl-variation/wrapper.py
+++ b/bio/reference/ensembl-variation/wrapper.py
@@ -15,6 +15,7 @@
 build = snakemake.params.build
 type = snakemake.params.type
 chromosome = snakemake.params.get("chromosome", "")
+protocol = snakemake.params.get("protocol", "ftp")
 
 
 branch = ""
@@ -63,12 +64,13 @@
 species_filename = species if release >= 91 else species.capitalize()
 
 urls = [
-    "ftp://ftp.ensembl.org/pub/{branch}release-{release}/variation/vcf/{species}/{species_filename}{suffix}.vcf.gz".format(
+    "{protocol}://ftp.ensembl.org/pub/{branch}release-{release}/variation/vcf/{species}/{species_filename}{suffix}.vcf.gz".format(
         release=release,
         species=species,
         suffix=suffix,
         species_filename=species_filename,
         branch=branch,
+        protocol=protocol,
     )
     for suffix in suffixes
 ]
diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py
index 543f1b26169..4c1d7857986 100644
--- a/bio/vep/cache/wrapper.py
+++ b/bio/vep/cache/wrapper.py
@@ -9,6 +9,7 @@
 
 
 extra = snakemake.params.get("extra", "")
+protocol = snakemake.params.get("protocol", "ftp")
 
 try:
     release = int(snakemake.params.release)
@@ -24,7 +25,7 @@
     )
     log = snakemake.log_fmt_shell(stdout=True, stderr=True)
     shell(
-        "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/"
+        "curl -L {protocol}://ftp.ensembl.org/pub/release-{snakemake.params.release}/"
         "variation/{vep_dir}/{cache_tarball} "
         "-o {tmpdir}/{cache_tarball} {log}"
     )

From d82fa7895fb566ce8647221e2664ecf0b6784ea2 Mon Sep 17 00:00:00 2001
From: Alex Petty <pettyalex@gmail.com>
Date: Mon, 19 Feb 2024 14:21:48 -0600
Subject: [PATCH 2/5] Add tests for getting reference data over http

---
 bio/reference/ensembl-annotation/test/Snakefile   | 15 +++++++++++++++
 bio/reference/ensembl-sequence/test/Snakefile     | 15 +++++++++++++++
 .../ensembl-variation/test/http_protocol.smk      | 13 +++++++++++++
 bio/vep/cache/test/Snakefile                      | 14 ++++++++++++++
 4 files changed, 57 insertions(+)
 create mode 100644 bio/reference/ensembl-variation/test/http_protocol.smk

diff --git a/bio/reference/ensembl-annotation/test/Snakefile b/bio/reference/ensembl-annotation/test/Snakefile
index fed87b2ee64..9999f698bce 100644
--- a/bio/reference/ensembl-annotation/test/Snakefile
+++ b/bio/reference/ensembl-annotation/test/Snakefile
@@ -28,3 +28,18 @@ rule get_annotation_gz:
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/reference/ensembl-annotation"
+
+rule get_annotation_http_protocol:
+    output:
+        "refs/annotation.gtf",
+    params:
+        species="homo_sapiens",
+        release="105",
+        build="GRCh37",
+        flavor="",
+        protocol="http",
+    log:
+        "logs/get_annotation.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
+    wrapper:
+        "master/bio/reference/ensembl-annotation"
diff --git a/bio/reference/ensembl-sequence/test/Snakefile b/bio/reference/ensembl-sequence/test/Snakefile
index a8227f4cdd1..de67094e031 100644
--- a/bio/reference/ensembl-sequence/test/Snakefile
+++ b/bio/reference/ensembl-sequence/test/Snakefile
@@ -44,3 +44,18 @@ rule get_multiple_chromosome:
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/reference/ensembl-sequence"
+
+rule get_genome_http_protocol:
+    output:
+        "refs/genome.fasta",
+    params:
+        species="saccharomyces_cerevisiae",
+        datatype="dna",
+        build="R64-1-1",
+        release="98",
+        protocol="http",
+    log:
+        "logs/get_genome.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
+    wrapper:
+        "master/bio/reference/ensembl-sequence"
\ No newline at end of file
diff --git a/bio/reference/ensembl-variation/test/http_protocol.smk b/bio/reference/ensembl-variation/test/http_protocol.smk
new file mode 100644
index 00000000000..1986c37b1ae
--- /dev/null
+++ b/bio/reference/ensembl-variation/test/http_protocol.smk
@@ -0,0 +1,13 @@
+rule get_variation_http_protocol:
+    output:
+        vcf="refs/variation.vcf.gz",
+    params:
+        species="saccharomyces_cerevisiae",
+        release="98",
+        build="R64-1-1",
+        type="all",
+        protocol="http"
+    log:
+        "logs/get_variation.log",
+    wrapper:
+        "master/bio/reference/ensembl-variation"
diff --git a/bio/vep/cache/test/Snakefile b/bio/vep/cache/test/Snakefile
index 4f6285893c1..1b5daded2ac 100644
--- a/bio/vep/cache/test/Snakefile
+++ b/bio/vep/cache/test/Snakefile
@@ -10,3 +10,17 @@ rule get_vep_cache:
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/vep/cache"
+
+rule get_vep_cache_http_protocol:
+    output:
+        directory("resources/vep/cache"),
+    params:
+        species="saccharomyces_cerevisiae",
+        build="R64-1-1",
+        release="98",
+        protocol="http",
+    log:
+        "logs/vep/cache.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
+    wrapper:
+        "master/bio/vep/cache"

From 94cdbfd88d7363ffc00c0d8fb03a093e95af9698 Mon Sep 17 00:00:00 2001
From: Alex Petty <pettyalex@gmail.com>
Date: Mon, 19 Feb 2024 14:31:23 -0600
Subject: [PATCH 3/5] Run snakefmt over reference download tools.

---
 bio/reference/ensembl-annotation/test/Snakefile        | 1 +
 bio/reference/ensembl-sequence/test/Snakefile          | 4 +++-
 bio/reference/ensembl-variation/test/chrom_wise.smk    | 6 +++---
 bio/reference/ensembl-variation/test/grch37.smk        | 8 ++++----
 bio/reference/ensembl-variation/test/http_protocol.smk | 2 +-
 bio/reference/ensembl-variation/test/old_release.smk   | 8 +++-----
 bio/reference/ensembl-variation/test/with_fai.smk      | 8 ++++----
 bio/vep/cache/test/Snakefile                           | 1 +
 8 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/bio/reference/ensembl-annotation/test/Snakefile b/bio/reference/ensembl-annotation/test/Snakefile
index 9999f698bce..ef58e7b7472 100644
--- a/bio/reference/ensembl-annotation/test/Snakefile
+++ b/bio/reference/ensembl-annotation/test/Snakefile
@@ -29,6 +29,7 @@ rule get_annotation_gz:
     wrapper:
         "master/bio/reference/ensembl-annotation"
 
+
 rule get_annotation_http_protocol:
     output:
         "refs/annotation.gtf",
diff --git a/bio/reference/ensembl-sequence/test/Snakefile b/bio/reference/ensembl-sequence/test/Snakefile
index de67094e031..52ab560d270 100644
--- a/bio/reference/ensembl-sequence/test/Snakefile
+++ b/bio/reference/ensembl-sequence/test/Snakefile
@@ -29,6 +29,7 @@ rule get_single_chromosome:
     wrapper:
         "master/bio/reference/ensembl-sequence"
 
+
 rule get_multiple_chromosome:
     output:
         "refs/chr1_and_chr2.fasta",
@@ -45,6 +46,7 @@ rule get_multiple_chromosome:
     wrapper:
         "master/bio/reference/ensembl-sequence"
 
+
 rule get_genome_http_protocol:
     output:
         "refs/genome.fasta",
@@ -58,4 +60,4 @@ rule get_genome_http_protocol:
         "logs/get_genome.log",
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
-        "master/bio/reference/ensembl-sequence"
\ No newline at end of file
+        "master/bio/reference/ensembl-sequence"
diff --git a/bio/reference/ensembl-variation/test/chrom_wise.smk b/bio/reference/ensembl-variation/test/chrom_wise.smk
index 3faacc41854..780617891e2 100644
--- a/bio/reference/ensembl-variation/test/chrom_wise.smk
+++ b/bio/reference/ensembl-variation/test/chrom_wise.smk
@@ -1,13 +1,13 @@
 rule get_variation:
     output:
-        vcf="refs/variation.vcf.gz"
+        vcf="refs/variation.vcf.gz",
     params:
         species="homo_sapiens",
         release="104",
         build="GRCh38",
-        type="all", # one of "all", "somatic", "structural_variation"
+        type="all",  # one of "all", "somatic", "structural_variation"
         chromosome="21",
     log:
-        "logs/get_variation.log"
+        "logs/get_variation.log",
     wrapper:
         "master/bio/reference/ensembl-variation"
diff --git a/bio/reference/ensembl-variation/test/grch37.smk b/bio/reference/ensembl-variation/test/grch37.smk
index a44a10d5ee6..97b788cd6a4 100644
--- a/bio/reference/ensembl-variation/test/grch37.smk
+++ b/bio/reference/ensembl-variation/test/grch37.smk
@@ -1,14 +1,14 @@
 rule get_variation_with_contig_lengths:
     input:
-        fai="refs/grch37.fasta.fai"
+        fai="refs/grch37.fasta.fai",
     output:
-        vcf="refs/variation.vcf.gz"
+        vcf="refs/variation.vcf.gz",
     params:
         species="homo_sapiens",
         release="100",
         build="GRCh37",
-        type="all" # one of "all", "somatic", "structural_variation"
+        type="all",  # one of "all", "somatic", "structural_variation"
     log:
-        "logs/get_variation.log"
+        "logs/get_variation.log",
     wrapper:
         "master/bio/reference/ensembl-variation"
diff --git a/bio/reference/ensembl-variation/test/http_protocol.smk b/bio/reference/ensembl-variation/test/http_protocol.smk
index 1986c37b1ae..2bd507ef109 100644
--- a/bio/reference/ensembl-variation/test/http_protocol.smk
+++ b/bio/reference/ensembl-variation/test/http_protocol.smk
@@ -6,7 +6,7 @@ rule get_variation_http_protocol:
         release="98",
         build="R64-1-1",
         type="all",
-        protocol="http"
+        protocol="http",
     log:
         "logs/get_variation.log",
     wrapper:
diff --git a/bio/reference/ensembl-variation/test/old_release.smk b/bio/reference/ensembl-variation/test/old_release.smk
index 23cafdb26cf..52dff157f67 100644
--- a/bio/reference/ensembl-variation/test/old_release.smk
+++ b/bio/reference/ensembl-variation/test/old_release.smk
@@ -1,16 +1,14 @@
 rule get_variation:
     output:
-        vcf="refs/variation.vcf.gz"
+        vcf="refs/variation.vcf.gz",
         # optional: add fai to get VCF with annotated contig lengths (as required by GATK)
         # fai="refs/genome.fasta.fai"
     params:
         species="saccharomyces_cerevisiae",
         release="98",
         build="R64-1-1",
-        type="all" # one of "all", "somatic", "structural_variation"
+        type="all",  # one of "all", "somatic", "structural_variation"
     log:
-        "logs/get_variation.log"
+        "logs/get_variation.log",
     wrapper:
         "master/bio/reference/ensembl-variation"
-
-
diff --git a/bio/reference/ensembl-variation/test/with_fai.smk b/bio/reference/ensembl-variation/test/with_fai.smk
index bc7746d998e..4952c31e10d 100644
--- a/bio/reference/ensembl-variation/test/with_fai.smk
+++ b/bio/reference/ensembl-variation/test/with_fai.smk
@@ -1,14 +1,14 @@
 rule get_variation_with_contig_lengths:
     input:
-        fai="refs/genome.fasta.fai"
+        fai="refs/genome.fasta.fai",
     output:
-        vcf="refs/variation.vcf.gz"
+        vcf="refs/variation.vcf.gz",
     params:
         species="saccharomyces_cerevisiae",
         release="98",
         build="R64-1-1",
-        type="all" # one of "all", "somatic", "structural_variation"
+        type="all",  # one of "all", "somatic", "structural_variation"
     log:
-        "logs/get_variation.log"
+        "logs/get_variation.log",
     wrapper:
         "master/bio/reference/ensembl-variation"
diff --git a/bio/vep/cache/test/Snakefile b/bio/vep/cache/test/Snakefile
index 1b5daded2ac..9b83b84996e 100644
--- a/bio/vep/cache/test/Snakefile
+++ b/bio/vep/cache/test/Snakefile
@@ -11,6 +11,7 @@ rule get_vep_cache:
     wrapper:
         "master/bio/vep/cache"
 
+
 rule get_vep_cache_http_protocol:
     output:
         directory("resources/vep/cache"),

From 17d4d28648e3f17749936270f2b8ac3c969fd0fb Mon Sep 17 00:00:00 2001
From: Alex Petty <pettyalex@gmail.com>
Date: Thu, 22 Feb 2024 10:06:51 -0600
Subject: [PATCH 4/5] Split off https into different snakemake files to avoid
 name collision. Add each new test case to test.py.

---
 .../ensembl-annotation/test/Snakefile         | 16 -----
 .../test/ensembl_annotation_https.smk         | 14 +++++
 bio/reference/ensembl-sequence/test/Snakefile | 16 -----
 .../test/ensembl_sequence_https.smk           | 14 +++++
 ...otocol.smk => ensembl_variation_https.smk} |  5 +-
 bio/vep/cache/test/Snakefile                  | 15 -----
 bio/vep/cache/test/vep_cache_https.smk        | 14 +++++
 test.py                                       | 63 +++++++++++++++++--
 8 files changed, 102 insertions(+), 55 deletions(-)
 create mode 100644 bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk
 create mode 100644 bio/reference/ensembl-sequence/test/ensembl_sequence_https.smk
 rename bio/reference/ensembl-variation/test/{http_protocol.smk => ensembl_variation_https.smk} (64%)
 create mode 100644 bio/vep/cache/test/vep_cache_https.smk

diff --git a/bio/reference/ensembl-annotation/test/Snakefile b/bio/reference/ensembl-annotation/test/Snakefile
index ef58e7b7472..fed87b2ee64 100644
--- a/bio/reference/ensembl-annotation/test/Snakefile
+++ b/bio/reference/ensembl-annotation/test/Snakefile
@@ -28,19 +28,3 @@ rule get_annotation_gz:
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/reference/ensembl-annotation"
-
-
-rule get_annotation_http_protocol:
-    output:
-        "refs/annotation.gtf",
-    params:
-        species="homo_sapiens",
-        release="105",
-        build="GRCh37",
-        flavor="",
-        protocol="http",
-    log:
-        "logs/get_annotation.log",
-    cache: "omit-software"  # save space and time with between workflow caching (see docs)
-    wrapper:
-        "master/bio/reference/ensembl-annotation"
diff --git a/bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk b/bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk
new file mode 100644
index 00000000000..3d075fa997a
--- /dev/null
+++ b/bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk
@@ -0,0 +1,14 @@
+rule get_annotation_https_protocol:
+    output:
+        "refs/annotation.gtf",
+    params:
+        species="homo_sapiens",
+        release="105",
+        build="GRCh37",
+        flavor="",
+        protocol="https",
+    log:
+        "logs/get_annotation.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
+    wrapper:
+        "master/bio/reference/ensembl-annotation"
diff --git a/bio/reference/ensembl-sequence/test/Snakefile b/bio/reference/ensembl-sequence/test/Snakefile
index 52ab560d270..6de1b22c485 100644
--- a/bio/reference/ensembl-sequence/test/Snakefile
+++ b/bio/reference/ensembl-sequence/test/Snakefile
@@ -45,19 +45,3 @@ rule get_multiple_chromosome:
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/reference/ensembl-sequence"
-
-
-rule get_genome_http_protocol:
-    output:
-        "refs/genome.fasta",
-    params:
-        species="saccharomyces_cerevisiae",
-        datatype="dna",
-        build="R64-1-1",
-        release="98",
-        protocol="http",
-    log:
-        "logs/get_genome.log",
-    cache: "omit-software"  # save space and time with between workflow caching (see docs)
-    wrapper:
-        "master/bio/reference/ensembl-sequence"
diff --git a/bio/reference/ensembl-sequence/test/ensembl_sequence_https.smk b/bio/reference/ensembl-sequence/test/ensembl_sequence_https.smk
new file mode 100644
index 00000000000..f19ffc93062
--- /dev/null
+++ b/bio/reference/ensembl-sequence/test/ensembl_sequence_https.smk
@@ -0,0 +1,14 @@
+rule get_genome_http_protocol:
+    output:
+        "refs/genome.fasta",
+    params:
+        species="saccharomyces_cerevisiae",
+        datatype="dna",
+        build="R64-1-1",
+        release="98",
+        protocol="http",
+    log:
+        "logs/get_genome.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
+    wrapper:
+        "master/bio/reference/ensembl-sequence"
diff --git a/bio/reference/ensembl-variation/test/http_protocol.smk b/bio/reference/ensembl-variation/test/ensembl_variation_https.smk
similarity index 64%
rename from bio/reference/ensembl-variation/test/http_protocol.smk
rename to bio/reference/ensembl-variation/test/ensembl_variation_https.smk
index 2bd507ef109..d51363eac9b 100644
--- a/bio/reference/ensembl-variation/test/http_protocol.smk
+++ b/bio/reference/ensembl-variation/test/ensembl_variation_https.smk
@@ -1,4 +1,4 @@
-rule get_variation_http_protocol:
+rule get_variation_https_protocol:
     output:
         vcf="refs/variation.vcf.gz",
     params:
@@ -6,8 +6,9 @@ rule get_variation_http_protocol:
         release="98",
         build="R64-1-1",
         type="all",
-        protocol="http",
+        protocol="https",
     log:
         "logs/get_variation.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/reference/ensembl-variation"
diff --git a/bio/vep/cache/test/Snakefile b/bio/vep/cache/test/Snakefile
index 9b83b84996e..4f6285893c1 100644
--- a/bio/vep/cache/test/Snakefile
+++ b/bio/vep/cache/test/Snakefile
@@ -10,18 +10,3 @@ rule get_vep_cache:
     cache: "omit-software"  # save space and time with between workflow caching (see docs)
     wrapper:
         "master/bio/vep/cache"
-
-
-rule get_vep_cache_http_protocol:
-    output:
-        directory("resources/vep/cache"),
-    params:
-        species="saccharomyces_cerevisiae",
-        build="R64-1-1",
-        release="98",
-        protocol="http",
-    log:
-        "logs/vep/cache.log",
-    cache: "omit-software"  # save space and time with between workflow caching (see docs)
-    wrapper:
-        "master/bio/vep/cache"
diff --git a/bio/vep/cache/test/vep_cache_https.smk b/bio/vep/cache/test/vep_cache_https.smk
new file mode 100644
index 00000000000..8b25352965c
--- /dev/null
+++ b/bio/vep/cache/test/vep_cache_https.smk
@@ -0,0 +1,14 @@
+
+rule get_vep_cache_https_protocol:
+    output:
+        directory("resources/vep/cache"),
+    params:
+        species="saccharomyces_cerevisiae",
+        build="R64-1-1",
+        release="98",
+        protocol="https",
+    log:
+        "logs/vep/cache.log",
+    cache: "omit-software"  # save space and time with between workflow caching (see docs)
+    wrapper:
+        "master/bio/vep/cache"
diff --git a/test.py b/test.py
index af555e9d58e..d34762dfb71 100644
--- a/test.py
+++ b/test.py
@@ -5329,6 +5329,22 @@ def test_ensembl_sequence_old_release():
     )
 
 
+@skip_if_not_modified
+def test_ensembl_sequence_https():
+    run(
+        "bio/reference/ensembl-sequence",
+        [
+            "snakemake",
+            "-s",
+            "ensembl_sequence_https.smk",
+            "--cores",
+            "1",
+            "--use-conda",
+            "-F",
+        ],
+    )
+
+
 @skip_if_not_modified
 def test_ensembl_sequence_chromosome():
     run(
@@ -5374,7 +5390,16 @@ def test_ensembl_annotation_gtf():
 def test_ensembl_annotation_gtf_gz():
     run(
         "bio/reference/ensembl-annotation",
-        ["snakemake", "--cores", "1", "refs/annotation.gtf.gz", "--use-conda", "-F"],
+        [
+            "snakemake",
+            "--cores",
+            "1",
+            "refs/annotation.gtf.gz",
+            "--use-conda",
+            "-F",
+            "-s",
+            "ensembl_annotation_https.smk",
+        ],
     )
 
 
@@ -5428,17 +5453,26 @@ def test_ensembl_variation_with_contig_lengths():
 
 
 @skip_if_not_modified
-def test_ega_fetch():
+def test_ensembl_variation_old_release():
     run(
-        "bio/ega/fetch",
+        "bio/reference/ensembl-variation",
         [
             "snakemake",
+            "-s",
+            "ensembl_variation_https.smk",
             "--cores",
             "1",
             "--use-conda",
             "-F",
-            "data/EGAF00007243774.cram"
-        ]
+        ],
+    )
+
+
+@skip_if_not_modified
+def test_ega_fetch():
+    run(
+        "bio/ega/fetch",
+        ["snakemake", "--cores", "1", "--use-conda", "-F", "data/EGAF00007243774.cram"],
     )
 
 
@@ -5801,11 +5835,28 @@ def test_vep_cache():
     )
 
 
+@skip_if_not_modified
+def test_vep_cache_https_protocol():
+    run(
+        "bio/vep/cache",
+        ["snakemake", "--cores", "1", "resources/vep/cache", "--use-conda", "-F"],
+    )
+
+
 @skip_if_not_modified
 def test_vep_plugins():
     run(
         "bio/vep/plugins",
-        ["snakemake", "--cores", "1", "resources/vep/plugins", "--use-conda", "-F"],
+        [
+            "snakemake",
+            "--cores",
+            "1",
+            "resources/vep/plugins",
+            "--use-conda",
+            "-F",
+            "-s",
+            "vep_cache_https.smk",
+        ],
     )
 
 

From 1a84082c465e039a67d7c0fe781550f3feaac448 Mon Sep 17 00:00:00 2001
From: Alex Petty <pettyalex@gmail.com>
Date: Fri, 23 Feb 2024 15:20:10 -0600
Subject: [PATCH 5/5] Fix tests for annotation and make https protocol output
 gzipped.

---
 .../ensembl-annotation/test/ensembl_annotation_https.smk  | 4 ++--
 test.py                                                   | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk b/bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk
index 3d075fa997a..a2586e2283c 100644
--- a/bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk
+++ b/bio/reference/ensembl-annotation/test/ensembl_annotation_https.smk
@@ -1,6 +1,6 @@
-rule get_annotation_https_protocol:
+rule get_annotation_https_protocol_gz:
     output:
-        "refs/annotation.gtf",
+        "refs/annotation.gtf.gz",
     params:
         species="homo_sapiens",
         release="105",
diff --git a/test.py b/test.py
index d34762dfb71..84a4faf7c10 100644
--- a/test.py
+++ b/test.py
@@ -5388,6 +5388,14 @@ def test_ensembl_annotation_gtf():
 
 @skip_if_not_modified
 def test_ensembl_annotation_gtf_gz():
+    run(
+        "bio/reference/ensembl-annotation",
+        ["snakemake", "--cores", "1", "refs/annotation.gtf.gz", "--use-conda", "-F"],
+    )
+
+
+@skip_if_not_modified
+def test_ensembl_annotation_gtf_https_gz():
     run(
         "bio/reference/ensembl-annotation",
         [