diff --git a/README.md b/README.md index 9b8d536..7268dfe 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ - `form.yml.erb`: Defines the web form and input parameters. - `template/script.sh.erb`: Main job script for running predictions. +- `template/sanitise_input.py`: Validates and normalises staged FASTA and samplesheet inputs. - `submit.yml.erb`: Job submission configuration. - `info.html.erb`: Displays result links after job completion. - `.github/workflows/`: CI/CD deployment workflows. diff --git a/completed.html.erb b/completed.html.erb index 164d1bb..8d1d347 100644 --- a/completed.html.erb +++ b/completed.html.erb @@ -25,7 +25,7 @@ def tail_text(path, max_bytes = 8192, max_lines = 10) end end -def sanitize_live_log(text) +def sanitise_live_log(text) return nil if text.nil? ansi_pattern = /\e\[[0-9;?]*[ -\/]*[@-~]/ @@ -69,7 +69,7 @@ def sanitize_live_log(text) compact.join("\n").strip end -def sanitize_output_log(text) +def sanitise_output_log(text) return nil if text.nil? lines = text.split("\n") @@ -116,9 +116,9 @@ output_log_url = file_url(results_url_base, output_log_path) summary_log_url = File.file?(output_log_path) ? output_log_url : live_log_url summary_log_label = File.file?(output_log_path) ? "output.log" : "live_log.log" log_text = if File.file?(output_log_path) - sanitize_output_log(tail_text(output_log_path, 16384, 40)) + sanitise_output_log(tail_text(output_log_path, 16384, 40)) else - sanitize_live_log(tail_text(live_log_path)) + sanitise_live_log(tail_text(live_log_path)) end %>
diff --git a/form.js b/form.js index ef7eaaa..e67caca 100644 --- a/form.js +++ b/form.js @@ -43,8 +43,8 @@ const parseTruthy = (value) => { if (value === null || value === undefined) return false; - const normalized = String(value).trim().toLowerCase(); - return normalized === "" || normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on"; + const normalised = String(value).trim().toLowerCase(); + return normalised === "" || normalised === "true" || normalised === "1" || normalised === "yes" || normalised === "on"; }; const hasHideAttribute = (element) => diff --git a/scripts/validate_ood_app.rb b/scripts/validate_ood_app.rb index 04e4edc..86ce493 100755 --- a/scripts/validate_ood_app.rb +++ b/scripts/validate_ood_app.rb @@ -16,6 +16,7 @@ info.html.erb view.html.erb template/script.sh.erb + template/sanitise_input.py ].freeze class TemplateContext @@ -129,6 +130,20 @@ def validate_shell_templates!(template_context) end end +def validate_python_templates! + Dir.glob(ROOT.join("template/*.py")).sort.each do |path| + source = File.read(path) + _stdout, stderr, status = Open3.capture3( + "python3", "-c", "import sys; compile(sys.stdin.read(), sys.argv[1], 'exec')", path.to_s, + stdin_data: source + ) + next if status.success? + + relative_path = Pathname(path).relative_path_from(ROOT) + abort("Python validation failed for #{relative_path}: #{stderr.strip}") + end +end + assert_required_files! validate_manifest! @@ -137,5 +152,6 @@ def validate_shell_templates!(template_context) validate_yaml_file!("form.yml.erb", template_context) validate_yaml_file!("submit.yml.erb", template_context) validate_shell_templates!(template_context) +validate_python_templates! puts "Open OnDemand app validation passed for #{ROOT.basename}" diff --git a/template/sanitise_input.py b/template/sanitise_input.py new file mode 100644 index 0000000..a55acd4 --- /dev/null +++ b/template/sanitise_input.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Validate and normalise FASTA inputs staged for proteinfold.""" + +import csv +import hashlib +import os +import shutil +import sys +import tempfile + + +FASTA_SUFFIXES = (".fa", ".fasta") +YAML_SUFFIXES = (".yaml", ".yml") + + +def strip_whitespace(value): + return "".join( + char for char in value if not char.isspace() and char not in {"\ufeff", "\u200b"} + ) + + +def sanitise_fasta(source_path, destination_path): + has_record = False + record_has_content = False + blank_lines = 0 + normalised_headers = 0 + normalised_sequence_lines = 0 + normalised_line_endings = 0 + removed_sequence_whitespace = 0 + digest = hashlib.sha256() + + with open(source_path, encoding="utf-8-sig", newline="") as source, open( + destination_path, "w", encoding="utf-8" + ) as destination: + + def write(value): + destination.write(value) + digest.update(value.encode("utf-8")) + + for line_number, raw_line in enumerate(source, start=1): + if raw_line.endswith("\r\n") or raw_line.endswith("\r"): + normalised_line_endings += 1 + line = raw_line.strip() + if not line: + blank_lines += 1 + continue + if line.startswith(">"): + if has_record and not record_has_content: + raise ValueError( + f"FASTA record before line {line_number} has no sequence/content" + ) + header = line[1:].strip() + if not header: + raise ValueError(f"FASTA header on line {line_number} is empty") + normalised_header = f">{header}\n" + if raw_line != normalised_header: + normalised_headers += 1 + write(normalised_header) + has_record = True + record_has_content = False + continue + if not has_record: + raise ValueError( + f"Sequence/content found before the first FASTA header on line {line_number}" + ) + sequence = strip_whitespace(raw_line) + if sequence: + normalised_sequence = f"{sequence}\n" + if raw_line != normalised_sequence: + normalised_sequence_lines += 1 + removed_sequence_whitespace += len(raw_line.rstrip("\r\n")) - len(sequence) + write(normalised_sequence) + record_has_content = True + + if not has_record: + raise ValueError("No FASTA header found") + if not record_has_content: + raise ValueError("Final FASTA record has no sequence/content") + + changes = [] + if blank_lines: + changes.append(f"removed {blank_lines} blank line(s)") + if normalised_line_endings: + changes.append(f"normalised line endings in {normalised_line_endings} line(s)") + if normalised_headers: + changes.append(f"normalised {normalised_headers} FASTA header(s)") + if normalised_sequence_lines: + changes.append(f"normalised {normalised_sequence_lines} sequence line(s)") + if removed_sequence_whitespace: + changes.append( + f"removed {removed_sequence_whitespace} whitespace character(s) from sequence data" + ) + return digest.hexdigest(), changes + + +def sanitise_in_place(path): + with tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as temporary: + temporary_path = temporary.name + try: + normalised_hash, changes = sanitise_fasta(path, temporary_path) + os.replace(temporary_path, path) + return normalised_hash, changes + except BaseException: + os.unlink(temporary_path) + raise + + +def write_warnings(warning_path, warnings): + if not warnings: + return + with open(warning_path, "w", encoding="utf-8") as handle: + handle.write("WARNING: Input entries were adjusted before running proteinfold.\n\n") + handle.write("\n".join(warnings) + "\n") + for warning in warnings: + print(f"WARNING: {warning}", file=sys.stderr) + + +def input_kind(path): + suffix = os.path.splitext(path)[1].lower() + if suffix in FASTA_SUFFIXES: + return "fasta" + if suffix in YAML_SUFFIXES: + return "yaml" + return None + + +def unique_sample_id(sample_id, used_ids): + unique_id = sample_id + duplicate_number = 2 + while unique_id in used_ids: + unique_id = f"{sample_id}-{duplicate_number}" + duplicate_number += 1 + used_ids.add(unique_id) + return unique_id + + +def sanitise_samplesheet(samplesheet_path, output_path, input_dir, warning_path, af_method): + os.makedirs(input_dir, exist_ok=True) + samplesheet_dir = os.path.dirname(os.path.abspath(samplesheet_path)) + with open(samplesheet_path, encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + fieldnames = reader.fieldnames or [] + if [name.strip() for name in fieldnames[:2]] != ["id", "fasta"]: + raise ValueError("Samplesheet must start with the required columns: id,fasta") + + id_column, fasta_column = fieldnames[:2] + rows = [] + seen_paths = {} + seen_normalised_hashes = {} + used_ids = set() + warnings = [] + + for index, row in enumerate(reader, start=2): + if all((value or "").strip() == "" for value in row.values()): + continue + + input_path = (row.get(fasta_column) or "").strip() + if not input_path: + raise ValueError(f"Samplesheet row {index} is missing an input path") + if not os.path.isabs(input_path): + input_path = os.path.normpath(os.path.join(samplesheet_dir, input_path)) + if not os.path.isfile(input_path): + raise ValueError(f"Samplesheet row {index} input file not found: {input_path}") + + kind = input_kind(input_path) + if kind is None: + raise ValueError( + f"Samplesheet row {index} has unsupported input type: {input_path}. " + "Expected a .fa or .fasta file, or a .yaml/.yml file for Boltz." + ) + if kind == "yaml" and af_method != "boltz": + raise ValueError( + f"Samplesheet row {index} uses YAML input, which is only supported by Boltz" + ) + + source_path = os.path.realpath(input_path) + if source_path in seen_paths: + warnings.append( + f"Skipped samplesheet row {index}: input path resolves to the same file as " + f"row {seen_paths[source_path]}: {source_path}" + ) + continue + seen_paths[source_path] = index + + staged_path = os.path.join(input_dir, f"{index}_{os.path.basename(input_path)}") + if kind == "fasta": + try: + normalised_hash, changes = sanitise_fasta(input_path, staged_path) + except (OSError, UnicodeError, ValueError) as error: + raise ValueError(f"Samplesheet row {index} has invalid FASTA input: {error}") + if normalised_hash in seen_normalised_hashes: + os.unlink(staged_path) + warnings.append( + f"Skipped samplesheet row {index}: normalised FASTA content matches " + f"row {seen_normalised_hashes[normalised_hash]}" + ) + continue + seen_normalised_hashes[normalised_hash] = index + if changes: + warnings.append( + f"Sanitised FASTA referenced by samplesheet row {index}: {source_path} " + f"({'; '.join(changes)})" + ) + else: + shutil.copy2(input_path, staged_path) + + sample_id = (row.get(id_column) or "").strip() + unique_id = unique_sample_id(sample_id, used_ids) + if unique_id != sample_id: + warnings.append(f"Renamed duplicate sample ID on row {index}: {sample_id} -> {unique_id}") + + row[id_column] = unique_id + row[fasta_column] = staged_path + rows.append(row) + + if not rows: + raise ValueError("Samplesheet does not contain any data rows") + + with open(output_path, "w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + write_warnings(warning_path, warnings) + + +def sanitise_directory(directory, required, warning_path): + fasta_files = [ + entry.path + for entry in os.scandir(directory) + if entry.is_file() and entry.name.lower().endswith(FASTA_SUFFIXES) + ] + if required and not fasta_files: + raise ValueError(f"No FASTA files were found in {directory}") + + warnings = [] + seen_normalised_hashes = {} + for fasta_file in fasta_files: + normalised_hash, changes = sanitise_in_place(fasta_file) + if normalised_hash in seen_normalised_hashes: + os.unlink(fasta_file) + warnings.append( + f"Skipped duplicate FASTA file {fasta_file}: normalised content matches " + f"{seen_normalised_hashes[normalised_hash]}" + ) + continue + seen_normalised_hashes[normalised_hash] = fasta_file + if changes: + warnings.append(f"Sanitised FASTA file: {fasta_file} ({'; '.join(changes)})") + write_warnings(warning_path, warnings) + + +def main(): + command, *arguments = sys.argv[1:] + if command == "directory" and len(arguments) == 3: + directory, required, warning_path = arguments + sanitise_directory(directory, required == "true", warning_path) + elif command == "samplesheet" and len(arguments) == 5: + sanitise_samplesheet(*arguments) + else: + raise ValueError(f"Unknown or invalid sanitisation input: {command}") + + +if __name__ == "__main__": + main() diff --git a/template/script.sh.erb b/template/script.sh.erb index 02107d8..22d0d62 100755 --- a/template/script.sh.erb +++ b/template/script.sh.erb @@ -119,15 +119,40 @@ function convert_colon_delimited_to_fasta() { fi } +function strip_all_whitespace() { + python3 -c ' +import sys + +print("".join(char for char in sys.stdin.read() if not char.isspace() and char not in {"\ufeff", "\u200b"}), end="") +' <<< "$1" +} + +function sanitise_input() { + if ! python3 ./sanitise_input.py "$@"; then + echo "WARNING: Input sanitisation failed; continuing with the original input." >&2 + printf '%s\n' "WARNING: Input sanitisation failed; continuing with the original input." >> "$(pwd)/WARNING.txt" + return 1 + fi +} + # Check if the input is an amino acid sequence or a path -user_input="<%= context.samplesheet %>" +raw_user_input="<%= context.samplesheet %>" +user_input="$(printf '%s' "${raw_user_input}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" +sequence_input="$(strip_all_whitespace "${user_input}")" MODE="" -[[ "${user_input}" =~ ^[\ A-Z:\*\-]*$ ]] && MODE="MANUAL_INPUT" +[[ "${sequence_input}" =~ ^[A-Z:\*\-]+$ ]] && MODE="MANUAL_INPUT" [[ "${user_input}" =~ ^.*\.csv$ ]] && MODE="SAMPLESHEET_INPUT" [[ "${user_input}" =~ \.(fa(sta)?|y(a)?ml)$ ]] && MODE="FASTA_INPUT" [[ "${MODE}" == "" ]] && MODE="DIRECTORY" +if [ "${MODE}" == "MANUAL_INPUT" ]; then + if [ "${raw_user_input}" != "${sequence_input}" ]; then + echo "WARNING: Removed whitespace from manual sequence input" | tee "$(pwd)/WARNING.txt" >&2 + fi + user_input="${sequence_input}" +fi + echo "${MODE}" # Validate input file format compatibility with selected method @@ -142,15 +167,6 @@ if [ "${MODE}" == "FASTA_INPUT" ] && [[ "${user_input}" =~ \.(yaml|yml)$ ]]; the fi fi -# Check JSON compatibility - only supported by HelixFold3 -if [ "${MODE}" == "FASTA_INPUT" ] && [[ "${user_input}" =~ \.json$ ]]; then - if [ "${af_method}" != "helixfold3" ]; then - echo "ERROR: JSON input files are only supported by the HelixFold3 method." - echo "Current method: ${af_method}" - exit 1 - fi -fi - # Check directory contents for incompatible file types if [ "${MODE}" == "DIRECTORY" ] && [ -d "${user_input}" ]; then # Check for YAML/YML files in directory @@ -231,7 +247,11 @@ fi if [ "${MODE}" == "SAMPLESHEET_INPUT" ]; then - SAMPLESHEET="${user_input}" + if sanitise_input samplesheet "${user_input}" "$(pwd)/sanitised_samplesheet.csv" "$(pwd)/fasta/samplesheet_inputs" "$(pwd)/WARNING.txt" "${af_method}"; then + SAMPLESHEET="$(pwd)/sanitised_samplesheet.csv" + else + SAMPLESHEET="${user_input}" + fi fi if [ "${MODE}" == "FASTA_INPUT" ]; @@ -239,6 +259,9 @@ then mkdir -p fasta cp "${user_input}" ./fasta/ dos2unix ./fasta/* + if [[ "${user_input}" =~ \.fa(sta)?$ ]]; then + sanitise_input directory "./fasta" "true" "$(pwd)/WARNING.txt" || true + fi create-samplesheet --directory "./fasta" --suffix "${prot_mode}" ${OUTPUT_FORMAT} SAMPLESHEET="$(pwd)/samplesheet.csv" fi @@ -256,10 +279,15 @@ then cd ${working_dir} dos2unix ./fasta/* + sanitise_input directory "./fasta" "false" "$(pwd)/WARNING.txt" || true create-samplesheet --directory "./fasta" --suffix "${prot_mode}" ${OUTPUT_FORMAT} SAMPLESHEET="$(pwd)/samplesheet.csv" fi +if [ -f "$(pwd)/WARNING.txt" ]; then + cp "$(pwd)/WARNING.txt" "${OUT_DIR}/WARNING.txt" +fi + PROT_MODE="" ARGS="" NEXTFLOW_CONFIG_ARGS=("-c" "${PROJECT_ROOT}/kod_proteinfold-${RUN_ENVIRONMENT}.config") diff --git a/view.html.erb b/view.html.erb index b6ea0b5..7ff2265 100644 --- a/view.html.erb +++ b/view.html.erb @@ -24,7 +24,7 @@ def tail_text(path, max_bytes = 65536, max_lines = 120) end end -def sanitize_live_log(text) +def sanitise_live_log(text) return nil if text.nil? ansi_pattern = /\e\[[0-9;?]*[ -\/]*[@-~]/ @@ -100,7 +100,7 @@ results_base = (defined?(results_url_base) && results_url_base && !results_url_b log_dir = (defined?(session_output_dir) && session_output_dir && !session_output_dir.to_s.empty?) ? session_output_dir.to_s : File.join(base_dir, run_dir.to_s) log_path = File.join(log_dir, "live_log.log") log_url = file_url(results_base, log_path) -log_text = sanitize_live_log(tail_text(log_path)) +log_text = sanitise_live_log(tail_text(log_path)) %>