Merge branch 'develop' into 'main'

Add Deployment Trigger

See merge request plos/editorial_manager_transfer_service!14

Author: Rose Reatherford

.gitlab-ci.yml

@@ -0,0 +1,19 @@
+trigger_pipeline_dev:
+  stage: deploy
+  script:
+    - 'curl --fail --request POST --form token=$TRIGGER_TOKEN --form ref=develop "${CI_API_V4_URL}/projects/67747905/trigger/pipeline"'
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    - if: $CI_COMMIT_BRANCH == "develop"
+  environment: develop
+
+trigger_pipeline_prod:
+  stage: deploy
+  script:
+    - 'curl --fail --request POST --form token=$TRIGGER_TOKEN --form ref=main "${CI_API_V4_URL}/projects/67747905/trigger/pipeline"'
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    - if: $CI_COMMIT_BRANCH == "main"
+  environment: prod

consts.py

@@ -50,3 +50,5 @@
 GO_FILE_ELEMENT_TAG_FILEGROUP = "filegroup"
 GO_FILE_ELEMENT_TAG_FILE = "file"
 GO_FILE_ELEMENT_TAG_METADATA_FILE = "metadata-file"
+
+JATS_XML_FILE = 'editorial_manager_transfer_service/encoding/article_jats_1_2_aries.xml'

dev-requirements.txt

@@ -1,3 +1,3 @@
 hypothesis==6.138.7
-pytest==8.4.1
-python-magic==0.4.27
+python-magic==0.4.27
+lxml

enums/report_state.py

@@ -0,0 +1,15 @@
+"""
+A file for tracking the transfer log message types.
+"""
+__author__ = "Rosetta Reatherford"
+__license__ = "AGPL v3"
+__maintainer__ = "The Public Library of Science (PLOS)"
+
+import django.db.models as models
+from django.utils.translation import gettext_lazy as _
+
+class ReportState(models.TextChoices):
+    NORMAL = "000", _("No Error Detected")
+    IN_FLIGHT = "001", _("Article is in Flight")
+    FAILED_BUNDLING = "002", _("Failed Bundling")
+    FAILED_INGEST = "003", _("Editorial Manager Rejected at SFTP")

file_exporter.py

@@ -8,20 +8,26 @@
 import os
 import uuid
 import xml.etree.cElementTree as ETree
-import zipfile
 from collections.abc import Sequence
 from typing import List
 
-from django.core.exceptions import ObjectDoesNotExist
+from janeway_ftp import helpers as deposit_helpers
 
 import plugins.editorial_manager_transfer_service.consts as consts
 import plugins.editorial_manager_transfer_service.logger_messages as logger_messages
 from core.models import File
 from journal.models import Journal
+from plugins.editorial_manager_transfer_service.enums.report_state import ReportState
 from plugins.editorial_manager_transfer_service.enums.transfer_log_message_type import TransferLogMessageType
 from plugins.editorial_manager_transfer_service.models import TransferLogs
+from plugins.editorial_manager_transfer_service.utils.jats import get_xml_license_code, generate_jats_metadata
+from plugins.editorial_manager_transfer_service.utils.settings import get_license_code, get_submission_partner_code, \
+    get_journal_code
+from plugins.editorial_manager_transfer_service.utils.transfer_report import get_or_create_transfer_report, \
+    resolve_transfer_report
+from plugins.production_transporter.utilities import data_fetch
+from plugins.production_transporter.utilities.file_utils import copy_files_to_temp_deposit_folder
 from submission.models import Article
-from utils import setting_handler
 from utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -55,6 +61,8 @@ def __init__(self, janeway_journal_code: str, article_id: int | None) -> None:
         self.article: Article | None = None
         self.journal: Journal | None = None
         self.export_folder: str | None = None
+        self.xml_filepath: str | None = None
+        self.__temp_folder: str | None = None
 
         # Gets the journal
         self.journal: Journal | None = self.__fetch_journal(janeway_journal_code)
@@ -74,6 +82,9 @@ def __init__(self, janeway_journal_code: str, article_id: int | None) -> None:
             return
         self.export_folder = export_folders
 
+        # Creates or fetches a report to track where this process is.
+        self.transfer_report = get_or_create_transfer_report(self.journal, self.article)
+
         # Start export process
         self.__create_export_file()
 
@@ -108,12 +119,17 @@ def __create_export_file(self):
             self.in_error_state = True
             return
 
-        # TODO: Attempt to create the metadata file.
-        # metadata_file: File | None = self.__create_metadata_file(self.article)
-        # if metadata_file is None:
-        #     logger.error(logger_messages.process_failed_fetching_metadata(self.article_id))
-        #     self.in_error_state = True
-        #     return
+        prefix: str = "{0}_{1}".format(self.get_submission_partner_code(), uuid.uuid4())
+
+        self.zip_filepath: str = os.path.join(self.export_folder, "{0}.zip".format(prefix))
+        self.__temp_folder = os.path.join(self.export_folder, "{0}".format(prefix))
+        os.makedirs(self.__temp_folder, exist_ok=True)
+
+        # Attempt to get the metadata file.
+        if self.__get_xml_filepath() is None:
+            logger.error(logger_messages.process_failed_fetching_metadata(self.article_id))
+            self.in_error_state = True
+            return
 
         # Attempt to fetch the article files.
         article_files: Sequence[File] = self.__fetch_article_files(self.article)
@@ -122,28 +138,26 @@ def __create_export_file(self):
             self.in_error_state = True
             return
 
-        prefix: str = "{0}_{1}".format(self.get_submission_partner_code(), uuid.uuid4())
+        filenames: List[str] = []
 
-        self.zip_filepath: str = os.path.join(self.export_folder, "{0}.zip".format(prefix))
-        with zipfile.ZipFile(self.zip_filepath, "w") as zipf:
-            # TODO: zipf.write(metadata_file.get_file_path(self.article))
-            for article_file in article_files:
-                zipf.write(article_file.get_file_path(self.article))
-            filenames: Sequence[str] = zipf.namelist()
-            zipf.close()
+        # Move files to temp folder.
+        for article_file in article_files:
+            filepath: str = article_file.get_file_path(self.article)
+            copy_files_to_temp_deposit_folder(filepath, self.__temp_folder)
+            filenames.append(os.path.basename(filepath))
 
-        # Remove the manuscript
+        deposit_helpers.zip_temp_folder(temp_folder=self.__temp_folder)
 
-        # TODO: Remove below and replace with 'self.__create_go_xml_file(metadata_file.uuid_filename, filenames, prefix)'
-        self.__create_go_xml_file("fake name", filenames, prefix)
+        # Remove the manuscript
+        self.__create_go_xml_file(os.path.basename(self.__get_xml_filepath()), filenames, prefix)
 
     def get_license_code(self) -> str:
         """
         Gets the license code for exporting files.
         :return: The license code or None, if the process failed.
         """
         if not self.__license_code:
-            self.__license_code: str = self.get_setting(consts.PLUGIN_SETTINGS_LICENSE_CODE)
+            self.__license_code: str = get_license_code(self.journal)
         return self.__license_code
 
     def get_journal_code(self) -> str:
@@ -152,7 +166,7 @@ def get_journal_code(self) -> str:
         :return: The journal code or None, if the process failed.
         """
         if not self.__journal_code:
-            self.__journal_code: str = self.get_setting(consts.PLUGIN_SETTINGS_JOURNAL_CODE)
+            self.__journal_code: str = get_journal_code(self.journal)
         return self.__journal_code
 
     def get_submission_partner_code(self) -> str:
@@ -161,7 +175,7 @@ def get_submission_partner_code(self) -> str:
         :return: The submission partner code or None, if the process failed.
         """
         if not self.__submission_partner_code:
-            self.__submission_partner_code: str = self.get_setting(consts.PLUGIN_SETTINGS_SUBMISSION_PARTNER_CODE)
+            self.__submission_partner_code: str = get_submission_partner_code(self.journal)
         return self.__submission_partner_code
 
     def can_export(self) -> bool:
@@ -176,20 +190,6 @@ def can_export(self) -> bool:
                 self.get_journal_code() is not None and
                 self.get_submission_partner_code() is not None)
 
-    def get_setting(self, setting_name: str) -> str:
-        """
-        Gets the setting for the given setting name.
-        :param setting_name: The name of the setting to get the value for.
-        :return: The value for the given setting or a blank string, if the process failed.
-        """
-        try:
-            return setting_handler.get_setting(setting_group_name=consts.PLUGIN_SETTINGS_GROUP_NAME,
-                                               setting_name=setting_name, journal=self.journal, ).processed_value
-        except ObjectDoesNotExist as e:
-            self.log_error("Could not get the following setting, '{0}'".format(setting_name), e)
-            self.in_error_state = True
-            return ""
-
     def __create_go_xml_file(self, metadata_filename: str, article_filenames: Sequence[str], filename: str):
         """
         Creates the go xml file for the export process for Editorial Manager.
@@ -219,8 +219,7 @@ def __create_go_xml_file(self, metadata_filename: str, article_filenames: Sequen
         parameters: ETree.Element = ETree.SubElement(header, consts.GO_FILE_ELEMENT_TAG_PARAMETERS)
         parameter: ETree.Element = ETree.SubElement(parameters, consts.GO_FILE_ELEMENT_TAG_PARAMETER)
         parameter.set(consts.GO_FILE_ATTRIBUTE_ELEMENT_NAME_KEY, consts.GO_FILE_PARAMETER_ELEMENT_NAME_VALUE)
-        parameter.set(consts.GO_FILE_PARAMETER_ELEMENT_VALUE_KEY,
-                      "{0}_{1}".format(self.get_submission_partner_code(), self.get_license_code()))
+        parameter.set(consts.GO_FILE_PARAMETER_ELEMENT_VALUE_KEY, get_xml_license_code(self.journal))
 
         # Begin the filegroup.
         filegroup: ETree.Element = ETree.SubElement(go, consts.GO_FILE_ELEMENT_TAG_FILEGROUP)
@@ -239,13 +238,20 @@ def __create_go_xml_file(self, metadata_filename: str, article_filenames: Sequen
         self.go_filepath = os.path.join(self.export_folder, "{0}.go.xml".format(filename))
         tree.write(self.go_filepath)
 
-    def __create_metadata_file(self, article: Article) -> File | None:
+    def __get_xml_filepath(self) -> str | None:
         """
         Creates the metadata file based on the given article.
-        :param article: The article to convert to JATS.
-        :return:
+        :return:The filepath to the JATS XML file.
         """
-        pass
+        if not self.xml_filepath:
+            filepath = generate_jats_metadata(self.journal, self.article, self.__temp_folder)
+
+            if not filepath:
+                self.in_error_state = True
+                return None
+
+            self.xml_filepath = filepath
+        return self.xml_filepath
 
     def __fetch_article(self, journal: Journal | None, article_id: int | None) -> Article | None:
         """
@@ -254,22 +260,14 @@ def __fetch_article(self, journal: Journal | None, article_id: int | None) -> Ar
         :param article_id: The ID of the article.
         :return: The article object with the given article ID.
         """
-        # If no article ID or journal, return an error.
-        if not article_id or article_id <= 0:
-            self.log_error(logger_messages.process_failed_no_article_id_provided())
-            self.in_error_state = True
-            return None
-
-        article: Article | None = None
-
         logger.debug(logger_messages.process_fetching_article(article_id))
-        try:
-            article = Article.get_article(journal, "id", article_id)
-            logger.debug(logger_messages.process_finished_fetching_article(article_id))
-        except Article.DoesNotExist as e:
-            self.log_error(logger_messages.process_failed_fetching_article(article_id), e)
+
+        article: Article | None = data_fetch.fetch_article(journal, article_id)
+        if article is None:
             self.in_error_state = True
+            return None
 
+        logger.debug(logger_messages.process_finished_fetching_article(article_id))
         return article
 
     def __fetch_journal(self, janeway_journal_code: str | None) -> Journal | None:
@@ -278,43 +276,51 @@ def __fetch_journal(self, janeway_journal_code: str | None) -> Journal | None:
         :param janeway_journal_code: The code of the Janeway journal to fetch.
         :return: The journal object with the given Janeway journal code, if there is one. None otherwise.
         """
-        # If no journal code, return an error.
-        if not janeway_journal_code or len(janeway_journal_code) <= 0:
-            self.log_error(logger_messages.process_failed_no_janeway_journal_code_provided())
-            self.in_error_state = True
-            return None
-
-        journal: Journal | None = None
-
         # Attempt to get the journal.
         logger.debug(logger_messages.process_fetching_journal(janeway_journal_code))
-        try:
-            journal = Journal.objects.get(code=janeway_journal_code)
-            logger.debug(logger_messages.process_finished_fetching_journal(janeway_journal_code))
-        except Journal.DoesNotExist as e:
-            self.log_error(logger_messages.process_failed_fetching_journal(janeway_journal_code), e)
+
+        journal: Journal | None = data_fetch.fetch_journal_data(janeway_journal_code)
+        if journal is None:
             self.in_error_state = True
+            return None
+
+        logger.debug(logger_messages.process_finished_fetching_journal(janeway_journal_code))
 
         return journal
 
-    def log_error(self, message: str, error: Exception = None) -> None:
+    def log_error(self, message: str, error: Exception = None,
+                  stage: ReportState = ReportState.FAILED_BUNDLING) -> None:
         """
         Logs the given error message in both the database and plaintext logs.
         :param message: The message to log.
         :param error: The exception, if there is one.
+        :param stage: Specify which stage this transfer is in.
         """
         logger.exception(error)
         logger.error(message)
-        TransferLogs.objects.create(journal=self.journal, article=self.article, message=message,
+        if self.transfer_report.report_state != stage:
+            self.transfer_report.report_state = stage
+            self.transfer_report.save()
+        TransferLogs.objects.create(report=self.transfer_report, journal=self.journal, article=self.article,
+                                    message=message,
                                     message_type=TransferLogMessageType.EXPORT, success=False)
 
-    def log_success(self) -> None:
+    def log_success_go_file(self) -> None:
+        """
+        Logs a success message in both the database and plaintext logs.
+        """
+        TransferLogs.objects.create(report=self.transfer_report, journal=self.journal, article=self.article,
+                                    message=logger_messages.export_go_file_process_succeeded(self.article_id),
+                                    message_type=TransferLogMessageType.EXPORT, success=True)
+
+    def log_success_zip_file(self) -> None:
         """
         Logs a success message in both the database and plaintext logs.
         """
-        TransferLogs.objects.create(journal=self.journal, article=self.article,
-                                    message=logger_messages.export_process_succeeded(self.article_id),
+        TransferLogs.objects.create(report=self.transfer_report, journal=self.journal, article=self.article,
+                                    message=logger_messages.export_zip_file_process_succeeded(self.article_id),
                                     message_type=TransferLogMessageType.EXPORT, success=True)
+        resolve_transfer_report(self.transfer_report)
 
     @staticmethod
     def __fetch_article_files(article: Article) -> List[File]: