Merge branch 'release/2.7.4'

Author: Matthieu Honel

README.md

@@ -52,6 +52,8 @@ Since data is pulled from YWH platform to your server, only regular outbound web
 - v2.7:
     - added synchronization of "fix verification" logs when "Upload status updates" is checked
     - fixed an issue with jira when scope contains special markdown characters
+    - fixed an issue when "Download bug trackers comments" feedback option is activated
+      and bug tracker attachments do not meet platform attachments requirements (unacceptable mime-type, maximum allowed size exceeded)
 - v2.6:
     - added work around bug trackers maximum size allowed for the text of the issues/comments (content put in Markdown file attachment when necessary)
 - v2.5:

docs/user-guide/User-Guide.pdf

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ywh2bt"
-version = "2.7.3"
+version = "2.7.4"
 description = "ywh2bt - YesWeHack to Bug Tracker"
 readme = "README.md"
 authors = ["m.honel <m.honel@yeswehack.com>"]

pyproject.toml

@@ -34,6 +34,7 @@
     YesWeHackConfiguration,
 )
 from ywh2bt.core.exceptions import CoreException
+from ywh2bt.version import __VERSION__
 
 
 class YesWeHackApiClientError(CoreException):
@@ -83,6 +84,9 @@ def _build_raw_client(
                 pat=configuration.pat,
                 verify=configuration.verify,
                 lazy=True,
+                headers={
+                    'User-Agent': f'ywh2bt/{__VERSION__}',
+                },
             )
         except (YesWeHackRawAPiError, requests.RequestException) as e:
             raise YesWeHackApiClientError('Unable to initialize YesWeHack API client') from e

ywh2bt/core/api/yeswehack.py

@@ -34,6 +34,7 @@
 )
 from ywh2bt.core.api.tracker import (
     SendLogsResult,
+    TrackerAttachment,
     TrackerClient,
     TrackerClientError,
     TrackerIssue,
@@ -76,6 +77,7 @@
     SynchronizerStartFetchReportsEvent,
     SynchronizerStartSendReportEvent,
 )
+from ywh2bt.size import sizeof_fmt_si
 
 
 class Synchronizer:
@@ -624,6 +626,7 @@ def _download_comment(
         tracker_issue: TrackerIssue,
         tracker_comment: TrackerIssueComment,
     ) -> None:
+        failed_attachments = {}
         attachments = {}
         for attachment_key, attachment in tracker_comment.attachments.items():
             try:
@@ -633,10 +636,9 @@ def _download_comment(
                     file_content=attachment.content,
                     file_type=attachment.mime_type,
                 )
-            except YesWeHackApiClientError as upload_attachment_error:
-                raise SynchronizerError(
-                    f'Unable to upload attachment {attachment.filename} for report #{self._report.report_id}',
-                ) from upload_attachment_error
+            except YesWeHackApiClientError:
+                failed_attachments[attachment_key] = attachment
+                continue
             attachments[attachment_key] = uploaded_attachment
         try:
             self._yeswehack_client.post_report_tracker_message(
@@ -647,6 +649,7 @@ def _download_comment(
                 comment=self._message_formatter.format_download_comment(
                     comment=tracker_comment,
                     attachments=attachments,
+                    failed_attachments=failed_attachments,
                 ),
                 attachments=[uploaded_attachment.name for attachment_key, uploaded_attachment in attachments.items()],
             )
@@ -783,13 +786,15 @@ def format_download_comment(
         self,
         comment: TrackerIssueComment,
         attachments: Dict[str, Attachment],
+        failed_attachments: Dict[str, TrackerAttachment],
     ) -> str:
         """
         Format a downloaded comment.
 
         Args:
             comment: a comment
             attachments: a dict of attachments
+            failed_attachments: a dict of tracker attachments that failed to be uploaded
         """
 
     @abstractmethod
@@ -835,6 +840,15 @@ class SynchronizerMessageFormatter(AbstractSynchronizerMessageFormatter):
         + '\n\n'
         + '${comment}',
     )
+    _failed_attachments_template: Template = Template(
+        '**YWH2BT note:**'
+        + '\n*There was an issue while uploading the following attachments from the bugtracker*:'
+        + '\n\n'
+        + '${attachments}',
+    )
+    _failed_attachment_template: Template = Template(
+        '- ${filename} (size=${size}, mime=${mimetype})',
+    )
     _status_update_comment_template: Template = Template(
         '${comment}',
     )
@@ -909,25 +923,42 @@ def format_download_comment(
         self,
         comment: TrackerIssueComment,
         attachments: Dict[str, Attachment],
+        failed_attachments: Dict[str, TrackerAttachment],
     ) -> str:
         """
         Format a downloaded comment.
 
         Args:
             comment: a comment
             attachments: a dict of attachments
+            failed_attachments: a dict of tracker attachments that failed to be uploaded
 
         Returns:
             a formatted comment
         """
-        return self._download_comment_template.substitute(
+        formatted_comment = self._download_comment_template.substitute(
             date=comment.created_at,
             author=comment.author,
             comment=markdown_to_ywh(
                 message=comment.body,
                 attachments=attachments,
             ),
         )
+        if failed_attachments:
+            formatted_attachments = []
+            for _attachment_key, failed_attachment in failed_attachments.items():
+                formatted_attachments.append(
+                    self._failed_attachment_template.substitute(
+                        filename=failed_attachment.filename,
+                        mimetype=failed_attachment.mime_type,
+                        size=sizeof_fmt_si(len(failed_attachment.content), precision=2),
+                    ),
+                )
+            formatted_failed_attachments = self._failed_attachments_template.substitute(
+                attachments='\n'.join(formatted_attachments),
+            )
+            formatted_comment = f'{formatted_comment}\n\n{formatted_failed_attachments}'
+        return formatted_comment
 
     def format_status_update_comment(
         self,

ywh2bt/core/synchronizer/synchronizer.py

@@ -0,0 +1,41 @@
+from typing import (
+    List,
+    Union,
+)
+
+
+def _sizeof_fmt(
+    num: Union[int, float],
+    base: int,
+    units: List[str],
+    max_unit: str,
+    precision: int = 1,
+    suffix: str = 'B'
+) -> str:
+    for unit in units:
+        if abs(num) < base:
+            return f'{num:.{precision}f}{unit}{suffix}'
+        num /= base
+    return f'{num:.{precision}f}{max_unit}{suffix}'
+
+
+def sizeof_fmt_iec(num: Union[int, float], precision: int = 1, suffix: str = 'B') -> str:
+    return _sizeof_fmt(
+        num,
+        base=1024,
+        units=['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi'],
+        max_unit='Yi',
+        precision=precision,
+        suffix=suffix,
+    )
+
+
+def sizeof_fmt_si(num: Union[int, float], precision: int = 1, suffix: str = 'B') -> str:
+    return _sizeof_fmt(
+        num,
+        base=1000,
+        units=['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z'],
+        max_unit='Y',
+        precision=precision,
+        suffix=suffix,
+    )

ywh2bt/size.py

@@ -36,6 +36,7 @@
 )
 from ywh2bt.core.api.tracker import (
     SendLogsResult,
+    TrackerAttachment,
     TrackerClient,
     TrackerIssue,
     TrackerIssueComment,
@@ -669,6 +670,7 @@ def format_download_comment(
         self,
         comment: TrackerIssueComment,
         attachments: Dict[str, Attachment],
+        failed_attachments: Dict[str, TrackerAttachment],
     ) -> str:
         return self.download_comment_format.format(
             author=comment.author,

ywh2bt/tests/core/synchronizer/test_synchronizer.py

@@ -0,0 +1,52 @@
+import unittest
+from typing import Union
+
+from ywh2bt.size import sizeof_fmt_si, sizeof_fmt_iec
+
+
+class TestCli(unittest.TestCase):
+    def _test_sizeof_fmt_si(self, expected: str, num: Union[int, float], precision: int) -> None:
+        with self.subTest(f'sizeof_fmt_si({num}, precision={precision})'):
+            self.assertEqual(expected, sizeof_fmt_si(num, precision=precision))
+
+    def _test_sizeof_fmt_iec(self, expected: str, num: Union[int, float], precision: int) -> None:
+        with self.subTest(f'sizeof_fmt_iec({num}, precision={precision})'):
+            self.assertEqual(expected, sizeof_fmt_iec(num, precision=precision))
+
+    def test_sizeof_fmt(self) -> None:
+        values = {
+            999: {
+                0: ['999B', '999B'],
+                1: ['999.0B', '999.0B'],
+                2: ['999.00B', '999.00B'],
+            },
+            1000: {
+                0: ['1kB', '1000B'],
+                1: ['1.0kB', '1000.0B'],
+                2: ['1.00kB', '1000.00B'],
+            },
+            1024: {
+                0: ['1kB', '1KiB'],
+                1: ['1.0kB', '1.0KiB'],
+                2: ['1.02kB', '1.00KiB'],
+            },
+            2000000: {
+                0: ['2MB', '2MiB'],
+                1: ['2.0MB', '1.9MiB'],
+                2: ['2.00MB', '1.91MiB'],
+            },
+            123456897101112: {
+                0: ['123TB', '112TiB'],
+                1: ['123.5TB', '112.3TiB'],
+                2: ['123.46TB', '112.28TiB'],
+            },
+            9999999999999999999999999: {
+                0: ['10YB', '8YiB'],
+                1: ['10.0YB', '8.3YiB'],
+                2: ['10.00YB', '8.27YiB'],
+            },
+        }
+        for num, precisions in values.items():
+            for precision, expected_values in precisions.items():
+                self._test_sizeof_fmt_si(expected_values[0], num=num, precision=precision)
+                self._test_sizeof_fmt_iec(expected_values[1], num=num, precision=precision)