From 4d348b578bb7b89263c688d4081f1d48786ea2e8 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 27 Nov 2020 10:34:24 +0100 Subject: [PATCH 01/77] Add edi_storage --- edi_storage_oca/README.rst | 105 ++++ edi_storage_oca/__init__.py | 2 + edi_storage_oca/__manifest__.py | 17 + edi_storage_oca/components/__init__.py | 3 + edi_storage_oca/components/base.py | 54 +++ edi_storage_oca/components/check.py | 83 ++++ edi_storage_oca/components/send.py | 39 ++ edi_storage_oca/demo/edi_backend_demo.xml | 14 + edi_storage_oca/models/__init__.py | 1 + edi_storage_oca/models/edi_backend.py | 67 +++ edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + edi_storage_oca/readme/DESCRIPTION.rst | 23 + edi_storage_oca/readme/USAGE.rst | 1 + edi_storage_oca/security/ir_model_access.xml | 12 + edi_storage_oca/static/description/icon.png | Bin 0 -> 9455 bytes edi_storage_oca/static/description/index.html | 450 ++++++++++++++++++ .../tests/_TODO_test_components_base.py | 43 ++ edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 116 +++++ .../tests/test_edi_backend_storage.py | 204 ++++++++ edi_storage_oca/views/edi_backend_views.xml | 21 + 21 files changed, 1257 insertions(+) create mode 100644 edi_storage_oca/README.rst create mode 100644 edi_storage_oca/__init__.py create mode 100644 edi_storage_oca/__manifest__.py create mode 100644 edi_storage_oca/components/__init__.py create mode 100644 edi_storage_oca/components/base.py create mode 100644 edi_storage_oca/components/check.py create mode 100644 edi_storage_oca/components/send.py create mode 100644 edi_storage_oca/demo/edi_backend_demo.xml create mode 100644 edi_storage_oca/models/__init__.py create mode 100644 edi_storage_oca/models/edi_backend.py create mode 100644 edi_storage_oca/readme/CONTRIBUTORS.rst create mode 100644 edi_storage_oca/readme/DESCRIPTION.rst create mode 100644 edi_storage_oca/readme/USAGE.rst create mode 100644 edi_storage_oca/security/ir_model_access.xml create mode 100644 edi_storage_oca/static/description/icon.png create mode 100644 edi_storage_oca/static/description/index.html create mode 100644 edi_storage_oca/tests/_TODO_test_components_base.py create mode 100644 edi_storage_oca/tests/__init__.py create mode 100644 edi_storage_oca/tests/common.py create mode 100644 edi_storage_oca/tests/test_edi_backend_storage.py create mode 100644 edi_storage_oca/views/edi_backend_views.xml diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst new file mode 100644 index 000000000..fffe95837 --- /dev/null +++ b/edi_storage_oca/README.rst @@ -0,0 +1,105 @@ +=========================== +EDI Storage backend support +=========================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github + :target: https://github.com/OCA/edi/tree/13.0/edi_storage + :alt: OCA/edi +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/edi-13-0/edi-13-0-edi_storage + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/226/13.0 + :alt: Try me on Runbot + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* ACSONE + +Contributors +~~~~~~~~~~~~ + +* Simone Orsi + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/edi `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__init__.py b/edi_storage_oca/__init__.py new file mode 100644 index 000000000..f24d3e242 --- /dev/null +++ b/edi_storage_oca/__init__.py @@ -0,0 +1,2 @@ +from . import components +from . import models diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py new file mode 100644 index 000000000..9d4ff373f --- /dev/null +++ b/edi_storage_oca/__manifest__.py @@ -0,0 +1,17 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "EDI Storage backend support", + "summary": """ + Base module to allow exchanging files via storage backend (eg: SFTP). + """, + "version": "13.0.1.0.0", + "development_status": "Alpha", + "license": "AGPL-3", + "author": "ACSONE,Odoo Community Association (OCA)", + "depends": ["edi", "storage_backend", "component"], + "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], + "demo": ["demo/edi_backend_demo.xml"], +} diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py new file mode 100644 index 000000000..9f64b792b --- /dev/null +++ b/edi_storage_oca/components/__init__.py @@ -0,0 +1,3 @@ +from . import base +from . import check +from . import send diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py new file mode 100644 index 000000000..2a337fed1 --- /dev/null +++ b/edi_storage_oca/components/base.py @@ -0,0 +1,54 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from pathlib import PurePath + +from odoo.addons.component.core import AbstractComponent + + +class EDIStorageSendComponentMixin(AbstractComponent): + + _name = "edi.storage.component.mixin" + + @property + def storage(self): + return self.backend.storage_id + + def _dir_by_state(self, direction, state): + """Return remote directory path by direction and state. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :return: PurePath object + """ + assert direction in ("input", "output") + assert state in ("pending", "done", "error") + return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) + + def _remote_file_path(self, direction, state, filename): + """Return remote file path by direction and state for give filename. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :param filename: string for file name + :return: PurePath object + """ + return self._dir_by_state(direction, state) / filename.strip("/ ") + + def _get_remote_file(self, state, filename=None, binary=False): + """Get file for current exchange_record in the given destination state. + + :param state: string ("pending", "done", "error") + :param filename: custom file name, exchange_record filename used by default + :return: remote file content as string + """ + filename = filename or self.exchange_record.exchange_filename + path = self._remote_file_path(self.exchange_record.direction, state, filename) + try: + # TODO: support match via pattern (eg: filename-prefix-*) + # otherwise is impossible to retrieve input files and acks + # (the date will never match) + return self.storage.get(path.as_posix(), binary=binary) + except FileNotFoundError: + return None diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py new file mode 100644 index 000000000..e76ec75aa --- /dev/null +++ b/edi_storage_oca/components/check.py @@ -0,0 +1,83 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import logging + +from odoo.tools import pycompat + +from odoo.addons.component.core import Component + +_logger = logging.getLogger(__name__) + + +class EDIStorageCheckComponentMixin(Component): + + _name = "edi.storage.component.check" + _inherit = [ + "edi.component.check.mixin", + "edi.storage.component.mixin", + ] + _usage = "edi.storage.check" + + def check(self): + return self._exchange_output_check() + + def _exchange_output_check(self): + """Check status output exchange and update record. + + 1. check if the file has been processed already (done) + 2. if yes, post message and exit + 3. if not, check for errors + 4. if no errors, return + + :return: boolean + * False if there's nothing else to be done + * True if file still need action + """ + if self._get_remote_file("done"): + _logger.info( + "%s done for: %s", + self.exchange_record.model, + self.exchange_record.name, + ) + if ( + not self.exchange_record.edi_exchange_state + == "output_sent_and_processed" + ): + self.exchange_record.edi_exchange_state = "output_sent_and_processed" + self.backend._notify_done(self.exchange_record) + if self.exchange_record.type_id.ack_needed: + self._exchange_output_handle_ack() + return False + + error = self._get_remote_file("error") + if error: + _logger.info( + "%s error for: %s", + self.exchange_record.model, + self.exchange_record.name, + ) + # Assume a text file will be placed there w/ the same name and error suffix + err_filename = self.exchange_record.exchange_filename + ".error" + error_report = self._get_remote_file("error", filename=err_filename) + if self.exchange_record.edi_exchange_state == "output_sent": + self.exchange_record.update( + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": pycompat.to_text(error_report), + } + ) + self.backend._notify_error(self.exchange_record, "process_ko") + return False + return True + + def _exchange_output_handle_ack(self): + ack_file = self._get_remote_file( + "done", filename=self.exchange_record.ack_filename + ) + if ack_file: + self.exchange_record._set_file_content(ack_file, field_name="ack_file") + self.backend._notify_ack_received(self.exchange_record) + else: + self.backend._notify_ack_missing(self.exchange_record) diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py new file mode 100644 index 000000000..4d3f37675 --- /dev/null +++ b/edi_storage_oca/components/send.py @@ -0,0 +1,39 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component + + +class EDIStorageSendComponent(Component): + + _name = "edi.storage.component.send" + _inherit = [ + "edi.component.send.mixin", + "edi.storage.component.mixin", + ] + _usage = "edi.storage.send" + + def send(self): + # If the file has been sent already, refresh its state + # TODO: double check if this is useless + # since the backend checks the state already + checker = self.component(usage="edi.storage.check") + result = checker.check() + if not result: + # all good here + return True + + direction = self.exchange_record.direction + filename = self.exchange_record.exchange_filename + filedata = self.exchange_record.exchange_file + path = self._remote_file_path(direction, "pending", filename) + self.storage.add(path.as_posix(), filedata, binary=False) + # TODO: delegate this to generic storage backend + # except paramiko.ssh_exception.AuthenticationException: + # # TODO this exc handling should be moved to sftp backend IMO + # error = _("Authentication error") + # state = "error_on_send" + # TODO: catch other specific exceptions + # this will swallow all the exceptions! + return True diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml new file mode 100644 index 000000000..d8ff5d972 --- /dev/null +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -0,0 +1,14 @@ + + + + Storage Demo EDI backend + + + demo_in/pending + demo_in/done + demo_in/error + demo_out/pending + demo_out/done + demo_out/error + + diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py new file mode 100644 index 000000000..4b2c54ac3 --- /dev/null +++ b/edi_storage_oca/models/__init__.py @@ -0,0 +1 @@ +from . import edi_backend diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py new file mode 100644 index 000000000..5ba933639 --- /dev/null +++ b/edi_storage_oca/models/edi_backend.py @@ -0,0 +1,67 @@ +# Copyright 2020 ACSONE SA +# @author Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + + +class EDIBackend(models.Model): + + _inherit = "edi.backend" + + storage_id = fields.Many2one( + string="Storage backend", + comodel_name="storage.backend", + help="Storage for in-out files", + ondelete="restrict", + ) + """ + We assume the exchanges happen it 2 ways (input, output) + and we have a hierarchy of directory like: + + from_A_to_B + |- pending + |- done + |- error + from_B_to_A + |- pending + |- done + |- error + + where A and B are the partners exchanging data and they are in turn + sender and receiver and vice versa. + """ + # TODO: these paths should probably be by type instead + # Here we can maybe set a common root folder for this exchange. + input_dir_pending = fields.Char( + "Input pending directory", help="Path to folder for pending operations" + ) + input_dir_done = fields.Char( + "Input done directory", help="Path to folder for doneful operations" + ) + input_dir_error = fields.Char( + "Input error directory", help="Path to folder for error operations" + ) + output_dir_pending = fields.Char( + "Output pending directory", help="Path to folder for pending operations" + ) + output_dir_done = fields.Char( + "Output done directory", help="Path to folder for doneful operations" + ) + output_dir_error = fields.Char( + "Output error directory", help="Path to folder for error operations" + ) + + def _get_component_usage_candidates(self, exchange_record, key): + candidates = super()._get_component_usage_candidates(exchange_record, key) + if not self.storage_id: + return candidates + storage_generic = "edi.storage.{}".format(key) + storage_by_backend_type = storage_generic + "." + self.storage_id.backend_type + type_code = exchange_record.type_id.code + return [ + storage_by_backend_type + "." + type_code, + storage_by_backend_type, + storage_generic + "." + exchange_record.direction, + storage_generic, + ] + candidates diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst new file mode 100644 index 000000000..f583948be --- /dev/null +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -0,0 +1 @@ +* Simone Orsi diff --git a/edi_storage_oca/readme/DESCRIPTION.rst b/edi_storage_oca/readme/DESCRIPTION.rst new file mode 100644 index 000000000..bdef14a8a --- /dev/null +++ b/edi_storage_oca/readme/DESCRIPTION.rst @@ -0,0 +1,23 @@ +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. diff --git a/edi_storage_oca/readme/USAGE.rst b/edi_storage_oca/readme/USAGE.rst new file mode 100644 index 000000000..a47c64c84 --- /dev/null +++ b/edi_storage_oca/readme/USAGE.rst @@ -0,0 +1 @@ +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. diff --git a/edi_storage_oca/security/ir_model_access.xml b/edi_storage_oca/security/ir_model_access.xml new file mode 100644 index 000000000..b7d7b8498 --- /dev/null +++ b/edi_storage_oca/security/ir_model_access.xml @@ -0,0 +1,12 @@ + + + + access_storage_backend EDI manager + + + + + + + + diff --git a/edi_storage_oca/static/description/icon.png b/edi_storage_oca/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html new file mode 100644 index 000000000..70dda24be --- /dev/null +++ b/edi_storage_oca/static/description/index.html @@ -0,0 +1,450 @@ + + + + + + +EDI Storage backend support + + + +
+

EDI Storage backend support

+ + +

Alpha License: AGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Allow exchange files using storage backends from OCA/storage.

+

This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files.

+

Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files).

+

Inside this folder you have this hierarchy:

+
+input/output folder
+    |- pending
+    |- done
+    |- error
+
+
    +
  • pending folder contains files that have been just sent
    • +
  • done folder contains files that have been processes successfully
    • +
  • error folder contains files with errors and cannot be processed
    • +
+

The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly.

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Usage

+

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/edi project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/edi_storage_oca/tests/_TODO_test_components_base.py b/edi_storage_oca/tests/_TODO_test_components_base.py new file mode 100644 index 000000000..ede7d946e --- /dev/null +++ b/edi_storage_oca/tests/_TODO_test_components_base.py @@ -0,0 +1,43 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from freezegun import freeze_time + +from .common import EDIBackendCommonTestCase + + +class EDIBackendTestCase(EDIBackendCommonTestCase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + path_obj = self.backend._remote_file_path(*_args) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.backend._remote_file_path("WHATEVER", "error", "foo.csv") + + with self.assertRaises(AssertionError): + self.backend._remote_file_path("input", "WHATEVER", "foo.csv") + + @freeze_time("2020-10-21 10:00:00") + def test_create_record(self): + vals = { + "model": self.partner._name, + "res_id": self.partner.id, + } + record = self.backend.create_record("test_csv_input", vals) + expected = { + "type_id": self.exchange_type_in.id, + "record_id": self.partner, + "edi_exchange_state": "new", + "exchange_filename": "EDI_EXC_TEST-test_csv_input-2020-10-21-10-00-00.csv", + } + self.assertRecordValues(record, [expected]) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py new file mode 100644 index 000000000..d81ae344d --- /dev/null +++ b/edi_storage_oca/tests/__init__.py @@ -0,0 +1 @@ +from . import test_edi_backend_storage diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py new file mode 100644 index 000000000..2d02206ab --- /dev/null +++ b/edi_storage_oca/tests/common.py @@ -0,0 +1,116 @@ +# Copyright 2020 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 +import functools + +import mock + +from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase + + +class TestEDIStorageBase(EDIBackendCommonComponentTestCase): + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage.demo_edi_backend_storage") + + @classmethod + def _setup_records(cls): + super()._setup_records() + cls.filedata = base64.b64encode(b"This is a simple file") + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": cls.filedata, + } + cls.record = cls.backend.create_record("test_csv_output", vals) + + cls.fakepath = "/tmp/{}".format(cls._filename(cls)) + with open(cls.fakepath, "w+b") as fakefile: + fakefile.write(b"filecontent") + + cls.fakepath_ack = "/tmp/{}.ack".format(cls._filename(cls)) + with open(cls.fakepath_ack, "w+b") as fakefile: + fakefile.write(b"ACK filecontent") + + cls.fakepath_error = "/tmp/{}.error".format(cls._filename(cls)) + with open(cls.fakepath_error, "w+b") as fakefile: + fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + + cls.checker = cls.backend._get_component( + ["edi.storage.check"], work_ctx={"exchange_record": cls.record} + ) + cls.sender = cls.backend._get_component( + ["edi.storage.send"], work_ctx={"exchange_record": cls.record} + ) + + def setUp(self): + super().setUp() + self._storage_backend_calls = [] + + def _filename(self, record=None, ack=False): + record = record or self.record + return record.exchange_filename if not ack else record.ack_filename + + def _file_fullpath(self, state, record=None, ack=False): + record = record or self.record + fname = self._filename(record, ack=ack) + if state == "error-report": + # Exception as we read from the same path but w/ error suffix + state = "error" + fname += ".error" + return ( + self.checker._remote_file_path(record.direction, state, fname) + ).as_posix() + + def _mocked_backend_get(self, mocked_paths, path, **kwargs): + self._storage_backend_calls.append(path) + if mocked_paths.get(path): + with open(mocked_paths.get(path), "rb") as remote_file: + return remote_file.read() + raise FileNotFoundError() + + def _mocked_backend_add(self, path, data, **kwargs): + self._storage_backend_calls.append(path) + + def _mock_storage_backend_get(self, mocked_paths): + mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" + mocked = functools.partial(self._mocked_backend_get, mocked_paths) + return mock.patch(mock_path + ".get", mocked) + + def _mock_storage_backend_add(self): + mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" + return mock.patch(mock_path + ".add", self._mocked_backend_add) + + def _test_result( + self, record, expected_values, expected_messages=None, state_paths=None, + ): + state_paths = state_paths or ("done", "pending", "error") + # Paths will be something like: + # [ + # 'demo_out/pending/$filename.csv', + # 'demo_out/pending/$filename.csv', + # 'demo_out/error/$filename.csv', + # ] + for state in state_paths: + path = self._file_fullpath(state, record=record) + self.assertIn(path, self._storage_backend_calls) + self.assertRecordValues(record, [expected_values]) + if expected_messages: + self.assertEqual(len(record.record.message_ids), len(expected_messages)) + for msg_rec, expected in zip(record.record.message_ids, expected_messages): + self.assertIn(expected["message"], msg_rec.body) + self.assertIn("level-" + expected["level"], msg_rec.body) + # TODO: test content of file sent + + def _test_send(self, record, mocked_paths=None): + with self._mock_storage_backend_add(): + if mocked_paths: + with self._mock_storage_backend_get(mocked_paths): + self.backend.exchange_send(record) + else: + self.backend.exchange_send(record) + + def _test_run_cron(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_get(mocked_paths): + self.backend._cron_check_output_exchange_sync() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py new file mode 100644 index 000000000..de240c955 --- /dev/null +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -0,0 +1,204 @@ +# Copyright 2020 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 + +from freezegun import freeze_time + +from odoo.tools import mute_logger + +from .common import TestEDIStorageBase + +LOGGERS = ( + "odoo.addons.edi_storage.components.check", + "odoo.addons.edi.models.edi_backend", +) + + +@freeze_time("2020-10-21 10:30:00") +class TestEDIBackendOutput(TestEDIStorageBase): + @mute_logger(*LOGGERS) + def test_export_file_sent(self): + """Send, no errors.""" + self.record.edi_exchange_state = "output_pending" + mocked_paths = {self._file_fullpath("pending"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + # self._test_send(self.record, mocked_paths=mocked_paths) + self._test_run_cron(mocked_paths) + self._test_result( + self.record, + {"edi_exchange_state": "output_sent"}, + expected_messages=[ + { + "message": self.record._exchange_status_message("send_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done(self): + """Already sent, successfully.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = {self._file_fullpath("done"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `done` folder, + # we should get the final good state + # and only one call to ftp + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed", "ack_file": False}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done_ack_needed_not_found(self): + self.record.edi_exchange_state = "output_sent" + self.record.type_id.ack_needed = True + mocked_paths = { + self._file_fullpath("done"): self.fakepath, + } + self._test_run_cron(mocked_paths) + # No ack file found, warning message is posted + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("ack_missing"), + "level": "warning", + }, + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + }, + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done_ack_needed_found(self): + self.record.edi_exchange_state = "output_sent" + self.record.type_id.ack_needed = True + mocked_paths = { + self._file_fullpath("done"): self.fakepath, + self._file_fullpath("done", ack=True): self.fakepath_ack, + } + self._test_run_cron(mocked_paths) + # Found ack file, set on record + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_processed", + "ack_file": base64.b64encode(b"ACK filecontent"), + }, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("ack_received"), + "level": "info", + }, + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + }, + ], + ) + + @mute_logger(*LOGGERS) + def test_already_sent_process_error(self): + """Already sent, error process.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = { + self._file_fullpath("error"): self.fakepath, + self._file_fullpath("error-report"): self.fakepath_error, + } + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `error` folder, + # we should get a call for: done, error and then the read of the report. + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_cron_full_flow(self): + """Already sent, update the state via cron.""" + self.record.edi_exchange_state = "output_sent" + rec1 = self.record + partner2 = self.env.ref("base.res_partner_2") + partner3 = self.env.ref("base.res_partner_3") + rec2 = self.record.copy( + { + "model": partner2._name, + "res_id": partner2.id, + "exchange_filename": "rec2.csv", + } + ) + rec3 = self.record.copy( + { + "model": partner3._name, + "res_id": partner3.id, + "exchange_filename": "rec3.csv", + "edi_exchange_state": "output_sent_and_error", + } + ) + mocked_paths = { + self._file_fullpath("done", record=rec1): self.fakepath, + self._file_fullpath("error", record=rec2): self.fakepath, + self._file_fullpath("error-report", record=rec2): self.fakepath_error, + self._file_fullpath("done", record=rec3): self.fakepath, + } + self._test_run_cron(mocked_paths) + self._test_result( + rec1, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec1._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + self._test_result( + rec2, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": rec2._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + self._test_result( + rec3, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec3._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml new file mode 100644 index 000000000..5c7916eb1 --- /dev/null +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -0,0 +1,21 @@ + + + + edi.backend + + + + + + + + + + + + + + + + + From cb14b9057cf2a50fab4e55394a076fab9200ce59 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 27 Nov 2020 13:50:23 +0100 Subject: [PATCH 02/77] edi_storage: add base tests for component --- .../tests/_TODO_test_components_base.py | 43 ------------------- edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 10 +++-- edi_storage_oca/tests/test_components_base.py | 43 +++++++++++++++++++ 4 files changed, 50 insertions(+), 47 deletions(-) delete mode 100644 edi_storage_oca/tests/_TODO_test_components_base.py create mode 100644 edi_storage_oca/tests/test_components_base.py diff --git a/edi_storage_oca/tests/_TODO_test_components_base.py b/edi_storage_oca/tests/_TODO_test_components_base.py deleted file mode 100644 index ede7d946e..000000000 --- a/edi_storage_oca/tests/_TODO_test_components_base.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020 ACSONE -# @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). - -from freezegun import freeze_time - -from .common import EDIBackendCommonTestCase - - -class EDIBackendTestCase(EDIBackendCommonTestCase): - def test_remote_file_path(self): - to_test = ( - (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), - (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), - (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), - (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), - (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), - (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), - ) - for _args, expected in to_test: - path_obj = self.backend._remote_file_path(*_args) - self.assertEqual(path_obj.as_posix(), expected) - - with self.assertRaises(AssertionError): - self.backend._remote_file_path("WHATEVER", "error", "foo.csv") - - with self.assertRaises(AssertionError): - self.backend._remote_file_path("input", "WHATEVER", "foo.csv") - - @freeze_time("2020-10-21 10:00:00") - def test_create_record(self): - vals = { - "model": self.partner._name, - "res_id": self.partner.id, - } - record = self.backend.create_record("test_csv_input", vals) - expected = { - "type_id": self.exchange_type_in.id, - "record_id": self.partner, - "edi_exchange_state": "new", - "exchange_filename": "EDI_EXC_TEST-test_csv_input-2020-10-21-10-00-00.csv", - } - self.assertRecordValues(record, [expected]) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index d81ae344d..b788a0071 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1 +1,2 @@ from . import test_edi_backend_storage +from . import test_components_base diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 2d02206ab..a6af8cb00 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -7,6 +7,10 @@ from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase +STORAGE_BACKEND_MOCK_PATH = ( + "odoo.addons.storage_backend.models.storage_backend.StorageBackend" +) + class TestEDIStorageBase(EDIBackendCommonComponentTestCase): @classmethod @@ -73,13 +77,11 @@ def _mocked_backend_add(self, path, data, **kwargs): self._storage_backend_calls.append(path) def _mock_storage_backend_get(self, mocked_paths): - mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" mocked = functools.partial(self._mocked_backend_get, mocked_paths) - return mock.patch(mock_path + ".get", mocked) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get", mocked) def _mock_storage_backend_add(self): - mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" - return mock.patch(mock_path + ".add", self._mocked_backend_add) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) def _test_result( self, record, expected_values, expected_messages=None, state_paths=None, diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py new file mode 100644 index 000000000..c2ba7275f --- /dev/null +++ b/edi_storage_oca/tests/test_components_base.py @@ -0,0 +1,43 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import mock + +from .common import STORAGE_BACKEND_MOCK_PATH, TestEDIStorageBase + + +class EDIBackendTestCase(TestEDIStorageBase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + path_obj = self.checker._remote_file_path(*_args) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.checker._remote_file_path("WHATEVER", "error", "foo.csv") + + with self.assertRaises(AssertionError): + self.checker._remote_file_path("input", "WHATEVER", "foo.csv") + + def test_get_remote_file(self): + with mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get") as mocked: + self.checker._get_remote_file("pending") + mocked.assert_called_with( + "demo_out/pending/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("done") + mocked.assert_called_with( + "demo_out/done/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("error") + mocked.assert_called_with( + "demo_out/error/{}".format(self._filename(self.record)), binary=False + ) From 05a4b25239ad91499ac878bee5832f62ee4e9621 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Sun, 29 Nov 2020 18:58:26 +0100 Subject: [PATCH 03/77] edi_storage: fix messages test isolation --- edi_storage_oca/tests/common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index a6af8cb00..d49b443a5 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -98,8 +98,12 @@ def _test_result( self.assertIn(path, self._storage_backend_calls) self.assertRecordValues(record, [expected_values]) if expected_messages: - self.assertEqual(len(record.record.message_ids), len(expected_messages)) - for msg_rec, expected in zip(record.record.message_ids, expected_messages): + # consider only edi related messages + messages = record.record.message_ids.filtered( + lambda x: "edi-exchange" in x.body + ) + self.assertEqual(len(messages), len(expected_messages)) + for msg_rec, expected in zip(messages, expected_messages): self.assertIn(expected["message"], msg_rec.body) self.assertIn("level-" + expected["level"], msg_rec.body) # TODO: test content of file sent From fcfdad9cb51bcc870267cafd145b6cd74565398b Mon Sep 17 00:00:00 2001 From: oca-travis Date: Mon, 30 Nov 2020 11:50:12 +0000 Subject: [PATCH 04/77] [UPD] Update edi_storage.pot --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/i18n/edi_storage.pot | 77 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 edi_storage_oca/i18n/edi_storage.pot diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 9d4ff373f..0d0500a2f 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.0.0", + "version": "13.0.1.1.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/i18n/edi_storage.pot b/edi_storage_oca/i18n/edi_storage.pot new file mode 100644 index 000000000..d6944e83b --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage.pot @@ -0,0 +1,77 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 13.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage +#: model:ir.model,name:edi_storage.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From 6889af8e2a88f509b24d3b4719ac65b23cb9dcad Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 2 Dec 2020 08:04:25 +0100 Subject: [PATCH 05/77] edi_storage: adapt to edi api changes --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/check.py | 8 ++++---- edi_storage_oca/tests/common.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0d0500a2f..494055c6c 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.1.0", + "version": "13.0.1.2.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index e76ec75aa..835f979b8 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -46,7 +46,7 @@ def _exchange_output_check(self): == "output_sent_and_processed" ): self.exchange_record.edi_exchange_state = "output_sent_and_processed" - self.backend._notify_done(self.exchange_record) + self.exchange_record._notify_done() if self.exchange_record.type_id.ack_needed: self._exchange_output_handle_ack() return False @@ -68,7 +68,7 @@ def _exchange_output_check(self): "exchange_error": pycompat.to_text(error_report), } ) - self.backend._notify_error(self.exchange_record, "process_ko") + self.exchange_record._notify_error("process_ko") return False return True @@ -78,6 +78,6 @@ def _exchange_output_handle_ack(self): ) if ack_file: self.exchange_record._set_file_content(ack_file, field_name="ack_file") - self.backend._notify_ack_received(self.exchange_record) + self.exchange_record._notify_ack_received() else: - self.backend._notify_ack_missing(self.exchange_record) + self.exchange_record._notify_ack_missing() diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d49b443a5..d55fe6774 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -40,10 +40,10 @@ def _setup_records(cls): with open(cls.fakepath_error, "w+b") as fakefile: fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") - cls.checker = cls.backend._get_component( + cls.checker = cls.backend._find_component( ["edi.storage.check"], work_ctx={"exchange_record": cls.record} ) - cls.sender = cls.backend._get_component( + cls.sender = cls.backend._find_component( ["edi.storage.send"], work_ctx={"exchange_record": cls.record} ) From a33e85f0cc9b096e5a8e159afad3a22d28588af6 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 2 Dec 2020 08:38:55 +0100 Subject: [PATCH 06/77] edi_storage: put ack handling on hold Ack files should be treated as incoming records. Receive files is not handled yet by the framework and will be added soon. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/check.py | 21 +++- edi_storage_oca/tests/common.py | 4 +- .../tests/test_edi_backend_storage.py | 109 +++++++++--------- 4 files changed, 73 insertions(+), 63 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 494055c6c..838e6579a 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.2.0", + "version": "13.0.1.3.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 835f979b8..965b01dfe 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -47,8 +47,6 @@ def _exchange_output_check(self): ): self.exchange_record.edi_exchange_state = "output_sent_and_processed" self.exchange_record._notify_done() - if self.exchange_record.type_id.ack_needed: - self._exchange_output_handle_ack() return False error = self._get_remote_file("error") @@ -72,12 +70,23 @@ def _exchange_output_check(self): return False return True + # FIXME: this is not used ATM -> should be refactored + # into an incoming exchange. + # The backend will look for records needing an ack + # and generate and ack record. def _exchange_output_handle_ack(self): - ack_file = self._get_remote_file( - "done", filename=self.exchange_record.ack_filename - ) + ack_type = self.exchange_record.type_id.ack_type_id + filename = ack_type._make_exchange_filename(self.exchange_record) + ack_file = self._get_remote_file("done", filename=filename) if ack_file: - self.exchange_record._set_file_content(ack_file, field_name="ack_file") + self.backend.create_record( + ack_type.code, + { + "parent_id": self.exchange_record.id, + "exchange_file": ack_file, + "edi_exchange_state": "input_received", + }, + ) self.exchange_record._notify_ack_received() else: self.exchange_record._notify_ack_missing() diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d55fe6774..992c4b44f 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -53,7 +53,9 @@ def setUp(self): def _filename(self, record=None, ack=False): record = record or self.record - return record.exchange_filename if not ack else record.ack_filename + if ack: + record.type_id.ack_type_id._make_exchange_filename(record) + return record.exchange_filename def _file_fullpath(self, state, record=None, ack=False): record = record or self.record diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index de240c955..1eda66121 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -1,7 +1,5 @@ # Copyright 2020 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -import base64 - from freezegun import freeze_time from odoo.tools import mute_logger @@ -47,7 +45,7 @@ def test_export_file_already_done(self): # and only one call to ftp self._test_result( self.record, - {"edi_exchange_state": "output_sent_and_processed", "ack_file": False}, + {"edi_exchange_state": "output_sent_and_processed"}, state_paths=("done",), expected_messages=[ { @@ -57,59 +55,60 @@ def test_export_file_already_done(self): ], ) - @mute_logger(*LOGGERS) - def test_export_file_already_done_ack_needed_not_found(self): - self.record.edi_exchange_state = "output_sent" - self.record.type_id.ack_needed = True - mocked_paths = { - self._file_fullpath("done"): self.fakepath, - } - self._test_run_cron(mocked_paths) - # No ack file found, warning message is posted - self._test_result( - self.record, - {"edi_exchange_state": "output_sent_and_processed"}, - state_paths=("done",), - expected_messages=[ - { - "message": self.record._exchange_status_message("ack_missing"), - "level": "warning", - }, - { - "message": self.record._exchange_status_message("process_ok"), - "level": "info", - }, - ], - ) + # FIXME: ack should be handle as an incoming record (new machinery to be added) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_not_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # } + # self._test_run_cron(mocked_paths) + # # No ack file found, warning message is posted + # self._test_result( + # self.record, + # {"edi_exchange_state": "output_sent_and_processed"}, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_missing"), + # "level": "warning", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) - @mute_logger(*LOGGERS) - def test_export_file_already_done_ack_needed_found(self): - self.record.edi_exchange_state = "output_sent" - self.record.type_id.ack_needed = True - mocked_paths = { - self._file_fullpath("done"): self.fakepath, - self._file_fullpath("done", ack=True): self.fakepath_ack, - } - self._test_run_cron(mocked_paths) - # Found ack file, set on record - self._test_result( - self.record, - { - "edi_exchange_state": "output_sent_and_processed", - "ack_file": base64.b64encode(b"ACK filecontent"), - }, - state_paths=("done",), - expected_messages=[ - { - "message": self.record._exchange_status_message("ack_received"), - "level": "info", - }, - { - "message": self.record._exchange_status_message("process_ok"), - "level": "info", - }, - ], - ) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # self._file_fullpath("done", ack=True): self.fakepath_ack, + # } + # self._test_run_cron(mocked_paths) + # # Found ack file, set on record + # self._test_result( + # self.record, + # { + # "edi_exchange_state": "output_sent_and_processed", + # "ack_file": base64.b64encode(b"ACK filecontent"), + # }, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_received"), + # "level": "info", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) @mute_logger(*LOGGERS) def test_already_sent_process_error(self): From d6497ae890a2b87d8a0cacfce597c5ecabd1c2f0 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Mon, 8 Feb 2021 09:47:58 +0100 Subject: [PATCH 07/77] edi_storage: adapt to new components lookup --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/base.py | 14 ++- edi_storage_oca/components/check.py | 2 +- edi_storage_oca/components/send.py | 4 +- edi_storage_oca/models/edi_backend.py | 29 ++++--- edi_storage_oca/tests/common.py | 8 +- edi_storage_oca/tests/test_component_match.py | 86 +++++++++++++++++++ edi_storage_oca/tests/test_components_base.py | 2 +- 8 files changed, 129 insertions(+), 18 deletions(-) create mode 100644 edi_storage_oca/tests/test_component_match.py diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 838e6579a..20db765f1 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.3.0", + "version": "13.0.1.4.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 2a337fed1..c6e67c148 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -7,9 +7,21 @@ from odoo.addons.component.core import AbstractComponent -class EDIStorageSendComponentMixin(AbstractComponent): +class EDIStorageComponentMixin(AbstractComponent): _name = "edi.storage.component.mixin" + _inherit = "edi.component.mixin" + # Components having `_storage_backend_type` will have precedence. + # If the value is not set, generic components will be used. + _storage_backend_type = None + + @classmethod + def _component_match(cls, work, usage=None, model_name=None, **kw): + res = super()._component_match(work, usage=usage, model_name=model_name, **kw) + storage_type = kw.get("storage_backend_type") + if storage_type and cls._storage_backend_type: + return cls._storage_backend_type == storage_type + return res @property def storage(self): diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 965b01dfe..6e3cfa99a 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -18,7 +18,7 @@ class EDIStorageCheckComponentMixin(Component): "edi.component.check.mixin", "edi.storage.component.mixin", ] - _usage = "edi.storage.check" + _usage = "storage.check" def check(self): return self._exchange_output_check() diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index 4d3f37675..b5ec45b5c 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -12,13 +12,13 @@ class EDIStorageSendComponent(Component): "edi.component.send.mixin", "edi.storage.component.mixin", ] - _usage = "edi.storage.send" + _usage = "storage.send" def send(self): # If the file has been sent already, refresh its state # TODO: double check if this is useless # since the backend checks the state already - checker = self.component(usage="edi.storage.check") + checker = self.component(usage="storage.check") result = checker.check() if not result: # all good here diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5ba933639..63aa10249 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -52,16 +52,25 @@ class EDIBackend(models.Model): "Output error directory", help="Path to folder for error operations" ) + _storage_actions = ("check", "send", "receive") + def _get_component_usage_candidates(self, exchange_record, key): candidates = super()._get_component_usage_candidates(exchange_record, key) - if not self.storage_id: + if not self.storage_id or key not in self._storage_actions: return candidates - storage_generic = "edi.storage.{}".format(key) - storage_by_backend_type = storage_generic + "." + self.storage_id.backend_type - type_code = exchange_record.type_id.code - return [ - storage_by_backend_type + "." + type_code, - storage_by_backend_type, - storage_generic + "." + exchange_record.direction, - storage_generic, - ] + candidates + return ["storage.{}".format(key)] + candidates + + def _component_match_attrs(self, exchange_record, key): + # Override to inject storage_backend_type + res = super()._component_match_attrs(exchange_record, key) + if not self.storage_id or key not in self._storage_actions: + return res + res["storage_backend_type"] = self.storage_id.backend_type + return res + + def _component_sort_key(self, component_class): + res = super()._component_sort_key(component_class) + # Override to give precedence by storage_backend_type when needed. + if not self.storage_id: + return res + return (1 if component_class._storage_backend_type else 0,) + res diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 992c4b44f..9cd239422 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -41,10 +41,14 @@ def _setup_records(cls): fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") cls.checker = cls.backend._find_component( - ["edi.storage.check"], work_ctx={"exchange_record": cls.record} + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record}, ) cls.sender = cls.backend._find_component( - ["edi.storage.send"], work_ctx={"exchange_record": cls.record} + cls.partner._name, + ["storage.send"], + work_ctx={"exchange_record": cls.record}, ) def setUp(self): diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py new file mode 100644 index 000000000..4cd1bad87 --- /dev/null +++ b/edi_storage_oca/tests/test_component_match.py @@ -0,0 +1,86 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component +from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage") + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage.demo_edi_backend_storage") + + def test_component_match(self): + """Lookup with special match method.""" + + class SFTPCheck(Component): + _name = "sftp.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class SFTPSend(Component): + _name = "sftp.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class S3Check(Component): + _name = "s3.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + class S3Send(Component): + _name = "s3.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Check._name) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPCheck._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPSend._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Send._name) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index c2ba7275f..27969a605 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -7,7 +7,7 @@ from .common import STORAGE_BACKEND_MOCK_PATH, TestEDIStorageBase -class EDIBackendTestCase(TestEDIStorageBase): +class EDIStorageComponentTestCase(TestEDIStorageBase): def test_remote_file_path(self): to_test = ( (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), From 543db9b251a7e4e7164632394ae5627e06bf6498 Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Wed, 10 Mar 2021 13:56:20 +0100 Subject: [PATCH 08/77] [CHG] edi: Use more permissive licence: AGPL-> LGPL --- edi_storage_oca/README.rst | 8 ++++---- edi_storage_oca/__manifest__.py | 4 ++-- edi_storage_oca/components/base.py | 2 +- edi_storage_oca/components/check.py | 2 +- edi_storage_oca/components/send.py | 2 +- edi_storage_oca/models/edi_backend.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- edi_storage_oca/tests/common.py | 2 +- edi_storage_oca/tests/test_component_match.py | 2 +- edi_storage_oca/tests/test_components_base.py | 2 +- edi_storage_oca/tests/test_edi_backend_storage.py | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index fffe95837..cc387fb2d 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -10,9 +10,9 @@ EDI Storage backend support .. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png :target: https://odoo-community.org/page/development-status :alt: Alpha -.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png - :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html - :alt: License: AGPL-3 +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github :target: https://github.com/OCA/edi/tree/13.0/edi_storage :alt: OCA/edi @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/13.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 20db765f1..503ae5cfe 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). { "name": "EDI Storage backend support", @@ -9,7 +9,7 @@ """, "version": "13.0.1.4.0", "development_status": "Alpha", - "license": "AGPL-3", + "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index c6e67c148..01d0c8b2c 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from pathlib import PurePath diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 6e3cfa99a..878cfffcb 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index b5ec45b5c..cbb6ed309 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 63aa10249..2437a6bc1 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE SA # @author Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo import fields, models diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 70dda24be..b034d89fe 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: AGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 9cd239422..134c807a9 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -1,5 +1,5 @@ # Copyright 2020 ACSONE SA/NV () -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). import base64 import functools diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index 4cd1bad87..e422c9698 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index 27969a605..57abd1385 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import mock diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 1eda66121..f49138fc0 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -1,5 +1,5 @@ # Copyright 2020 ACSONE SA/NV () -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). from freezegun import freeze_time from odoo.tools import mute_logger From 698aee32a3a54a4aed708cf5f28e0521f98a6cd1 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Mon, 15 Mar 2021 21:46:44 +0000 Subject: [PATCH 09/77] [UPD] README.rst --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index cc387fb2d..0cceccfbf 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/13.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 503ae5cfe..0df82391d 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.4.0", + "version": "13.0.1.4.1", "development_status": "Alpha", "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 20f3289a4e7c9367613a5d5a1a41cfcbd9681ec1 Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Wed, 7 Apr 2021 10:35:07 +0200 Subject: [PATCH 10/77] [FIX] edi_storage: sort key was not working properly --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/models/edi_backend.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0df82391d..0243ab8cf 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.4.1", + "version": "13.0.1.5.0", "development_status": "Alpha", "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 2437a6bc1..0ba578fa0 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -73,4 +73,6 @@ def _component_sort_key(self, component_class): # Override to give precedence by storage_backend_type when needed. if not self.storage_id: return res - return (1 if component_class._storage_backend_type else 0,) + res + return ( + 1 if getattr(component_class, "_storage_backend_type", False) else 0, + ) + res From 3316a5c94c642da32875d266259a5d9ec865c95c Mon Sep 17 00:00:00 2001 From: fshah Date: Wed, 5 May 2021 14:43:32 +0530 Subject: [PATCH 11/77] [IMP] edi_storage_oca: isort, black, prettier. --- edi_storage_oca/__manifest__.py | 1 + edi_storage_oca/tests/common.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0243ab8cf..488f60613 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -10,6 +10,7 @@ "version": "13.0.1.5.0", "development_status": "Alpha", "license": "LGPL-3", + "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 134c807a9..c5ed77c45 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -90,7 +90,11 @@ def _mock_storage_backend_add(self): return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) def _test_result( - self, record, expected_values, expected_messages=None, state_paths=None, + self, + record, + expected_values, + expected_messages=None, + state_paths=None, ): state_paths = state_paths or ("done", "pending", "error") # Paths will be something like: From f4a6c7815c3d097ca1810d03217be7d4ff8867a4 Mon Sep 17 00:00:00 2001 From: fshah Date: Wed, 5 May 2021 15:07:19 +0530 Subject: [PATCH 12/77] [MIG] Migrate module edi_storage_oca to v14. --- edi_storage_oca/README.rst | 11 ++++++----- edi_storage_oca/__manifest__.py | 4 ++-- edi_storage_oca/demo/edi_backend_demo.xml | 2 +- edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + edi_storage_oca/static/description/index.html | 6 +++--- edi_storage_oca/tests/common.py | 4 ++-- edi_storage_oca/tests/test_component_match.py | 4 ++-- edi_storage_oca/tests/test_edi_backend_storage.py | 4 ++-- edi_storage_oca/views/edi_backend_views.xml | 2 +- 9 files changed, 20 insertions(+), 18 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 0cceccfbf..14f8cd692 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -14,13 +14,13 @@ EDI Storage backend support :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github - :target: https://github.com/OCA/edi/tree/13.0/edi_storage + :target: https://github.com/OCA/edi/tree/14.0/edi_storage_oca :alt: OCA/edi .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/edi-13-0/edi-13-0-edi_storage + :target: https://translation.odoo-community.org/projects/edi-14-0/edi-14-0-edi_storage_oca :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/13.0 + :target: https://runbot.odoo-community.org/runbot/226/14.0 :alt: Try me on Runbot |badge1| |badge2| |badge3| |badge4| |badge5| @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -86,6 +86,7 @@ Contributors ~~~~~~~~~~~~ * Simone Orsi +* Foram Shah Maintainers ~~~~~~~~~~~ @@ -100,6 +101,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/edi `_ project on GitHub. +This module is part of the `OCA/edi `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 488f60613..b8606ae4c 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,12 +7,12 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.5.0", + "version": "14.0.1.0.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", - "depends": ["edi", "storage_backend", "component"], + "depends": ["edi_oca", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml index d8ff5d972..5b003460f 100644 --- a/edi_storage_oca/demo/edi_backend_demo.xml +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -2,7 +2,7 @@ Storage Demo EDI backend - + demo_in/pending demo_in/done diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst index f583948be..e3fa6d8e9 100644 --- a/edi_storage_oca/readme/CONTRIBUTORS.rst +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -1 +1,2 @@ * Simone Orsi +* Foram Shah diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index b034d89fe..da7347ce6 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -441,7 +441,7 @@

Maintainers

OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

-

This module is part of the OCA/edi project on GitHub.

+

This module is part of the OCA/edi project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index c5ed77c45..4be4c1daf 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -5,7 +5,7 @@ import mock -from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentTestCase STORAGE_BACKEND_MOCK_PATH = ( "odoo.addons.storage_backend.models.storage_backend.StorageBackend" @@ -15,7 +15,7 @@ class TestEDIStorageBase(EDIBackendCommonComponentTestCase): @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage.demo_edi_backend_storage") + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") @classmethod def _setup_records(cls): diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index e422c9698..e4fbfbe82 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -10,11 +10,11 @@ class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): @classmethod def setUpClass(cls): super().setUpClass() - cls._load_module_components(cls, "edi_storage") + cls._load_module_components(cls, "edi_storage_oca") @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage.demo_edi_backend_storage") + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") def test_component_match(self): """Lookup with special match method.""" diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index f49138fc0..319d6e1cd 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -7,8 +7,8 @@ from .common import TestEDIStorageBase LOGGERS = ( - "odoo.addons.edi_storage.components.check", - "odoo.addons.edi.models.edi_backend", + "odoo.addons.edi_storage_oca.components.check", + "odoo.addons.edi_oca.models.edi_backend", ) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 5c7916eb1..6d2e4d5d3 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -2,7 +2,7 @@ edi.backend - + From 99eed9281b82c44a3c7701d3d83ea499b34e1835 Mon Sep 17 00:00:00 2001 From: oca-travis Date: Thu, 27 May 2021 13:11:31 +0000 Subject: [PATCH 13/77] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 edi_storage_oca/i18n/edi_storage_oca.pot diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot new file mode 100644 index 000000000..fdd8638b1 --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -0,0 +1,92 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage_oca +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 14.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +msgid "Display Name" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +msgid "ID" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +msgid "Last Modified on" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From 481d5925531550501582de3fed80c5a6ced48063 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Thu, 27 May 2021 13:16:44 +0000 Subject: [PATCH 14/77] [UPD] README.rst --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/static/description/index.html | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 14f8cd692..516004710 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index da7347ce6..a52f95fda 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -432,6 +432,7 @@

Authors

Contributors

From ebc3bcee9789cf734ea19b512df17d926039ea19 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Tue, 22 Jun 2021 13:32:31 +0200 Subject: [PATCH 15/77] [IMP] edi_storage_oca: implement basis of the input process. * A cron job check related storage on edi backends to create the needed exchange files. * Add basic receive component. --- edi_storage_oca/__manifest__.py | 6 ++- edi_storage_oca/components/__init__.py | 1 + edi_storage_oca/components/base.py | 8 ++-- edi_storage_oca/components/receive.py | 27 ++++++++++++++ edi_storage_oca/data/cron.xml | 15 ++++++++ edi_storage_oca/models/edi_backend.py | 49 +++++++++++++++++++++++++ edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + 7 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 edi_storage_oca/components/receive.py create mode 100644 edi_storage_oca/data/cron.xml diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index b8606ae4c..c15efdd67 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -13,6 +13,10 @@ "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi_oca", "storage_backend", "component"], - "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], + "data": [ + "data/cron.xml", + "security/ir_model_access.xml", + "views/edi_backend_views.xml", + ], "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py index 9f64b792b..7333ccfb3 100644 --- a/edi_storage_oca/components/__init__.py +++ b/edi_storage_oca/components/__init__.py @@ -1,3 +1,4 @@ from . import base from . import check from . import send +from . import receive diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 01d0c8b2c..ca0fa5cef 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -27,7 +27,7 @@ def _component_match(cls, work, usage=None, model_name=None, **kw): def storage(self): return self.backend.storage_id - def _dir_by_state(self, direction, state): + def _dir_by_state(self, direction, state, absolute=False): """Return remote directory path by direction and state. :param direction: string stating direction of the exchange @@ -36,9 +36,11 @@ def _dir_by_state(self, direction, state): """ assert direction in ("input", "output") assert state in ("pending", "done", "error") + if absolute: + return PurePath(self.backend[direction + "_dir_" + state] or "") return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) - def _remote_file_path(self, direction, state, filename): + def _remote_file_path(self, direction, state, filename, absolute=False): """Return remote file path by direction and state for give filename. :param direction: string stating direction of the exchange @@ -46,7 +48,7 @@ def _remote_file_path(self, direction, state, filename): :param filename: string for file name :return: PurePath object """ - return self._dir_by_state(direction, state) / filename.strip("/ ") + return self._dir_by_state(direction, state, absolute) / filename.strip("/ ") def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py new file mode 100644 index 000000000..e421df38a --- /dev/null +++ b/edi_storage_oca/components/receive.py @@ -0,0 +1,27 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component + + +class EDIStorageReceiveComponent(Component): + + _name = "edi.storage.component.receive" + _inherit = [ + "edi.component.receive.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.receive" + + def receive(self): + checker = self.component(usage="storage.check") + result = checker.check() + if not result: + # all good here + return True + + direction = self.exchange_record.direction + filename = self.exchange_record.exchange_filename + path = self._remote_file_path(direction, "pending", filename, absolute=True) + filedata = self.storage.get(path.as_posix(), binary=True) + return filedata diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml new file mode 100644 index 000000000..0dad93786 --- /dev/null +++ b/edi_storage_oca/data/cron.xml @@ -0,0 +1,15 @@ + + + + EDI backend storage check pending input + + + 1 + hours + -1 + + + code + model.search([])._cron_check_storage_pending_input() + + diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 0ba578fa0..315b63b16 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -1,9 +1,14 @@ # Copyright 2020 ACSONE SA # @author Simone Orsi +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +import logging + from odoo import fields, models +_logger = logging.getLogger(__name__) + class EDIBackend(models.Model): @@ -76,3 +81,47 @@ def _component_sort_key(self, component_class): return ( 1 if getattr(component_class, "_storage_backend_type", False) else 0, ) + res + + def _cron_check_storage_pending_input(self, **kw): + for backend in self.filtered(lambda b: b.storage_id): + backend._check_storage_pending_input(**kw) + + def _check_storage_pending_input(self, **kw): + self.ensure_one() + if not self.storage_id or not self.input_dir_pending: + _logger.info( + "%s ignored. No storage and/or input directory specified.", self.name + ) + return False + + pending_files = self.storage_id.list_files(self.input_dir_pending) + exchange_type = self.env["edi.exchange.type"].search( + self._domain_exchange_type_storage_pending_input() + ) + if not len(exchange_type) == 1: + _logger.info("%s ignored. More than one exchange type found.", self.name) + return False + for file_name in pending_files: + existing = self.env["edi.exchange.record"].search( + [ + ("backend_id", "=", self.id), + ("type_id", "=", exchange_type.id), + ("exchange_filename", "=", file_name), + ] + ) + if existing: + continue + self.create_record( + exchange_type.code, self._get_exchange_record_vals(file_name) + ) + _logger.debug("%s: new exchange record generated.", self.name) + return True + + def _domain_exchange_type_storage_pending_input(self): + return [ + ("backend_type_id", "=", self.backend_type_id.id), + ("direction", "=", "input"), + ] + + def _get_exchange_record_vals(self, file_name): + return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst index e3fa6d8e9..55ca1c7e0 100644 --- a/edi_storage_oca/readme/CONTRIBUTORS.rst +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -1,2 +1,3 @@ * Simone Orsi * Foram Shah +* Lois Rilo From 2f004bfd0588d6719cde27a0278d711e8d4c489d Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Tue, 22 Jun 2021 13:34:51 +0200 Subject: [PATCH 16/77] [FIX] edi_oca: show the exchange file name while there is no file yet. --- edi_storage_oca/components/base.py | 12 +++--- edi_storage_oca/components/receive.py | 4 +- edi_storage_oca/models/edi_backend.py | 4 +- edi_storage_oca/tests/common.py | 38 ++++++++++++++++--- .../tests/test_edi_backend_storage.py | 29 ++++++++++++++ 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index ca0fa5cef..5010e6844 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -27,7 +27,7 @@ def _component_match(cls, work, usage=None, model_name=None, **kw): def storage(self): return self.backend.storage_id - def _dir_by_state(self, direction, state, absolute=False): + def _dir_by_state(self, direction, state): """Return remote directory path by direction and state. :param direction: string stating direction of the exchange @@ -36,11 +36,11 @@ def _dir_by_state(self, direction, state, absolute=False): """ assert direction in ("input", "output") assert state in ("pending", "done", "error") - if absolute: - return PurePath(self.backend[direction + "_dir_" + state] or "") - return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) + return PurePath( + (self.backend[direction + "_dir_" + state] or "").strip().rstrip("/") + ) - def _remote_file_path(self, direction, state, filename, absolute=False): + def _remote_file_path(self, direction, state, filename): """Return remote file path by direction and state for give filename. :param direction: string stating direction of the exchange @@ -48,7 +48,7 @@ def _remote_file_path(self, direction, state, filename, absolute=False): :param filename: string for file name :return: PurePath object """ - return self._dir_by_state(direction, state, absolute) / filename.strip("/ ") + return self._dir_by_state(direction, state) / filename.strip("/ ") def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index e421df38a..695c40778 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -22,6 +22,6 @@ def receive(self): direction = self.exchange_record.direction filename = self.exchange_record.exchange_filename - path = self._remote_file_path(direction, "pending", filename, absolute=True) - filedata = self.storage.get(path.as_posix(), binary=True) + path = self._remote_file_path(direction, "pending", filename) + filedata = self.storage.get(path.as_posix()) return filedata diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 315b63b16..451495876 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -83,7 +83,7 @@ def _component_sort_key(self, component_class): ) + res def _cron_check_storage_pending_input(self, **kw): - for backend in self.filtered(lambda b: b.storage_id): + for backend in self: backend._check_storage_pending_input(**kw) def _check_storage_pending_input(self, **kw): @@ -102,7 +102,7 @@ def _check_storage_pending_input(self, **kw): _logger.info("%s ignored. More than one exchange type found.", self.name) return False for file_name in pending_files: - existing = self.env["edi.exchange.record"].search( + existing = self.env["edi.exchange.record"].search_count( [ ("backend_id", "=", self.id), ("type_id", "=", exchange_type.id), diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 4be4c1daf..7519ccfba 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -40,6 +40,14 @@ def _setup_records(cls): with open(cls.fakepath_error, "w+b") as fakefile: fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + cls.fakepath_input_pending_1 = "/tmp/test-input-001.csv" + with open(cls.fakepath_input_pending_1, "w+b") as fakefile: + fakefile.write(b"I received this in my storage.") + + cls.fakepath_input_pending_2 = "/tmp/test-input-002.csv" + with open(cls.fakepath_input_pending_2, "w+b") as fakefile: + fakefile.write(b"I received that in my storage.") + cls.checker = cls.backend._find_component( cls.partner._name, ["storage.check"], @@ -61,16 +69,19 @@ def _filename(self, record=None, ack=False): record.type_id.ack_type_id._make_exchange_filename(record) return record.exchange_filename - def _file_fullpath(self, state, record=None, ack=False): + def _file_fullpath( + self, state, record=None, ack=False, direction=False, fname=None + ): record = record or self.record - fname = self._filename(record, ack=ack) + if not fname: + fname = self._filename(record, ack=ack) + if not direction: + direction = record.direction if state == "error-report": # Exception as we read from the same path but w/ error suffix state = "error" fname += ".error" - return ( - self.checker._remote_file_path(record.direction, state, fname) - ).as_posix() + return (self.checker._remote_file_path(direction, state, fname)).as_posix() def _mocked_backend_get(self, mocked_paths, path, **kwargs): self._storage_backend_calls.append(path) @@ -82,6 +93,14 @@ def _mocked_backend_get(self, mocked_paths, path, **kwargs): def _mocked_backend_add(self, path, data, **kwargs): self._storage_backend_calls.append(path) + def _mocked_backend_list_files(self, mocked_paths, path, **kwargs): + files = [] + path_length = len(path) + for p in mocked_paths.keys(): + if path in p and path != p: + files.append(p[path_length:]) + return files + def _mock_storage_backend_get(self, mocked_paths): mocked = functools.partial(self._mocked_backend_get, mocked_paths) return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get", mocked) @@ -89,6 +108,10 @@ def _mock_storage_backend_get(self, mocked_paths): def _mock_storage_backend_add(self): return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) + def _mock_storage_backend_list_files(self, mocked_paths): + mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".list_files", mocked) + def _test_result( self, record, @@ -130,3 +153,8 @@ def _test_run_cron(self, mocked_paths): with self._mock_storage_backend_add(): with self._mock_storage_backend_get(mocked_paths): self.backend._cron_check_output_exchange_sync() + + def _test_run_cron_pending_input(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_list_files(mocked_paths): + self.backend._cron_check_storage_pending_input() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 319d6e1cd..d4779a704 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -201,3 +201,32 @@ def test_cron_full_flow(self): } ], ) + + @mute_logger(*LOGGERS) + def test_create_input_exchange_file_from_file_received(self): + exch_type = self.exchange_type_in + input_dir = "/test_input/pending/" + file_names = ["some-file.csv", "another-file.csv"] + self.backend.input_dir_pending = input_dir + mocked_paths = { + input_dir: "/tmp/", + self._file_fullpath( + "pending", direction="input", fname=file_names[0] + ): self.fakepath_input_pending_1, + self._file_fullpath( + "pending", direction="input", fname=file_names[1] + ): self.fakepath_input_pending_2, + } + existing = self.env["edi.exchange.record"].search_count( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + self.assertEqual(existing, 0) + # Run cron action: + self._test_run_cron_pending_input(mocked_paths) + new_records = self.env["edi.exchange.record"].search( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + self.assertEqual(len(new_records), 2) + for rec in new_records: + self.assertIn(rec.exchange_filename, file_names) + self.assertEqual(rec.edi_exchange_state, "input_pending") From b386bf5b14c84340ffc719c81ea3e90a62f93b9f Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 23 Jun 2021 15:46:00 +0200 Subject: [PATCH 17/77] edi_storage: improve input process --- edi_storage_oca/data/cron.xml | 2 +- edi_storage_oca/models/edi_backend.py | 78 ++++++++++++++++++--------- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml index 0dad93786..d57316cd4 100644 --- a/edi_storage_oca/data/cron.xml +++ b/edi_storage_oca/data/cron.xml @@ -10,6 +10,6 @@ code - model.search([])._cron_check_storage_pending_input() + model.search([('storage_backend_id', '!=', False)])._cron_check_storage_pending_input() diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 451495876..4f14f9935 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -4,6 +4,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging +import os from odoo import fields, models @@ -82,46 +83,73 @@ def _component_sort_key(self, component_class): 1 if getattr(component_class, "_storage_backend_type", False) else 0, ) + res - def _cron_check_storage_pending_input(self, **kw): + def _storage_cron_check_pending_input(self, **kw): for backend in self: - backend._check_storage_pending_input(**kw) + backend._storage_check_pending_input(**kw) - def _check_storage_pending_input(self, **kw): + def _storage_check_pending_input(self, **kw): + """Create new exchange records if new files found. + + Collect input exchange types and for each of them, + check by pattern if the a new exchange record is required. + """ self.ensure_one() if not self.storage_id or not self.input_dir_pending: _logger.info( - "%s ignored. No storage and/or input directory specified.", self.name + "%s ignored: no storage and/or input directory specified.", self.name ) return False - pending_files = self.storage_id.list_files(self.input_dir_pending) - exchange_type = self.env["edi.exchange.type"].search( - self._domain_exchange_type_storage_pending_input() + exchange_types = self.env["edi.exchange.type"].search( + self._storage_exchange_type_pending_input_domain() ) - if not len(exchange_type) == 1: - _logger.info("%s ignored. More than one exchange type found.", self.name) - return False - for file_name in pending_files: - existing = self.env["edi.exchange.record"].search_count( - [ - ("backend_id", "=", self.id), - ("type_id", "=", exchange_type.id), - ("exchange_filename", "=", file_name), - ] - ) - if existing: - continue - self.create_record( - exchange_type.code, self._get_exchange_record_vals(file_name) + for exchange_type in exchange_types: + # NOTE: this call might keep hanging the cron + # if the remote storage is slow (eg: too many files) + # We should probably run this code in a separate job per exchange type. + file_names = self._storage_get_input_filenames(exchange_type) + _logger.info( + "Processing exchange type '%s': found %s files to process", + exchange_type.display_name, + len(file_names), ) - _logger.debug("%s: new exchange record generated.", self.name) + for file_name in file_names: + # TODO: add config for job + self.with_delay()._storage_create_record_if_missing( + exchange_type, file_name + ) return True - def _domain_exchange_type_storage_pending_input(self): + def _storage_exchange_type_pending_input_domain(self): + """Domain for retrieving input exchange types. + """ return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), ] - def _get_exchange_record_vals(self, file_name): + def _storage_create_record_if_missing(self, exchange_type, remote_file_name): + """Create a new exchange record for given type and file name if missing. + """ + file_name = os.path.basename(remote_file_name) + extra_domain = [("exchange_filename", "=", file_name)] + existing = self._find_existing_exchange_records( + exchange_type, extra_domain=extra_domain, count_only=True + ) + if existing: + return + record = self.create_record( + exchange_type.code, self._storage_new_exchange_record_vals(file_name) + ) + _logger.debug("%s: new exchange record generated.", self.name) + return record.identifier + + def _storage_get_input_filenames(self, exchange_type): + bits = [exchange_type.exchange_filename_pattern] + if exchange_type.exchange_file_ext: + bits.append("*." + exchange_type.exchange_file_ext) + pattern = "".join(bits) + return self.storage_id.find_files(self.input_dir_pending, pattern=pattern) + + def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} From e410999c8e519f50e20d5c3eb7c5a5100e81e681 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Wed, 23 Jun 2021 17:10:11 +0200 Subject: [PATCH 18/77] [FIX] edi_storage_oca: support the use or not of filename patern + small runtime fixes. --- edi_storage_oca/data/cron.xml | 4 +++- edi_storage_oca/models/edi_backend.py | 7 ++++++- edi_storage_oca/tests/common.py | 2 +- edi_storage_oca/tests/test_edi_backend_storage.py | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml index d57316cd4..df21892a0 100644 --- a/edi_storage_oca/data/cron.xml +++ b/edi_storage_oca/data/cron.xml @@ -10,6 +10,8 @@ code - model.search([('storage_backend_id', '!=', False)])._cron_check_storage_pending_input() + model.search([('storage_id', '!=', False)])._storage_cron_check_pending_input() diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 4f14f9935..2b9674a53 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -145,11 +145,16 @@ def _storage_create_record_if_missing(self, exchange_type, remote_file_name): return record.identifier def _storage_get_input_filenames(self, exchange_type): + if not exchange_type.exchange_filename_pattern: + # If there is not patter, return everything + return self.storage_id.list_files(self.input_dir_pending) + + # TODO: validate pattern usage. bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: bits.append("*." + exchange_type.exchange_file_ext) pattern = "".join(bits) - return self.storage_id.find_files(self.input_dir_pending, pattern=pattern) + return self.storage_id.find_files(pattern, self.input_dir_pending) def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 7519ccfba..5aaa9951e 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -157,4 +157,4 @@ def _test_run_cron(self, mocked_paths): def _test_run_cron_pending_input(self, mocked_paths): with self._mock_storage_backend_add(): with self._mock_storage_backend_list_files(mocked_paths): - self.backend._cron_check_storage_pending_input() + self.backend._storage_cron_check_pending_input() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index d4779a704..06448474b 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -203,8 +203,9 @@ def test_cron_full_flow(self): ) @mute_logger(*LOGGERS) - def test_create_input_exchange_file_from_file_received(self): + def test_create_input_exchange_file_from_file_received_no_pattern(self): exch_type = self.exchange_type_in + exch_type.exchange_filename_pattern = "" input_dir = "/test_input/pending/" file_names = ["some-file.csv", "another-file.csv"] self.backend.input_dir_pending = input_dir From 1f5379247861577c5ce65ba48959c267264de5d2 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Thu, 1 Jul 2021 09:29:10 +0200 Subject: [PATCH 19/77] [IMP] edi_storage_oca: add specific channel and job function definitions --- edi_storage_oca/__manifest__.py | 2 ++ edi_storage_oca/data/job_channel_data.xml | 6 ++++++ edi_storage_oca/data/queue_job_function_data.xml | 7 +++++++ edi_storage_oca/models/edi_backend.py | 1 - 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 edi_storage_oca/data/job_channel_data.xml create mode 100644 edi_storage_oca/data/queue_job_function_data.xml diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index c15efdd67..c950bd37f 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -15,6 +15,8 @@ "depends": ["edi_oca", "storage_backend", "component"], "data": [ "data/cron.xml", + "data/job_channel_data.xml", + "data/queue_job_function_data.xml", "security/ir_model_access.xml", "views/edi_backend_views.xml", ], diff --git a/edi_storage_oca/data/job_channel_data.xml b/edi_storage_oca/data/job_channel_data.xml new file mode 100644 index 000000000..b3b3b770d --- /dev/null +++ b/edi_storage_oca/data/job_channel_data.xml @@ -0,0 +1,6 @@ + + + edi_storage + + + diff --git a/edi_storage_oca/data/queue_job_function_data.xml b/edi_storage_oca/data/queue_job_function_data.xml new file mode 100644 index 000000000..196528419 --- /dev/null +++ b/edi_storage_oca/data/queue_job_function_data.xml @@ -0,0 +1,7 @@ + + + + _storage_create_record_if_missing + + + diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 2b9674a53..5e229a8a4 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -114,7 +114,6 @@ def _storage_check_pending_input(self, **kw): len(file_names), ) for file_name in file_names: - # TODO: add config for job self.with_delay()._storage_create_record_if_missing( exchange_type, file_name ) From f35489f6a854a1a73af0892cdefeb1331211bfd8 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Thu, 1 Jul 2021 12:57:49 +0200 Subject: [PATCH 20/77] [FIX] edi_storage_oca: support regex pattern searching when fetching filenames from storage. Also extend help in `exchange_filename_pattern` field to clarify usage. --- edi_storage_oca/models/__init__.py | 1 + edi_storage_oca/models/edi_backend.py | 7 ++++--- edi_storage_oca/models/edi_exchange_type.py | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 edi_storage_oca/models/edi_exchange_type.py diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py index 4b2c54ac3..f34a72163 100644 --- a/edi_storage_oca/models/__init__.py +++ b/edi_storage_oca/models/__init__.py @@ -1 +1,2 @@ from . import edi_backend +from . import edi_exchange_type diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5e229a8a4..21d34095e 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -148,12 +148,13 @@ def _storage_get_input_filenames(self, exchange_type): # If there is not patter, return everything return self.storage_id.list_files(self.input_dir_pending) - # TODO: validate pattern usage. bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: - bits.append("*." + exchange_type.exchange_file_ext) + bits.append(r"\." + exchange_type.exchange_file_ext) pattern = "".join(bits) - return self.storage_id.find_files(pattern, self.input_dir_pending) + full_paths = self.storage_id.find_files(pattern, self.input_dir_pending) + pending_path_len = len(self.input_dir_pending) + return [p[pending_path_len:].strip("/") for p in full_paths] def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py new file mode 100644 index 000000000..42a7f7008 --- /dev/null +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -0,0 +1,21 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo import fields, models + + +class EDIExchangeType(models.Model): + _inherit = "edi.exchange.type" + + # Extend help to explain new usage. + exchange_filename_pattern = fields.Char( + help="For output exchange types this should be a formatting string " + "with the following variables available (to be used between " + "brackets, `{}`): `exchange_record`, `record_name`, `type` and " + "`dt`. For instance, a valid string would be " + "{record_name}-{type.code}-{dt}\n" + "For input exchange types related to storage backends " + "it should be a regex expression to filter " + "the files to be fetched from the pending directory in the related " + "storage. E.g: `.*my-type-[0-9]*.\\.csv`" + ) From 9ff1cae875e6e9b7de74ea786d7266029d5e8149 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Fri, 2 Jul 2021 09:23:52 +0200 Subject: [PATCH 21/77] [FIX] edi_storage: consider if the exchange type has a specific backend assigned. --- edi_storage_oca/models/edi_backend.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 21d34095e..0f8196be7 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -125,6 +125,9 @@ def _storage_exchange_type_pending_input_domain(self): return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), + "|", + ("backend_id", "=", False), + ("backend_id", "=", self.id), ] def _storage_create_record_if_missing(self, exchange_type, remote_file_name): From 9246cc3409f2fa25a431dd5955c37963b5c349a0 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 5 Jul 2021 15:56:58 +0200 Subject: [PATCH 22/77] [IMP] : black, isort, prettier --- edi_storage_oca/models/edi_backend.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 0f8196be7..5f95e3382 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -120,8 +120,7 @@ def _storage_check_pending_input(self, **kw): return True def _storage_exchange_type_pending_input_domain(self): - """Domain for retrieving input exchange types. - """ + """Domain for retrieving input exchange types.""" return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), @@ -131,8 +130,7 @@ def _storage_exchange_type_pending_input_domain(self): ] def _storage_create_record_if_missing(self, exchange_type, remote_file_name): - """Create a new exchange record for given type and file name if missing. - """ + """Create a new exchange record for given type and file name if missing.""" file_name = os.path.basename(remote_file_name) extra_domain = [("exchange_filename", "=", file_name)] existing = self._find_existing_exchange_records( From 144c434e1429b303fe1c36a8120dbcdd273d2e6f Mon Sep 17 00:00:00 2001 From: oca-travis Date: Tue, 6 Jul 2021 09:43:30 +0000 Subject: [PATCH 23/77] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index fdd8638b1..784ca4c68 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -15,6 +15,7 @@ msgstr "" #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__display_name msgid "Display Name" msgstr "" @@ -23,8 +24,33 @@ msgstr "" msgid "EDI Backend" msgstr "" +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type +msgid "EDI Exchange Type" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server +#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input +#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input +msgid "EDI backend storage check pending input" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "Exchange Filename Pattern" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "" +"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__id msgid "ID" msgstr "" @@ -45,6 +71,7 @@ msgstr "" #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type____last_update msgid "Last Modified on" msgstr "" From d8fee6396c93784dab6b98652fa9c28843c7dc5e Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 6 Jul 2021 09:49:11 +0000 Subject: [PATCH 24/77] [UPD] README.rst --- edi_storage_oca/README.rst | 1 + edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 516004710..dc3a04a12 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -87,6 +87,7 @@ Contributors * Simone Orsi * Foram Shah +* Lois Rilo Maintainers ~~~~~~~~~~~ diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index c950bd37f..e25ee456d 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.0.0", + "version": "14.0.1.1.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index a52f95fda..257c2d221 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -433,6 +433,7 @@

Contributors

From 52f0b8840ff371a6a646ec92e4d738c93de1e2e2 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 5 Jul 2021 18:36:03 +0200 Subject: [PATCH 25/77] [IMP] edi_storage: add listener to move files to done/error dirs when a file is processed the listener will move it to the done directory or the error directory based on the result of processing the file. --- edi_storage_oca/components/__init__.py | 1 + edi_storage_oca/components/listener.py | 59 +++++++++++++++ edi_storage_oca/tests/__init__.py | 1 + .../tests/test_edi_storage_listener.py | 71 +++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 edi_storage_oca/components/listener.py create mode 100644 edi_storage_oca/tests/test_edi_storage_listener.py diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py index 7333ccfb3..7bff9ddfc 100644 --- a/edi_storage_oca/components/__init__.py +++ b/edi_storage_oca/components/__init__.py @@ -2,3 +2,4 @@ from . import check from . import send from . import receive +from . import listener diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py new file mode 100644 index 000000000..a1f248b7a --- /dev/null +++ b/edi_storage_oca/components/listener.py @@ -0,0 +1,59 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import functools +from pathlib import PurePath + +from odoo.addons.component.core import Component + + +class EdiStorageListener(Component): + _name = "edi.storage.component.listener" + _inherit = "base.event.listener" + + def _move_file(self, storage, from_dir_str, to_dir_str, filename): + from_dir = PurePath(from_dir_str) + to_dir = PurePath(to_dir_str) + if filename not in storage.list_files(from_dir): + # The file might have been moved after a previous error. + return False + self._add_after_commit_hook( + storage._move_files, [(from_dir / filename).as_posix()], to_dir.as_posix() + ) + return True + + def _add_after_commit_hook(self, move_func, sftp_filepath, sftp_destination_path): + """Add hook after commit to move the file when transaction is over.""" + self.env.cr.after( + "commit", + functools.partial(move_func, sftp_filepath, sftp_destination_path), + ) + + def on_edi_exchange_done(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.backend_id.input_dir_pending + done_dir = record.backend_id.input_dir_done + error_dir = record.backend_id.input_dir_error + if not done_dir: + return res + res = self._move_file(storage, pending_dir, done_dir, file) + if not res: + # If a file previously failed it should have been previously + # moved to the error dir, therefore it is not present in the + # pending dir and we need to retry from error dir. + res = self._move_file(storage, error_dir, done_dir, file) + return res + + def on_edi_exchange_error(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.backend_id.input_dir_pending + error_dir = record.backend_id.input_dir_error + if error_dir: + res = self._move_file(storage, pending_dir, error_dir, file) + return res diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index b788a0071..a91d5f238 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1,2 +1,3 @@ from . import test_edi_backend_storage from . import test_components_base +from . import test_edi_storage_listener diff --git a/edi_storage_oca/tests/test_edi_storage_listener.py b/edi_storage_oca/tests/test_edi_storage_listener.py new file mode 100644 index 000000000..4f12f3311 --- /dev/null +++ b/edi_storage_oca/tests/test_edi_storage_listener.py @@ -0,0 +1,71 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 + +import mock + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.fake_components import FakeInputProcess + +LISTENER_MOCK_PATH = ( + "odoo.addons.edi_storage_oca.components.listener.EdiStorageListener" +) + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage_oca") + cls._build_components( + cls, + FakeInputProcess, + ) + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": base64.b64encode(b"1234"), + } + cls.record = cls.backend.create_record("test_csv_input", vals) + cls.fake_move_args = None + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + def setUp(self): + super().setUp() + FakeInputProcess.reset_faked() + + def _move_file_mocked(self, *args): + self.fake_move_args = [*args] + if not all([*args]): + return False + return True + + def _mock_listener_move_file(self): + return mock.patch(LISTENER_MOCK_PATH + "._move_file", self._move_file_mocked) + + def test_01_process_record_success(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record.action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_done) + self.assertEqual(filename, self.record.exchange_filename) + + def test_02_process_record_with_error(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record._set_file_content("TEST %d" % self.record.id) + self.record.with_context( + test_break_process="OOPS! Something went wrong :(" + ).action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_error) + self.assertEqual(filename, self.record.exchange_filename) From 8cc85e8448cc6008fe1874d1cd8b89baf8e9f47f Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Fri, 9 Jul 2021 11:25:05 +0200 Subject: [PATCH 26/77] [FIX] edi_storage: `list_files` expects a string. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/listener.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index e25ee456d..fb84ffcbb 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.1.0", + "version": "14.0.1.2.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py index a1f248b7a..7b551c81e 100644 --- a/edi_storage_oca/components/listener.py +++ b/edi_storage_oca/components/listener.py @@ -14,7 +14,7 @@ class EdiStorageListener(Component): def _move_file(self, storage, from_dir_str, to_dir_str, filename): from_dir = PurePath(from_dir_str) to_dir = PurePath(to_dir_str) - if filename not in storage.list_files(from_dir): + if filename not in storage.list_files(from_dir.as_posix()): # The file might have been moved after a previous error. return False self._add_after_commit_hook( From eb55385fae0b21c67e40ffebb6e2a740591ffd48 Mon Sep 17 00:00:00 2001 From: JordiMForgeFlow Date: Wed, 4 Aug 2021 09:23:35 +0200 Subject: [PATCH 27/77] [14.0][FIX] edi_storage_oca: remove duplicate backend_type_id field in edi backend form --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/views/edi_backend_views.xml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index fb84ffcbb..7fd2405b1 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.0", + "version": "14.0.1.2.1", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 6d2e4d5d3..520965fb3 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -6,7 +6,6 @@ - From b9a46402a329c276c3104d593044e9e7c8063f1c Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 6 Sep 2021 12:04:31 +0200 Subject: [PATCH 28/77] [FIX] edi_storage: incorrect call to output checker in receive component. checker is only relevant for output direction. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/receive.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 7fd2405b1..efe19ecca 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.1", + "version": "14.0.1.2.2", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index 695c40778..0061ac1d9 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -14,12 +14,6 @@ class EDIStorageReceiveComponent(Component): _usage = "storage.receive" def receive(self): - checker = self.component(usage="storage.check") - result = checker.check() - if not result: - # all good here - return True - direction = self.exchange_record.direction filename = self.exchange_record.exchange_filename path = self._remote_file_path(direction, "pending", filename) From 241c45bd1b21f765f64843b9a63e9689b276d6e9 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 17 Sep 2021 17:14:19 +0200 Subject: [PATCH 29/77] edi: improve backend view extension --- edi_storage_oca/views/edi_backend_views.xml | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 520965fb3..ead51ef51 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -4,17 +4,19 @@ edi.backend - - - - - - - - - - - + + + + + + + + + + + + + From 3f2a48ce5b9942b2b1307b83074540a0814451b8 Mon Sep 17 00:00:00 2001 From: oca-travis Date: Wed, 13 Oct 2021 14:22:09 +0000 Subject: [PATCH 30/77] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/i18n/edi_storage_oca.pot | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index efe19ecca..3481bef65 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.2", + "version": "14.0.1.3.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index 784ca4c68..eb9e80688 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -108,6 +108,11 @@ msgstr "" msgid "Path to folder for pending operations" msgstr "" +#. module: edi_storage_oca +#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form +msgid "Storage" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id msgid "Storage backend" From da890c2fc03fa129ca48a9e887c479210b709b20 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 22 Sep 2021 20:07:37 +0200 Subject: [PATCH 31/77] edi: load main vars at init --- edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/test_component_match.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index a91d5f238..678979f8c 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1,3 +1,4 @@ from . import test_edi_backend_storage from . import test_components_base +from . import test_component_match from . import test_edi_storage_listener diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index e4fbfbe82..f0a776ae9 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -3,7 +3,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component -from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): @@ -49,9 +49,13 @@ class S3Send(Component): self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + # Record not relevant for these tests + work_ctx = {"exchange_record": self.env["edi.exchange.record"].browse()} + component = self.backend._find_component( "res.partner", ["storage.check"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="s3", @@ -61,6 +65,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.check"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="sftp", @@ -70,6 +75,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.send"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="sftp", @@ -79,6 +85,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.send"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="s3", From 422d8d193bd0822c0e26d0f6c4846615b7d404fd Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Sat, 30 Oct 2021 08:52:28 +0200 Subject: [PATCH 32/77] edi_storage: fix find files test --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/tests/common.py | 6 ++++++ edi_storage_oca/tests/test_edi_backend_storage.py | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 3481bef65..d0cb44158 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.3.0", + "version": "14.0.1.4.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 5aaa9951e..31e4077c0 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -112,6 +112,12 @@ def _mock_storage_backend_list_files(self, mocked_paths): mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".list_files", mocked) + def _mock_storage_backend_find_files(self, result): + def _result(self, pattern, relative_path=None, **kw): + return result + + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".find_files", _result) + def _test_result( self, record, diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 06448474b..2f396d39d 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -223,7 +223,9 @@ def test_create_input_exchange_file_from_file_received_no_pattern(self): ) self.assertEqual(existing, 0) # Run cron action: - self._test_run_cron_pending_input(mocked_paths) + found_files = [input_dir + fname for fname in file_names] + with self._mock_storage_backend_find_files(found_files): + self._test_run_cron_pending_input(mocked_paths) new_records = self.env["edi.exchange.record"].search( [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] ) From 0c5d63d42d293d8abc5c1f5f255e58598d62c1a2 Mon Sep 17 00:00:00 2001 From: Iryna Vyshnevska Date: Mon, 6 Dec 2021 00:28:47 +0200 Subject: [PATCH 33/77] [MIG][15.0] edi_storage_oca --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/i18n/edi_storage.pot | 77 -------------- edi_storage_oca/i18n/edi_storage_oca.pot | 124 ----------------------- edi_storage_oca/tests/common.py | 1 + 4 files changed, 2 insertions(+), 202 deletions(-) delete mode 100644 edi_storage_oca/i18n/edi_storage.pot delete mode 100644 edi_storage_oca/i18n/edi_storage_oca.pot diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index d0cb44158..a7480dafd 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.4.0", + "version": "15.0.1.0.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/i18n/edi_storage.pot b/edi_storage_oca/i18n/edi_storage.pot deleted file mode 100644 index d6944e83b..000000000 --- a/edi_storage_oca/i18n/edi_storage.pot +++ /dev/null @@ -1,77 +0,0 @@ -# Translation of Odoo Server. -# This file contains the translation of the following modules: -# * edi_storage -# -msgid "" -msgstr "" -"Project-Id-Version: Odoo Server 13.0\n" -"Report-Msgid-Bugs-To: \n" -"Last-Translator: \n" -"Language-Team: \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: \n" -"Plural-Forms: \n" - -#. module: edi_storage -#: model:ir.model,name:edi_storage.model_edi_backend -msgid "EDI Backend" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_done -msgid "Input done directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_error -msgid "Input error directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_pending -msgid "Input pending directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_done -msgid "Output done directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_error -msgid "Output error directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_pending -msgid "Output pending directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_done -#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_done -msgid "Path to folder for doneful operations" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_error -#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_error -msgid "Path to folder for error operations" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_pending -#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_pending -msgid "Path to folder for pending operations" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__storage_id -msgid "Storage backend" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__storage_id -msgid "Storage for in-out files" -msgstr "" diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot deleted file mode 100644 index eb9e80688..000000000 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ /dev/null @@ -1,124 +0,0 @@ -# Translation of Odoo Server. -# This file contains the translation of the following modules: -# * edi_storage_oca -# -msgid "" -msgstr "" -"Project-Id-Version: Odoo Server 14.0\n" -"Report-Msgid-Bugs-To: \n" -"Last-Translator: \n" -"Language-Team: \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: \n" -"Plural-Forms: \n" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__display_name -msgid "Display Name" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model,name:edi_storage_oca.model_edi_backend -msgid "EDI Backend" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type -msgid "EDI Exchange Type" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server -#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input -#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input -msgid "EDI backend storage check pending input" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern -msgid "Exchange Filename Pattern" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern -msgid "" -"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" -"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__id -msgid "ID" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done -msgid "Input done directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error -msgid "Input error directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending -msgid "Input pending directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type____last_update -msgid "Last Modified on" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done -msgid "Output done directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error -msgid "Output error directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending -msgid "Output pending directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done -msgid "Path to folder for doneful operations" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error -msgid "Path to folder for error operations" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending -msgid "Path to folder for pending operations" -msgstr "" - -#. module: edi_storage_oca -#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form -msgid "Storage" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id -msgid "Storage backend" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id -msgid "Storage for in-out files" -msgstr "" diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 31e4077c0..385b1a5f5 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -58,6 +58,7 @@ def _setup_records(cls): ["storage.send"], work_ctx={"exchange_record": cls.record}, ) + return def setUp(self): super().setUp() From f623aaeb1b626f7fe4239407cf9fc081c616b15c Mon Sep 17 00:00:00 2001 From: oca-ci Date: Tue, 15 Mar 2022 12:12:00 +0000 Subject: [PATCH 34/77] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 106 +++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 edi_storage_oca/i18n/edi_storage_oca.pot diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot new file mode 100644 index 000000000..3458c862e --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -0,0 +1,106 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage_oca +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 15.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type +msgid "EDI Exchange Type" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server +#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input +#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input +msgid "EDI backend storage check pending input" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "Exchange Filename Pattern" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "" +"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage_oca +#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form +msgid "Storage" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From c63f7945c4e32b5b2511fbab558a21b32f3a730d Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 15 Mar 2022 12:15:16 +0000 Subject: [PATCH 35/77] [UPD] README.rst --- edi_storage_oca/README.rst | 10 +++++----- edi_storage_oca/static/description/index.html | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index dc3a04a12..e75484b38 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -14,13 +14,13 @@ EDI Storage backend support :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github - :target: https://github.com/OCA/edi/tree/14.0/edi_storage_oca + :target: https://github.com/OCA/edi/tree/15.0/edi_storage_oca :alt: OCA/edi .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/edi-14-0/edi-14-0-edi_storage_oca + :target: https://translation.odoo-community.org/projects/edi-15-0/edi-15-0-edi_storage_oca :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/14.0 + :target: https://runbot.odoo-community.org/runbot/226/15.0 :alt: Try me on Runbot |badge1| |badge2| |badge3| |badge4| |badge5| @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -102,6 +102,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/edi `_ project on GitHub. +This module is part of the `OCA/edi `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 257c2d221..8fa1825de 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -443,7 +443,7 @@

Maintainers

OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

-

This module is part of the OCA/edi project on GitHub.

+

This module is part of the OCA/edi project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

From f79424372caa0b0c028488bbefb2c6de8213ffda Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Thu, 10 Mar 2022 12:41:35 +0100 Subject: [PATCH 36/77] edi: get rid of name, use identifier Name contained redundant information like type name and related record. On for and tree view is totally useless. To still provide information on linked resource name the 'related_name' field has been added. --- edi_storage_oca/components/check.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 878cfffcb..90f13c951 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -37,9 +37,8 @@ def _exchange_output_check(self): """ if self._get_remote_file("done"): _logger.info( - "%s done for: %s", - self.exchange_record.model, - self.exchange_record.name, + "%s done", + self.exchange_record.identifier, ) if ( not self.exchange_record.edi_exchange_state @@ -52,9 +51,8 @@ def _exchange_output_check(self): error = self._get_remote_file("error") if error: _logger.info( - "%s error for: %s", - self.exchange_record.model, - self.exchange_record.name, + "%s error", + self.exchange_record.identifier, ) # Assume a text file will be placed there w/ the same name and error suffix err_filename = self.exchange_record.exchange_filename + ".error" From 199b44664ba0526fecfa89746f26e58425eb72f3 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Thu, 10 Mar 2022 16:28:30 +0100 Subject: [PATCH 37/77] edi_storage: fail gracefully w/o error report --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/check.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index a7480dafd..f9b30ed42 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.0.0", + "version": "15.0.1.1.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 90f13c951..2cd97fb4a 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -56,7 +56,9 @@ def _exchange_output_check(self): ) # Assume a text file will be placed there w/ the same name and error suffix err_filename = self.exchange_record.exchange_filename + ".error" - error_report = self._get_remote_file("error", filename=err_filename) + error_report = ( + self._get_remote_file("error", filename=err_filename) or "no-report" + ) if self.exchange_record.edi_exchange_state == "output_sent": self.exchange_record.update( { From e9449d003a5d1993181fd27721b2f9101725559f Mon Sep 17 00:00:00 2001 From: Jasmin Solanki Date: Tue, 12 Apr 2022 16:45:22 +0530 Subject: [PATCH 38/77] [IMP] edi_storage_oca: Update development status --- edi_storage_oca/README.rst | 6 +++--- edi_storage_oca/__manifest__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index e75484b38..335f991fe 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -7,9 +7,9 @@ EDI Storage backend support !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status - :alt: Alpha + :alt: Beta .. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/15.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index f9b30ed42..92310bce1 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -8,7 +8,7 @@ Base module to allow exchanging files via storage backend (eg: SFTP). """, "version": "15.0.1.1.0", - "development_status": "Alpha", + "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", From df13900246294ed47dd3022ba72986f169268cc1 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 12 Apr 2022 14:39:57 +0000 Subject: [PATCH 39/77] [UPD] README.rst --- edi_storage_oca/README.rst | 7 +------ edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 8 +------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 335f991fe..c182ec9be 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/15.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. @@ -49,11 +49,6 @@ Inside this folder you have this hierarchy:: The storage handlers take care of reading files and putting files in/from the right place and update exchange records data accordingly. -.. IMPORTANT:: - This is an alpha version, the data model and design can change at any time without warning. - Only for development or testing purpose, do not use in production. - `More details on development status `_ - **Table of contents** .. contents:: diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 92310bce1..6eeee4e89 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.1.0", + "version": "15.0.1.1.1", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 8fa1825de..d08dd56bb 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Beta License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -389,12 +389,6 @@

EDI Storage backend support

The storage handlers take care of reading files and putting files in/from the right place and update exchange records data accordingly.

-
-

Important

-

This is an alpha version, the data model and design can change at any time without warning. -Only for development or testing purpose, do not use in production. -More details on development status

-

Table of contents

    From 67f110e7239390396c5f933f02cbeaada0649ce3 Mon Sep 17 00:00:00 2001 From: jcoux Date: Tue, 17 May 2022 16:50:06 +0200 Subject: [PATCH 40/77] edi_storage_oca: improve error handling When trying to get done/failed directories, if we don't found it on SFTP server, we ignore the error. Current implementation only ignores FileNotFoundError, but depending on the SFTP server, we can have OSError. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/base.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 6eeee4e89..d9715bb91 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.1.1", + "version": "15.0.1.1.2", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 5010e6844..40e1dcb2f 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,11 +1,13 @@ # Copyright 2020 ACSONE # @author: Simone Orsi # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). - +import logging from pathlib import PurePath from odoo.addons.component.core import AbstractComponent +_logger = logging.getLogger(__file__) + class EDIStorageComponentMixin(AbstractComponent): @@ -65,4 +67,19 @@ def _get_remote_file(self, state, filename=None, binary=False): # (the date will never match) return self.storage.get(path.as_posix(), binary=binary) except FileNotFoundError: + _logger.info( + "Ignored FileNotFoundError when trying " + "to get file %s into path %s for state %s", + filename, + path, + state, + ) + return None + except OSError: + _logger.info( + "Ignored OSError when trying to get file %s into path %s for state %s", + filename, + path, + state, + ) return None From c33fa860271279450c7de33e0bab432ff629f1be Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 6 May 2022 12:20:57 +0200 Subject: [PATCH 41/77] edi_storage: path configurable by type and param By setting specific storage path on exchange type advanced settings, we can configure specific path for each exchange type (input or output). Path can also be configured by config parameters to allow to easily set different values depending of the environment. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/base.py | 18 ++++- edi_storage_oca/components/listener.py | 20 ++++-- edi_storage_oca/components/receive.py | 4 +- edi_storage_oca/components/send.py | 5 +- edi_storage_oca/models/edi_backend.py | 16 +++-- edi_storage_oca/models/edi_exchange_type.py | 41 ++++++++++++ edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 15 +++-- edi_storage_oca/tests/test_components_base.py | 12 ++-- .../tests/test_edi_backend_storage.py | 13 ++-- edi_storage_oca/tests/test_exchange_type.py | 67 +++++++++++++++++++ 12 files changed, 179 insertions(+), 35 deletions(-) create mode 100644 edi_storage_oca/tests/test_exchange_type.py diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index d9715bb91..2e490068d 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.1.2", + "version": "15.0.1.2.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 40e1dcb2f..db1a2c8a3 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,4 +1,5 @@ # Copyright 2020 ACSONE +# Copyright 2022 Camptocamp # @author: Simone Orsi # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging @@ -50,8 +51,22 @@ def _remote_file_path(self, direction, state, filename): :param filename: string for file name :return: PurePath object """ + _logger.warning( + "Call of deprecated function `_remote_file_path`. " + "Please use `_get_remote_file_path` instead.", + ) return self._dir_by_state(direction, state) / filename.strip("/ ") + def _get_remote_file_path(self, state, filename=None): + """Retrieve remote path for current exchange record.""" + filename = filename or self.exchange_record.exchange_filename + direction = self.exchange_record.direction + directory = self._dir_by_state(direction, state).as_posix() + path = self.exchange_record.type_id._storage_fullpath( + directory=directory, filename=filename + ) + return path + def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. @@ -59,8 +74,7 @@ def _get_remote_file(self, state, filename=None, binary=False): :param filename: custom file name, exchange_record filename used by default :return: remote file content as string """ - filename = filename or self.exchange_record.exchange_filename - path = self._remote_file_path(self.exchange_record.direction, state, filename) + path = self._get_remote_file_path(state, filename=filename) try: # TODO: support match via pattern (eg: filename-prefix-*) # otherwise is impossible to retrieve input files and acks diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py index 7b551c81e..54ad1ec01 100644 --- a/edi_storage_oca/components/listener.py +++ b/edi_storage_oca/components/listener.py @@ -34,9 +34,15 @@ def on_edi_exchange_done(self, record): res = False if record.direction == "input" and storage: file = record.exchange_filename - pending_dir = record.backend_id.input_dir_pending - done_dir = record.backend_id.input_dir_done - error_dir = record.backend_id.input_dir_error + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + done_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_done + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() if not done_dir: return res res = self._move_file(storage, pending_dir, done_dir, file) @@ -52,8 +58,12 @@ def on_edi_exchange_error(self, record): res = False if record.direction == "input" and storage: file = record.exchange_filename - pending_dir = record.backend_id.input_dir_pending - error_dir = record.backend_id.input_dir_error + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() if error_dir: res = self._move_file(storage, pending_dir, error_dir, file) return res diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index 0061ac1d9..b0ab8257e 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -14,8 +14,6 @@ class EDIStorageReceiveComponent(Component): _usage = "storage.receive" def receive(self): - direction = self.exchange_record.direction - filename = self.exchange_record.exchange_filename - path = self._remote_file_path(direction, "pending", filename) + path = self._get_remote_file_path("pending") filedata = self.storage.get(path.as_posix()) return filedata diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index cbb6ed309..95c6ecdc6 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -23,11 +23,8 @@ def send(self): if not result: # all good here return True - - direction = self.exchange_record.direction - filename = self.exchange_record.exchange_filename filedata = self.exchange_record.exchange_file - path = self._remote_file_path(direction, "pending", filename) + path = self._get_remote_file_path("pending") self.storage.add(path.as_posix(), filedata, binary=False) # TODO: delegate this to generic storage backend # except paramiko.ssh_exception.AuthenticationException: diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5f95e3382..3669eabf9 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -145,16 +145,24 @@ def _storage_create_record_if_missing(self, exchange_type, remote_file_name): return record.identifier def _storage_get_input_filenames(self, exchange_type): + full_input_dir_pending = exchange_type._storage_fullpath( + self.input_dir_pending + ).as_posix() if not exchange_type.exchange_filename_pattern: - # If there is not patter, return everything - return self.storage_id.list_files(self.input_dir_pending) + # If there is not pattern, return everything + filenames = [ + x + for x in self.storage_id.list_files(full_input_dir_pending) + if x.strip("/") + ] + return filenames bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: bits.append(r"\." + exchange_type.exchange_file_ext) pattern = "".join(bits) - full_paths = self.storage_id.find_files(pattern, self.input_dir_pending) - pending_path_len = len(self.input_dir_pending) + full_paths = self.storage_id.find_files(pattern, full_input_dir_pending) + pending_path_len = len(full_input_dir_pending) return [p[pending_path_len:].strip("/") for p in full_paths] def _storage_new_exchange_record_vals(self, file_name): diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py index 42a7f7008..258576edc 100644 --- a/edi_storage_oca/models/edi_exchange_type.py +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -1,6 +1,8 @@ # Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +from pathlib import PurePath + from odoo import fields, models @@ -19,3 +21,42 @@ class EDIExchangeType(models.Model): "the files to be fetched from the pending directory in the related " "storage. E.g: `.*my-type-[0-9]*.\\.csv`" ) + + def _storage_path(self): + """Retrieve specific path for current exchange type. + + In your exchange type you can pass this config: + + storage: + # simple string + path: path/to/file + + Or + + storage: + # name of the param containing the path + path_config_param: path/to/file + + Thanks to the param you could even configure it by env. + """ + self.ensure_one() + storage_settings = self.advanced_settings.get("storage", {}) + path = storage_settings.get("path") + if path: + return PurePath(path) + path_config_param = storage_settings.get("path_config_param") + if path_config_param: + icp = self.env["ir.config_parameter"].sudo() + path = icp.get_param(path_config_param) + if path: + return PurePath(path) + + def _storage_fullpath(self, directory=None, filename=None): + self.ensure_one() + path_prefix = self._storage_path() + path = PurePath((directory or "").strip().rstrip("/")) + if path_prefix: + path = path_prefix / path + if filename: + path = path / filename.strip("/") + return path diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index 678979f8c..18bc95300 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -2,3 +2,4 @@ from . import test_components_base from . import test_component_match from . import test_edi_storage_listener +from . import test_exchange_type diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 385b1a5f5..cdca0f2e0 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -27,6 +27,7 @@ def _setup_records(cls): "exchange_file": cls.filedata, } cls.record = cls.backend.create_record("test_csv_output", vals) + cls.record_input = cls.backend.create_record("test_csv_input", vals) cls.fakepath = "/tmp/{}".format(cls._filename(cls)) with open(cls.fakepath, "w+b") as fakefile: @@ -53,6 +54,11 @@ def _setup_records(cls): ["storage.check"], work_ctx={"exchange_record": cls.record}, ) + cls.checker_input = cls.backend._find_component( + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record_input}, + ) cls.sender = cls.backend._find_component( cls.partner._name, ["storage.send"], @@ -70,19 +76,16 @@ def _filename(self, record=None, ack=False): record.type_id.ack_type_id._make_exchange_filename(record) return record.exchange_filename - def _file_fullpath( - self, state, record=None, ack=False, direction=False, fname=None - ): + def _file_fullpath(self, state, record=None, ack=False, fname=None, checker=None): record = record or self.record + checker = checker or self.checker if not fname: fname = self._filename(record, ack=ack) - if not direction: - direction = record.direction if state == "error-report": # Exception as we read from the same path but w/ error suffix state = "error" fname += ".error" - return (self.checker._remote_file_path(direction, state, fname)).as_posix() + return checker._get_remote_file_path(state, filename=fname).as_posix() def _mocked_backend_get(self, mocked_paths, path, **kwargs): self._storage_backend_calls.append(path) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index 57abd1385..9388993b0 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -18,14 +18,16 @@ def test_remote_file_path(self): (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), ) for _args, expected in to_test: - path_obj = self.checker._remote_file_path(*_args) + direction, state, filename = _args + if direction == "input": + checker = self.checker_input + else: + checker = self.checker + path_obj = checker._get_remote_file_path(state, filename) self.assertEqual(path_obj.as_posix(), expected) with self.assertRaises(AssertionError): - self.checker._remote_file_path("WHATEVER", "error", "foo.csv") - - with self.assertRaises(AssertionError): - self.checker._remote_file_path("input", "WHATEVER", "foo.csv") + self.checker_input._get_remote_file_path("WHATEVER", "foo.csv") def test_get_remote_file(self): with mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get") as mocked: diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 2f396d39d..0d4793d3f 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -212,22 +212,25 @@ def test_create_input_exchange_file_from_file_received_no_pattern(self): mocked_paths = { input_dir: "/tmp/", self._file_fullpath( - "pending", direction="input", fname=file_names[0] + "pending", fname=file_names[0], checker=self.checker_input ): self.fakepath_input_pending_1, self._file_fullpath( - "pending", direction="input", fname=file_names[1] + "pending", fname=file_names[1], checker=self.checker_input ): self.fakepath_input_pending_2, } - existing = self.env["edi.exchange.record"].search_count( + existing_records = self.env["edi.exchange.record"].search( [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] ) - self.assertEqual(existing, 0) # Run cron action: found_files = [input_dir + fname for fname in file_names] with self._mock_storage_backend_find_files(found_files): self._test_run_cron_pending_input(mocked_paths) new_records = self.env["edi.exchange.record"].search( - [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + [ + ("backend_id", "=", self.backend.id), + ("type_id", "=", exch_type.id), + ("id", "not in", existing_records.ids), + ] ) self.assertEqual(len(new_records), 2) for rec in new_records: diff --git a/edi_storage_oca/tests/test_exchange_type.py b/edi_storage_oca/tests/test_exchange_type.py new file mode 100644 index 000000000..6b886eab3 --- /dev/null +++ b/edi_storage_oca/tests/test_exchange_type.py @@ -0,0 +1,67 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonTestCase + + +class EDIExchangeTypeTestCase(EDIBackendCommonTestCase): + def _check_test_storage_fullpath(self, wanted_fullpath, directory, filename): + fullpath = self.exchange_type_out._storage_fullpath(directory, filename) + self.assertEqual(fullpath.as_posix(), wanted_fullpath) + + def _do_test_storage_fullpath(self, prefix=""): + # Test with no directory and no filename + wanted_fullpath = prefix or "." + self._check_test_storage_fullpath(wanted_fullpath, None, None) + + # Test with directory + directory = "test_directory" + wanted_fullpath = f"{prefix}/{directory}" if prefix else directory + self._check_test_storage_fullpath(wanted_fullpath, directory, None) + + # Test with filename + filename = "test_filename.csv" + wanted_fullpath = f"{prefix}/{filename}" if prefix else filename + self._check_test_storage_fullpath(wanted_fullpath, None, filename) + + # Test with directory and filename + wanted_fullpath = ( + f"{prefix}/{directory}/{filename}" if prefix else f"{directory}/{filename}" + ) + self._check_test_storage_fullpath(wanted_fullpath, directory, filename) + + def test_storage_fullpath(self): + """ + Test storage fullpath defined into advanced settings. + Example of pattern: + storage: + # simple string + path: path/to/file + # name of the param containing the path + path_config_param: path/to/file + """ + + # Test without any prefix + self._do_test_storage_fullpath() + + # Force path on advanced settings + prefix = "prefix/path" + self.exchange_type_out.advanced_settings_edit = f""" + storage: + path: {prefix} + """ + self._do_test_storage_fullpath(prefix=prefix) + + # Force path on advanced settings using config param, but not defined + self.exchange_type_out.advanced_settings_edit = """ + storage: + path_config_param: prefix_path_config_param + """ + self._do_test_storage_fullpath() + + # Define config param + prefix = "prefix/path/by/config/param" + self.env["ir.config_parameter"].sudo().set_param( + "prefix_path_config_param", prefix + ) + self._do_test_storage_fullpath(prefix=prefix) From 13fef0df2f4000b22e58bd52f5129a602e6ad987 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Sun, 3 Sep 2023 12:44:19 +0000 Subject: [PATCH 42/77] [UPD] README.rst --- edi_storage_oca/README.rst | 15 +++++--- edi_storage_oca/static/description/index.html | 38 ++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index c182ec9be..dcd6d47fd 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -2,10 +2,13 @@ EDI Storage backend support =========================== -.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:f2718fe12ff53362a2da3e253d22e0641ebcd6a01195db14f8d9c8c71878080d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status @@ -19,11 +22,11 @@ EDI Storage backend support .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png :target: https://translation.odoo-community.org/projects/edi-15-0/edi-15-0-edi_storage_oca :alt: Translate me on Weblate -.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/15.0 - :alt: Try me on Runbot +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi&target_branch=15.0 + :alt: Try me on Runboat -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. @@ -64,7 +67,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed +If you spotted it first, help us to smash it by providing a detailed and welcomed `feedback `_. Do not contact contributors directly about support or help with technical issues. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index d08dd56bb..3a323e627 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -1,20 +1,20 @@ - + - + EDI Storage backend support -
    -

    EDI Storage backend support

    +
    + + +Odoo Community Association + +
    +

    EDI Storage backend support

    -

    Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

    +

    Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

    Allow exchange files using storage backends from OCA/storage.

    This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) and the @@ -406,18 +411,18 @@

    EDI Storage backend support

-

Usage

+

Usage

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

-

Known issues / Roadmap

+

Known issues / Roadmap

  • clean deprecated methods in the storage
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed @@ -425,15 +430,15 @@

Bug Tracker

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • ACSONE
-

Contributors

+

Contributors

-

Other credits

+

Other credits

The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp.

-

Maintainers

+

Maintainers

This module is maintained by the OCA.

Odoo Community Association @@ -464,5 +469,6 @@

Maintainers

+ From 386db9f65012edb7aba448ada927035fee916b38 Mon Sep 17 00:00:00 2001 From: Hector Vior Date: Sat, 6 Dec 2025 21:21:46 +0100 Subject: [PATCH 63/77] [FIX] edi_storage_oca: Remove legacy storage component code --- edi_storage_oca/models/edi_backend.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index ba983524f..e6767a789 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -59,27 +59,6 @@ class EDIBackend(models.Model): "Output error directory", help="Path to folder for error operations" ) - def _get_component_usage_candidates(self, exchange_record, key): - candidates = super()._get_component_usage_candidates(exchange_record, key) - if not self.storage_id or key not in self._storage_actions: - return candidates - return [f"storage.{key}"] + candidates - - def _component_match_attrs(self, exchange_record, key): - # Override to inject storage_type - res = super()._component_match_attrs(exchange_record, key) - if not self.storage_id or key not in self._storage_actions: - return res - res["storage_type"] = self.sudo().storage_id.protocol - return res - - def _component_sort_key(self, component_class): - res = super()._component_sort_key(component_class) - # Override to give precedence by storage_type when needed. - if not self.storage_id: - return res - return (1 if getattr(component_class, "_storage_type", False) else 0,) + res - def _storage_cron_check_pending_input(self, **kw): for backend in self: backend._storage_check_pending_input(**kw) From a9f086a17430ef9792c5810b1d83c0ed12a0aafa Mon Sep 17 00:00:00 2001 From: JordiMForgeFlow Date: Tue, 9 Dec 2025 11:35:36 +0100 Subject: [PATCH 64/77] [FIX] edi_storage_oca: fix migration script --- edi_storage_oca/migrations/18.0.1.0.0/post-migration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py index 7f1454af1..245c80bc4 100644 --- a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py +++ b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py @@ -42,6 +42,7 @@ def migrate(env, version): AND edi_backend.backend_type_id = edi_exchange_type.backend_type_id ) AND edi_backend.active IS TRUE + ) ) """, ) From c5daa2c6fa086051446cf851d8f5cc57b5fecda9 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 9 Dec 2025 11:16:17 +0000 Subject: [PATCH 65/77] [BOT] post-merge updates --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 7457af899..b3fb70c5c 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -11,7 +11,7 @@ EDI Storage backend support !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:c9c2898d23ca2c445e2088cefb226956d525da3fb8928068c1c4bcbb9d668f3c + !! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 151985031..95c5186ce 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.0", + "version": "18.0.1.0.1", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 080e90e83..fb16aaae0 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -372,7 +372,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:c9c2898d23ca2c445e2088cefb226956d525da3fb8928068c1c4bcbb9d668f3c +!! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

From a6bd6512e3081295d50563e95976f2e958aff1df Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 9 Dec 2025 12:44:24 +0000 Subject: [PATCH 66/77] [BOT] post-merge updates --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index b3fb70c5c..52b9f7ece 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -11,7 +11,7 @@ EDI Storage backend support !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 + !! source digest: sha256:ae52c0238c8bab278e4e2d6a48d86d4222672b893df1f171be046f3b7a3c58ae !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 95c5186ce..39f47e247 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.1", + "version": "18.0.1.0.2", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index fb16aaae0..87b66f49d 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -372,7 +372,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 +!! source digest: sha256:ae52c0238c8bab278e4e2d6a48d86d4222672b893df1f171be046f3b7a3c58ae !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

From a557fe75f57824c5316f728145e2a2fa0bb5ada3 Mon Sep 17 00:00:00 2001 From: Arnau Date: Tue, 13 Jan 2026 15:37:24 +0100 Subject: [PATCH 67/77] [FIX] edi_storage_oca: ls whitespace filename with ftp protocol --- edi_storage_oca/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/edi_storage_oca/utils.py b/edi_storage_oca/utils.py index 0b93e5da0..0b791c09b 100644 --- a/edi_storage_oca/utils.py +++ b/edi_storage_oca/utils.py @@ -48,6 +48,8 @@ def list_files(storage, relative_path="", pattern=False): if pattern: relative_path = fs.sep.join([relative_path, pattern]) return fs.glob(relative_path) + if fs.protocol == "ftp" and fs.ftp: + return fs.ftp.nlst(relative_path) return fs.ls(relative_path, detail=False) From 8021d65c4925affc9472483c14c50450e10b74cd Mon Sep 17 00:00:00 2001 From: Arnau Date: Tue, 12 May 2026 14:19:38 +0200 Subject: [PATCH 68/77] [FIX] edi_storage_oca: raise error if file is missing when performing receive --- .../models/edi_oca_storage_handler.py | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/edi_storage_oca/models/edi_oca_storage_handler.py b/edi_storage_oca/models/edi_oca_storage_handler.py index ce0f7682f..3e2a3d938 100644 --- a/edi_storage_oca/models/edi_oca_storage_handler.py +++ b/edi_storage_oca/models/edi_oca_storage_handler.py @@ -9,6 +9,7 @@ from pathlib import PurePath from odoo import models +from odoo.exceptions import UserError from .. import utils @@ -34,7 +35,9 @@ def send(self, exchange_record): utils.add_file(exchange_record.backend_id.storage_id, path.as_posix(), filedata) def receive(self, exchange_record): - return self._get_remote_file(exchange_record, "pending", binary=True) + return self._get_remote_file( + exchange_record, "pending", binary=True, raise_if_missing=True + ) def _dir_by_state(self, backend, direction, state): """Return remote directory path by direction and state. @@ -61,11 +64,21 @@ def _get_remote_file_path(self, exchange_record, state, filename=None): ) return path - def _get_remote_file(self, exchange_record, state, filename=None, binary=False): + def _get_remote_file( + self, + exchange_record, + state, + filename=None, + binary=False, + raise_if_missing=False, + ): """Get file for current exchange_record in the given destination state. :param state: string ("pending", "done", "error") :param filename: custom file name, exchange_record filename used by default + :param raise_if_missing: when True, storage errors are re-raised as + swallable exceptions so the caller transitions the record to an + error state instead of treating "no content" as a successful read. :return: remote file content as string """ path = self._get_remote_file_path(exchange_record, state, filename=filename) @@ -84,14 +97,25 @@ def _get_remote_file(self, exchange_record, state, filename=None, binary=False): path, state, ) + if raise_if_missing: + raise return None - except OSError: - _logger.info( - "Ignored OSError when trying to get file %s into path %s for state %s", + except OSError as err: + _logger.warning( + "OSError when trying to get file %s into path %s for state %s: %s", filename, path, state, + err, ) + if raise_if_missing: + raise UserError( + self.env._( + "Storage error while reading %(path)s: %(error)s", + path=path.as_posix(), + error=str(err), + ) + ) from err return None def check(self, exchange_record): From 92de08643b5d571a264373b87c0ca24ab372e341 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 27 May 2026 06:41:14 +0000 Subject: [PATCH 69/77] [BOT] post-merge updates --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 52b9f7ece..ff34c4027 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -11,7 +11,7 @@ EDI Storage backend support !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:ae52c0238c8bab278e4e2d6a48d86d4222672b893df1f171be046f3b7a3c58ae + !! source digest: sha256:2511ea918ba213d754909a73c92b80e89a7ab320816acd446f9e2dcf26c538d0 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 39f47e247..1f7b1d333 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.2", + "version": "18.0.1.0.3", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 87b66f49d..9d7a9d130 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -372,7 +372,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:ae52c0238c8bab278e4e2d6a48d86d4222672b893df1f171be046f3b7a3c58ae +!! source digest: sha256:2511ea918ba213d754909a73c92b80e89a7ab320816acd446f9e2dcf26c538d0 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

From 121f538972a1b8cbea3c053f36adc62706d4de17 Mon Sep 17 00:00:00 2001 From: oca-ci Date: Wed, 27 May 2026 06:49:48 +0000 Subject: [PATCH 70/77] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index cba90f769..7b3703040 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -114,6 +114,12 @@ msgstr "" msgid "Storage Handler for EDI" msgstr "" +#. module: edi_storage_oca +#. odoo-python +#: code:addons/edi_storage_oca/models/edi_oca_storage_handler.py:0 +msgid "Storage error while reading %(path)s: %(error)s" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id msgid "Storage for in-out files" From 0dbd5dfdbf43a7ac9f11bf16a76b09e32e7c4de7 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 27 May 2026 06:56:45 +0000 Subject: [PATCH 71/77] [BOT] post-merge updates --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index ff34c4027..f880503b3 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -11,7 +11,7 @@ EDI Storage backend support !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:2511ea918ba213d754909a73c92b80e89a7ab320816acd446f9e2dcf26c538d0 + !! source digest: sha256:ca9c4e0d1ecd9744b5cfa517900a5dace2a206007360ce2c1299dc1da5910bfa !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 1f7b1d333..87b4c5f92 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.3", + "version": "18.0.1.0.4", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 9d7a9d130..fe56fe211 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -372,7 +372,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:2511ea918ba213d754909a73c92b80e89a7ab320816acd446f9e2dcf26c538d0 +!! source digest: sha256:ca9c4e0d1ecd9744b5cfa517900a5dace2a206007360ce2c1299dc1da5910bfa !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

From a9dec245a9f3b32ad388305196a51331c8c9e105 Mon Sep 17 00:00:00 2001 From: Weblate Date: Wed, 27 May 2026 06:56:55 +0000 Subject: [PATCH 72/77] Update translation files Updated by "Update PO files to match POT (msgmerge)" hook in Weblate. Translation: edi-framework-18.0/edi-framework-18.0-edi_storage_oca Translate-URL: https://translation.odoo-community.org/projects/edi-framework-18-0/edi-framework-18-0-edi_storage_oca/ --- edi_storage_oca/i18n/it.po | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/i18n/it.po b/edi_storage_oca/i18n/it.po index e35011ab4..ce7ead09f 100644 --- a/edi_storage_oca/i18n/it.po +++ b/edi_storage_oca/i18n/it.po @@ -28,7 +28,6 @@ msgstr "Tipo scambio EDI" #. module: edi_storage_oca #: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server -#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input msgid "EDI backend storage check pending input" msgstr "Controllo ingressi in attesa deposito backend EDI" @@ -51,8 +50,13 @@ msgstr "Deposito FS" #. module: edi_storage_oca #: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern msgid "" -"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" -"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +"For output exchange types this should be a formatting string with the " +"following variables available (to be used between brackets, `{}`): " +"`exchange_record`, `record_name`, `type` and `dt`. For instance, a valid " +"string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex " +"expression to filter the files to be fetched from the pending directory in " +"the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" msgstr "" "Per i tipi di scambio in uscita questo dovrebbe essere una stringa di " "formattazione con le seguenti variabili disponibili (da utilizzare tra " @@ -120,6 +124,17 @@ msgstr "Record creato da un file trovato nel deposito FS" msgid "Storage" msgstr "Deposito" +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_oca_storage_handler +msgid "Storage Handler for EDI" +msgstr "" + +#. module: edi_storage_oca +#. odoo-python +#: code:addons/edi_storage_oca/models/edi_oca_storage_handler.py:0 +msgid "Storage error while reading %(path)s: %(error)s" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id msgid "Storage for in-out files" From 23643be20e820719f01dd6c797376ebc446fe474 Mon Sep 17 00:00:00 2001 From: Arnau Date: Tue, 19 May 2026 13:01:31 +0200 Subject: [PATCH 73/77] [IMP] edi_storage_oca: post-process input file via conf --- edi_storage_oca/__manifest__.py | 1 + edi_storage_oca/data/edi_configuration.xml | 29 +++++++ edi_storage_oca/models/edi_backend.py | 50 +++++++++++ edi_storage_oca/readme/CONFIGURE.md | 10 +++ edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/test_event_listener.py | 88 ++++++++++++++++++++ edi_storage_oca/utils.py | 12 +++ 7 files changed, 191 insertions(+) create mode 100644 edi_storage_oca/data/edi_configuration.xml create mode 100644 edi_storage_oca/readme/CONFIGURE.md create mode 100644 edi_storage_oca/tests/test_event_listener.py diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 87b4c5f92..7f971e130 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -15,6 +15,7 @@ "depends": ["edi_core_oca", "fs_storage"], "data": [ "data/cron.xml", + "data/edi_configuration.xml", "security/ir_model_access.xml", "views/edi_backend_views.xml", ], diff --git a/edi_storage_oca/data/edi_configuration.xml b/edi_storage_oca/data/edi_configuration.xml new file mode 100644 index 000000000..53956f2eb --- /dev/null +++ b/edi_storage_oca/data/edi_configuration.xml @@ -0,0 +1,29 @@ + + + + + Storage: move input file to done + + + + record.backend_id._storage_on_edi_exchange_done(record) + + + + Storage: move input file to error + + + + record.backend_id._storage_on_edi_exchange_error(record) + + diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index e6767a789..a6d5b40fc 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -148,3 +148,53 @@ def _storage_new_exchange_record_vals(self, file_name): "edi_exchange_state": "input_pending", "storage_id": self.storage_id.id, } + + def _storage_on_edi_exchange_done(self, exchange_record): + """ + Move an input file from the pending dir to the done dir. + + Intended to be invoked from a global 'edi.configuration' snippet + bound to the 'on_edi_exchange_done' trigger. + """ + self.ensure_one() + storage = exchange_record.storage_id + if exchange_record.direction != "input" or not storage: + return False + if not self.input_dir_done: + return False + file_name = exchange_record.exchange_filename + pending_dir = exchange_record.type_id._storage_fullpath( + self.input_dir_pending + ).as_posix() + done_dir = exchange_record.type_id._storage_fullpath( + self.input_dir_done + ).as_posix() + error_dir = exchange_record.type_id._storage_fullpath( + self.input_dir_error + ).as_posix() + res = utils.move_file(storage, pending_dir, done_dir, file_name) + if not res and self.input_dir_error: + res = utils.move_file(storage, error_dir, done_dir, file_name) + return res + + def _storage_on_edi_exchange_error(self, exchange_record): + """ + Move an input file from the pending dir to the error dir. + + Intended to be invoked from a global 'edi.configuration' snippet + bound to the 'on_edi_exchange_error' trigger. + """ + self.ensure_one() + storage = exchange_record.storage_id + if exchange_record.direction != "input" or not storage: + return False + if not self.input_dir_error: + return False + file_name = exchange_record.exchange_filename + pending_dir = exchange_record.type_id._storage_fullpath( + self.input_dir_pending + ).as_posix() + error_dir = exchange_record.type_id._storage_fullpath( + self.input_dir_error + ).as_posix() + return utils.move_file(storage, pending_dir, error_dir, file_name) diff --git a/edi_storage_oca/readme/CONFIGURE.md b/edi_storage_oca/readme/CONFIGURE.md new file mode 100644 index 000000000..4050a7424 --- /dev/null +++ b/edi_storage_oca/readme/CONFIGURE.md @@ -0,0 +1,10 @@ +This module has two **inactive** global `edi.configuration` records +that move the input file across the storage directories on +`on_edi_exchange_done` / `on_edi_exchange_error`: + +- *Storage: move input file, pending → done (fallback error → done) +- *Storage: move input file, pending → error + +Before enabling them you **must** set `backend_id` on the record: +otherwise the global match runs against every backend in the database, +including non-storage ones. diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index 0a73fb8a1..54510bc1a 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1,2 +1,3 @@ from . import test_edi_backend_storage +from . import test_event_listener from . import test_exchange_type diff --git a/edi_storage_oca/tests/test_event_listener.py b/edi_storage_oca/tests/test_event_listener.py new file mode 100644 index 000000000..af9be8741 --- /dev/null +++ b/edi_storage_oca/tests/test_event_listener.py @@ -0,0 +1,88 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# Copyright 2026 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 +from unittest import mock + +from odoo_test_helper import FakeModelLoader + +from odoo.addons.edi_core_oca.tests.common import EDIBackendCommonTestCase +from odoo.addons.edi_core_oca.tests.fake_models import EdiTestExecution + +STORAGE_MOVE_FILE_PATH = "odoo.addons.edi_storage_oca.utils.move_file" + + +class TestStorageEventListener(EDIBackendCommonTestCase): + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.conf_done = cls.env.ref("edi_storage_oca.edi_conf_storage_move_on_done") + cls.conf_error = cls.env.ref("edi_storage_oca.edi_conf_storage_move_on_error") + (cls.conf_done | cls.conf_error).write( + {"active": True, "backend_id": cls.backend.id} + ) + + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": base64.b64encode(b"1234"), + "storage_id": cls.backend.storage_id.id, + } + cls.record = cls.backend.create_record("test_csv_input", vals) + + def setUp(self): + super().setUp() + self.loader = FakeModelLoader(self.env, self.__module__) + self.loader.backup_registry() + + self.loader.update_registry((EdiTestExecution,)) + fake_model = self.env["ir.model"].search( + [("model", "=", "edi.framework.test.execution")] + ) + self.exchange_type_in.process_model_id = fake_model + self.exchange_type_in.input_validate_model_id = fake_model + + def tearDown(self): + self.loader.restore_registry() + super().tearDown() + + def _patch_move_file(self): + return mock.patch(STORAGE_MOVE_FILE_PATH, autospec=True, return_value=True) + + def _expected_dir(self, raw_dir): + return self.exchange_type_in._storage_fullpath(raw_dir).as_posix() + + def test_01_process_record_success(self): + self.record.write({"edi_exchange_state": "input_received"}) + with self._patch_move_file() as mocked: + self.record.action_exchange_process() + mocked.assert_called_once() + storage, from_dir_str, to_dir_str, filename = mocked.call_args[0] + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual( + from_dir_str, self._expected_dir(self.backend.input_dir_pending) + ) + self.assertEqual(to_dir_str, self._expected_dir(self.backend.input_dir_done)) + self.assertEqual(filename, self.record.exchange_filename) + + def test_02_process_record_with_error(self): + self.record.write({"edi_exchange_state": "input_received"}) + self.record._set_file_content("TEST %d" % self.record.id) + with self._patch_move_file() as mocked: + self.record.with_context( + test_break_process="OOPS! Something went wrong :(" + ).action_exchange_process() + mocked.assert_called_once() + storage, from_dir_str, to_dir_str, filename = mocked.call_args[0] + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual( + from_dir_str, self._expected_dir(self.backend.input_dir_pending) + ) + self.assertEqual(to_dir_str, self._expected_dir(self.backend.input_dir_error)) + self.assertEqual(filename, self.record.exchange_filename) diff --git a/edi_storage_oca/utils.py b/edi_storage_oca/utils.py index 0b791c09b..179c25904 100644 --- a/edi_storage_oca/utils.py +++ b/edi_storage_oca/utils.py @@ -2,8 +2,10 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl) import base64 +import functools import os import re +from pathlib import PurePath def add_file(storage, path, filedata, binary=False): @@ -60,3 +62,13 @@ def move_files(storage, files, destination_path, **kw): storage.fs.sep.join([destination_path, os.path.basename(file_path)]), **kw, ) + + +# TODO: drop this helper once https://github.com/OCA/storage/pull/606 is merged. +def move_file(storage, from_dir_str, to_dir_str, filename): + src = (PurePath(from_dir_str) / filename).as_posix() + if not storage.fs.exists(src): + return False + dst = (PurePath(to_dir_str) / filename).as_posix() + storage.env.cr.postcommit.add(functools.partial(storage.fs.move, src, dst)) + return True From 8bef029996948b05ac11aace59ef9de2d3fcfe15 Mon Sep 17 00:00:00 2001 From: mymage Date: Fri, 29 May 2026 11:39:58 +0000 Subject: [PATCH 74/77] Translated using Weblate (Italian) Currently translated at 100.0% (21 of 21 strings) Translation: edi-framework-18.0/edi-framework-18.0-edi_storage_oca Translate-URL: https://translation.odoo-community.org/projects/edi-framework-18-0/edi-framework-18-0-edi_storage_oca/it/ --- edi_storage_oca/i18n/it.po | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/i18n/it.po b/edi_storage_oca/i18n/it.po index ce7ead09f..64925eccc 100644 --- a/edi_storage_oca/i18n/it.po +++ b/edi_storage_oca/i18n/it.po @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: Odoo Server 16.0\n" "Report-Msgid-Bugs-To: \n" -"PO-Revision-Date: 2024-11-15 17:06+0000\n" +"PO-Revision-Date: 2026-05-29 13:46+0000\n" "Last-Translator: mymage \n" "Language-Team: none\n" "Language: it\n" @@ -14,7 +14,7 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: \n" "Plural-Forms: nplurals=2; plural=n != 1;\n" -"X-Generator: Weblate 5.6.2\n" +"X-Generator: Weblate 5.15.2\n" #. module: edi_storage_oca #: model:ir.model,name:edi_storage_oca.model_edi_backend @@ -127,13 +127,13 @@ msgstr "Deposito" #. module: edi_storage_oca #: model:ir.model,name:edi_storage_oca.model_edi_oca_storage_handler msgid "Storage Handler for EDI" -msgstr "" +msgstr "Gestore archivio per EDI" #. module: edi_storage_oca #. odoo-python #: code:addons/edi_storage_oca/models/edi_oca_storage_handler.py:0 msgid "Storage error while reading %(path)s: %(error)s" -msgstr "" +msgstr "Errore archivio nella lettura di %(path)s: %(error)s" #. module: edi_storage_oca #: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id From 21394fc1ffd58df3371c57581d9632a15edddd38 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 3 Jun 2026 14:28:54 +0000 Subject: [PATCH 75/77] [BOT] post-merge updates --- edi_storage_oca/README.rst | 16 ++++++- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 48 ++++++++++++------- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index f880503b3..1194f6dae 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -11,7 +11,7 @@ EDI Storage backend support !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:ca9c4e0d1ecd9744b5cfa517900a5dace2a206007360ce2c1299dc1da5910bfa + !! source digest: sha256:f0ff3ee30416f5a8090d6c738df2c392381b61bc0d104c7679a147ce3b503018 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png @@ -62,6 +62,20 @@ in/from the right place and update exchange records data accordingly. .. contents:: :local: +Configuration +============= + +This module has two **inactive** global ``edi.configuration`` records +that move the input file across the storage directories on +``on_edi_exchange_done`` / ``on_edi_exchange_error``: + +- \*Storage: move input file, pending → done (fallback error → done) +- \*Storage: move input file, pending → error + +Before enabling them you **must** set ``backend_id`` on the record: +otherwise the global match runs against every backend in the database, +including non-storage ones. + Usage ===== diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 7f971e130..ea681386e 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.4", + "version": "18.0.1.1.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index fe56fe211..77611d9c7 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -372,7 +372,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:ca9c4e0d1ecd9744b5cfa517900a5dace2a206007360ce2c1299dc1da5910bfa +!! source digest: sha256:f0ff3ee30416f5a8090d6c738df2c392381b61bc0d104c7679a147ce3b503018 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

@@ -398,31 +398,45 @@

EDI Storage backend support

Table of contents

+
+

Configuration

+

This module has two inactive global edi.configuration records +that move the input file across the storage directories on +on_edi_exchange_done / on_edi_exchange_error:

+
    +
  • *Storage: move input file, pending → done (fallback error → done)
    • +
  • *Storage: move input file, pending → error
    • +
+

Before enabling them you must set backend_id on the record: +otherwise the global match runs against every backend in the database, +including non-storage ones.

+
-

Usage

+

Usage

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

-

Known issues / Roadmap

+

Known issues / Roadmap

  • clean deprecated methods in the storage
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed @@ -430,15 +444,15 @@

Bug Tracker

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • ACSONE
-

Contributors

+

Contributors

-

Other credits

+

Other credits

The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp.

-

Maintainers

+

Maintainers

This module is maintained by the OCA.

Odoo Community Association From 680c18fd3017fe87085fba32d3062a94a37c80e5 Mon Sep 17 00:00:00 2001 From: Sergij Pfaifer Date: Mon, 8 Jun 2026 15:59:09 +0300 Subject: [PATCH 76/77] [IMP] edi_storage_oca: pre-commit auto fixes --- edi_storage_oca/migrations/18.0.1.0.0/post-migration.py | 2 +- edi_storage_oca/tests/test_event_listener.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py index 245c80bc4..f33bd2d76 100644 --- a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py +++ b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py @@ -8,7 +8,7 @@ def migrate(env, version): env.cr, f""" UPDATE edi_exchange_type - SET send_model_id = { model.id } + SET send_model_id = {model.id} WHERE direction = 'output' AND send_model_id IS NULL AND EXISTS ( diff --git a/edi_storage_oca/tests/test_event_listener.py b/edi_storage_oca/tests/test_event_listener.py index af9be8741..2a756b9c2 100644 --- a/edi_storage_oca/tests/test_event_listener.py +++ b/edi_storage_oca/tests/test_event_listener.py @@ -73,7 +73,7 @@ def test_01_process_record_success(self): def test_02_process_record_with_error(self): self.record.write({"edi_exchange_state": "input_received"}) - self.record._set_file_content("TEST %d" % self.record.id) + self.record._set_file_content("TEST %d") with self._patch_move_file() as mocked: self.record.with_context( test_break_process="OOPS! Something went wrong :(" From 700bb1c5d0d4b1327e7bd47d8956e0b9ca2c2c36 Mon Sep 17 00:00:00 2001 From: Sergij Pfaifer Date: Mon, 8 Jun 2026 17:04:53 +0300 Subject: [PATCH 77/77] [MIG] edi_storage_oca: Migration to 19.0 --- edi_storage_oca/README.rst | 10 ++-- edi_storage_oca/__manifest__.py | 3 +- edi_storage_oca/demo/edi_backend_demo.xml | 14 ----- .../migrations/18.0.1.0.0/post-migration.py | 48 ---------------- edi_storage_oca/static/description/index.html | 6 +- edi_storage_oca/tests/common.py | 27 ++++++++- .../tests/test_edi_backend_storage.py | 14 ++++- edi_storage_oca/tests/test_event_listener.py | 57 +++++++++++++------ 8 files changed, 88 insertions(+), 91 deletions(-) delete mode 100644 edi_storage_oca/demo/edi_backend_demo.xml delete mode 100644 edi_storage_oca/migrations/18.0.1.0.0/post-migration.py diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 1194f6dae..16a20af7c 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -21,13 +21,13 @@ EDI Storage backend support :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi--framework-lightgray.png?logo=github - :target: https://github.com/OCA/edi-framework/tree/18.0/edi_storage_oca + :target: https://github.com/OCA/edi-framework/tree/19.0/edi_storage_oca :alt: OCA/edi-framework .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/edi-framework-18-0/edi-framework-18-0-edi_storage_oca + :target: https://translation.odoo-community.org/projects/edi-framework-19-0/edi-framework-19-0-edi_storage_oca :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png - :target: https://runboat.odoo-community.org/builds?repo=OCA/edi-framework&target_branch=18.0 + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi-framework&target_branch=19.0 :alt: Try me on Runboat |badge1| |badge2| |badge3| |badge4| |badge5| @@ -93,7 +93,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -135,6 +135,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/edi-framework `_ project on GitHub. +This module is part of the `OCA/edi-framework `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index ea681386e..d0dd35013 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.1.0", + "version": "19.0.1.0.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", @@ -19,5 +19,4 @@ "security/ir_model_access.xml", "views/edi_backend_views.xml", ], - "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml deleted file mode 100644 index a8740b0d4..000000000 --- a/edi_storage_oca/demo/edi_backend_demo.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - Storage Demo EDI backend - - - demo_in/pending - demo_in/done - demo_in/error - demo_out/pending - demo_out/done - demo_out/error - - diff --git a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py deleted file mode 100644 index f33bd2d76..000000000 --- a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py +++ /dev/null @@ -1,48 +0,0 @@ -from openupgradelib import openupgrade - - -@openupgrade.migrate() -def migrate(env, version): - model = env.ref("edi_storage_oca.model_edi_oca_storage_handler") - openupgrade.logged_query( - env.cr, - f""" - UPDATE edi_exchange_type - SET send_model_id = {model.id} - WHERE direction = 'output' - AND send_model_id IS NULL - AND EXISTS ( - SELECT 1 - FROM edi_backend - WHERE ( - edi_backend.id = edi_exchange_type.backend_id - OR ( - edi_exchange_type.backend_id IS NULL - AND edi_backend.backend_type_id = edi_exchange_type.backend_type_id - ) - AND edi_backend.active IS TRUE - ) - ) - """, - ) - openupgrade.logged_query( - env.cr, - f""" - UPDATE edi_exchange_type - SET receive_model_id = {model.id} - WHERE direction = 'input' - AND receive_model_id IS NULL - AND EXISTS ( - SELECT 1 - FROM edi_backend - WHERE ( - edi_backend.id = edi_exchange_type.backend_id - OR ( - edi_exchange_type.backend_id IS NULL - AND edi_backend.backend_type_id = edi_exchange_type.backend_type_id - ) - AND edi_backend.active IS TRUE - ) - ) - """, - ) diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 77611d9c7..45a839ea3 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -374,7 +374,7 @@

EDI Storage backend support

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! source digest: sha256:f0ff3ee30416f5a8090d6c738df2c392381b61bc0d104c7679a147ce3b503018 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

+

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) and the @@ -440,7 +440,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -478,7 +478,7 @@

Maintainers

OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

-

This module is part of the OCA/edi-framework project on GitHub.

+

This module is part of the OCA/edi-framework project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d8606d4ba..64e8a159d 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -12,7 +12,32 @@ class TestEDIStorageBase(EDIBackendCommonTestCase): @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + backend_type = cls.env["edi.backend.type"].create( + { + "name": "Demo EDI backend type", + "code": "demo_backend", + } + ) + storage = cls.env["fs.storage"].create( + { + "name": "Odoo Filesystem Backend", + "protocol": "odoofs", + "code": "odoofs", + } + ) + return cls.env["edi.backend"].create( + { + "name": "Storage Demo EDI backend", + "backend_type_id": backend_type.id, + "storage_id": storage.id, + "input_dir_pending": "demo_in/pending", + "input_dir_done": "demo_in/done", + "input_dir_error": "demo_in/error", + "output_dir_pending": "demo_out/pending", + "output_dir_done": "demo_out/done", + "output_dir_error": "demo_out/error", + } + ) @classmethod def _setup_records(cls): diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index e5fbe0a21..b84bd7bed 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -141,8 +141,18 @@ def test_cron_full_flow(self): """Already sent, update the state via cron.""" self.record.edi_exchange_state = "output_sent" rec1 = self.record - partner2 = self.env.ref("base.res_partner_2") - partner3 = self.env.ref("base.res_partner_3") + partner2, partner3 = self.env["res.partner"].create( + [ + { + "name": "Acme Corporation", + "is_company": True, + }, + { + "name": "Gemini Furniture", + "is_company": True, + }, + ] + ) rec2 = self.record.copy( { "model": partner2._name, diff --git a/edi_storage_oca/tests/test_event_listener.py b/edi_storage_oca/tests/test_event_listener.py index 2a756b9c2..0f1221888 100644 --- a/edi_storage_oca/tests/test_event_listener.py +++ b/edi_storage_oca/tests/test_event_listener.py @@ -6,10 +6,9 @@ import base64 from unittest import mock -from odoo_test_helper import FakeModelLoader +from odoo.orm.model_classes import add_to_registry from odoo.addons.edi_core_oca.tests.common import EDIBackendCommonTestCase -from odoo.addons.edi_core_oca.tests.fake_models import EdiTestExecution STORAGE_MOVE_FILE_PATH = "odoo.addons.edi_storage_oca.utils.move_file" @@ -17,7 +16,32 @@ class TestStorageEventListener(EDIBackendCommonTestCase): @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + backend_type = cls.env["edi.backend.type"].create( + { + "name": "Demo EDI backend type", + "code": "demo_backend", + } + ) + storage = cls.env["fs.storage"].create( + { + "name": "Odoo Filesystem Backend", + "protocol": "odoofs", + "code": "odoofs", + } + ) + return cls.env["edi.backend"].create( + { + "name": "Storage Demo EDI backend", + "backend_type_id": backend_type.id, + "storage_id": storage.id, + "input_dir_pending": "demo_in/pending", + "input_dir_done": "demo_in/done", + "input_dir_error": "demo_in/error", + "output_dir_pending": "demo_out/pending", + "output_dir_done": "demo_out/done", + "output_dir_error": "demo_out/error", + } + ) @classmethod def setUpClass(cls): @@ -36,21 +60,22 @@ def setUpClass(cls): } cls.record = cls.backend.create_record("test_csv_input", vals) - def setUp(self): - super().setUp() - self.loader = FakeModelLoader(self.env, self.__module__) - self.loader.backup_registry() + @classmethod + def _setup_records(cls): # pylint:disable=missing-return + super()._setup_records() + # Load fake models ->/ + from odoo.addons.edi_core_oca.tests.fake_models import EdiTestExecution - self.loader.update_registry((EdiTestExecution,)) - fake_model = self.env["ir.model"].search( - [("model", "=", "edi.framework.test.execution")] + add_to_registry(cls.registry, EdiTestExecution) + cls.registry._setup_models__(cls.env.cr, ["edi.framework.test.execution"]) + cls.registry.init_models( + cls.env.cr, ["edi.framework.test.execution"], {"models_to_check": True} ) - self.exchange_type_in.process_model_id = fake_model - self.exchange_type_in.input_validate_model_id = fake_model - - def tearDown(self): - self.loader.restore_registry() - super().tearDown() + cls.addClassCleanup(cls.registry.__delitem__, "edi.framework.test.execution") + cls.ExecutionAbstractModel = cls.env["edi.framework.test.execution"] + cls.model = cls.env["ir.model"]._get("edi.framework.test.execution") + cls.exchange_type_in.process_model_id = cls.model + cls.exchange_type_in.input_validate_model_id = cls.model def _patch_move_file(self): return mock.patch(STORAGE_MOVE_FILE_PATH, autospec=True, return_value=True)