diff --git a/base_external_dbsource_importer/README.rst b/base_external_dbsource_importer/README.rst new file mode 100644 index 000000000..24e079b7e --- /dev/null +++ b/base_external_dbsource_importer/README.rst @@ -0,0 +1,93 @@ +=============================== +Base External Dbsource Importer +=============================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:4b5317a7f0ec00abe7f7b879b78bb04e606b03d8b3574560cfd9d8fcd64a0225 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github + :target: https://github.com/OCA/server-backend/tree/18.0/base_external_dbsource_importer + :alt: OCA/server-backend +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-backend-18-0/server-backend-18-0-base_external_dbsource_importer + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-backend&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module extends the functionality of base_external_dbsource + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Configuration +============= + +To configure this module, you need to: + +1. Database sources can be configured in Settings > Technical > Database + Structure > Database sources. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Tecnativa + +Contributors +------------ + +- `Tecnativa `__: + + - Sergio Teruel + - Carlos Dauden + - David Bañón Gil + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/server-backend `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_external_dbsource_importer/__init__.py b/base_external_dbsource_importer/__init__.py new file mode 100644 index 000000000..31660d6a9 --- /dev/null +++ b/base_external_dbsource_importer/__init__.py @@ -0,0 +1,3 @@ +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import models diff --git a/base_external_dbsource_importer/__manifest__.py b/base_external_dbsource_importer/__manifest__.py new file mode 100644 index 000000000..923850d4e --- /dev/null +++ b/base_external_dbsource_importer/__manifest__.py @@ -0,0 +1,16 @@ +# Copyright 2018 Tecnativa - Sergio Teruel +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +{ + "name": "Base External Dbsource Importer", + "summary": "Import data from external DB Sources", + "version": "18.0.1.0.0", + "development_status": "Alpha", + "category": "Tools", + "website": "https://github.com/OCA/server-backend", + "author": "Tecnativa, Odoo Community Association (OCA)", + "license": "AGPL-3", + "installable": True, + "depends": ["base_external_dbsource", "base_location", "base_vat"], + "external_dependencies": {"python": ["sqlalchemy"]}, + "data": ["security/ir.model.access.csv", "views/base_external_dbsource_view.xml"], +} diff --git a/base_external_dbsource_importer/models/__init__.py b/base_external_dbsource_importer/models/__init__.py new file mode 100644 index 000000000..011ba87c0 --- /dev/null +++ b/base_external_dbsource_importer/models/__init__.py @@ -0,0 +1,5 @@ +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from . import base_external_dbsource +from . import external_mixin +from . import mapped_mixin diff --git a/base_external_dbsource_importer/models/base_external_dbsource.py b/base_external_dbsource_importer/models/base_external_dbsource.py new file mode 100644 index 000000000..2fc8ea77c --- /dev/null +++ b/base_external_dbsource_importer/models/base_external_dbsource.py @@ -0,0 +1,385 @@ +# Copyright 2018 Tecnativa - Sergio Teruel +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +import base64 +import logging +import string +from collections import namedtuple +from queue import Queue +from threading import Event, Thread + +import xlrd +from psycopg2.extras import RealDictCursor +from sqlalchemy import text + +from odoo import api, fields, models +from odoo.exceptions import UserError +from odoo.tools import ormcache + +_logger = logging.getLogger(__name__) + +LETTERS = {ord(d): str(i) for i, d in enumerate(string.digits + string.ascii_uppercase)} +BackgroundFetch = namedtuple("BackgroundFetch", ["queue", "killswitch", "thread"]) + + +class BaseExternalModelImporter: + _name = "base.external.model.importer" + _external_key = None + _mapped_model = None + + def __init__(self, dbsource, file_path="", file_name=""): + self.env = dbsource.env + self.dbsource: BaseExternalDbsource = dbsource + self.file_path = file_path + self.file_name = file_name + + def execute_query(self, sql, params, metadata): + return True + + def _get_external_records(self, table_name, fields="*", where=""): + sql = f"SELECT {fields} FROM {table_name} {where};" + rows, cols = self.execute_query(text(sql), [], metadata=True) + return rows + + def _get_external_records_from_file(self): + """Return the same structure of db query but from a file. + To be implemented by other modules + """ + return {} + + def load_data( + self, + model_name, + table_name, + fields="*", + where="", + odoo_key="", + load_all_odoo_records=False, + origin=False, + ): + if not origin: + fds_records = self._get_external_records( + table_name, fields=fields, where=where + ) + else: + fds_records = self._get_external_records_from_file() + odoo_key = odoo_key or self._external_key + records, records_dic = self.load_odoo_records( + model_name, odoo_key, load_all_odoo_records + ) + return fds_records, records, records_dic + + def load_odoo_records(self, model_name, odoo_key=None, load_all=False): + return self.dbsource.load_odoo_records( + model_name, odoo_key or self._external_key, load_all + ) + + def upsert( + self, + fds_key, + records, + records_dic, + vals, + update_vals=False, + update_method=True, + specific_record=None, + only_update=False, + force_update=False, + ): + force_update = force_update or self.dbsource.force_update + model_name = records._name + record = specific_record or records.browse(records_dic.get(fds_key, False)) + if record: + if not update_method: + return record + # Performance issue + # record = records.filtered(lambda x: x.id == record_id) + values = (update_vals or vals).copy() + if not force_update: + for k, v in values.copy().items(): + if self.dbsource.skip_update_field(model_name, k) or ( + record._fields[k].convert_to_write(record[k], record) == v + ): + values.pop(k) + if values: + record.with_context(tracking_disable=True).write(values) + else: + Model = self.env[model_name] + if only_update: + return Model.browse() + record = Model.with_context(tracking_disable=True).create(vals) + records_dic[fds_key] = record.id + return record + + def get_m2_odoo_id( + self, + model_name, + key_value, + field_key=False, + mapped_model=False, + return_field="id", + ): + return self.dbsource.get_m2_odoo_id( + model_name, + key_value, + field_key or self._external_key, + mapped_model or self._mapped_model, + return_field, + ) + + def with_context(self, *args, **kwargs): + context = dict(args[0] if args else self.dbsource._context, **kwargs) + return self.dbsource.with_context(**context) + + +class BaseExternalDbsource(models.Model): + """Provides logic for connection to an external data source.""" + + _inherit = "base.external.dbsource" + + fields_to_update_ids = fields.One2many( + comodel_name="base.external.dbsource.fields.update", + inverse_name="db_source_id", + string="Fields To Update", + ) + only_update = fields.Boolean(string="Only update values", default=True) + force_update = fields.Boolean(string="Force update all values") + date_from = fields.Date() + date_to = fields.Date() + data_mapper_file = fields.Binary(string="Excel file with data mapped") + data_mapper_filename = fields.Char( + string="Excel file filename", + ) + + @ormcache("query", "execute_params", "metadata") + def execute_and_cache( + self, query=None, execute_params=None, metadata=False, **kwargs + ): + """Caches query response, but requires execute_params to be a tuple""" + return self.execute(query, list(execute_params), metadata, **kwargs) + + def action_clear_cache(self): + self.env.registry.clear_all_caches() + + @api.model + @ormcache("model_name", "key_value", "field_key", "mapped_model", "return_field") + def get_m2_odoo_id( + self, + model_name, + key_value, + field_key="fds_key", + mapped_model=False, + return_field="id", + ): + if not key_value: + return False + record = self.env[model_name].search_external( + key_value, field_key, mapped_model + ) + return record.id if return_field == "id" else record[return_field].id + + def _number_iban(self, iban): + return (iban[4:] + iban[:4]).translate(LETTERS) + + def generate_iban_check_digits(self, iban): + number_iban = self._number_iban(iban[:2] + "00" + iban[4:]) + return f"{98 - (int(number_iban) % 97):0>2}" + + @api.model + @ormcache("self.fields_to_update_ids.field_ids", "model_name", "field_name") + def skip_update_field(self, model_name, field_name): + return field_name not in self.fields_to_update_ids.filtered( + lambda x: x.model_id.model == model_name + ).mapped("field_ids.name") + + @api.model + def load_odoo_records(self, model_name, odoo_key, load_all=False): + Model = self.env[model_name].with_context( + active_test=False, prefetch_fields=False + ) + domain = [] + if not load_all: + domain = [(odoo_key, "!=", False)] + records = Model.search(domain) + records_dic = { + rec[odoo_key]: rec["id"] + for rec in records.read([odoo_key]) + if rec.get(odoo_key) + } + return records, records_dic + + @api.model + @ormcache("code", "country_code") + def _state_country_from_zip(self, code=None, country_code=None): + CityZip = city_zip = self.env["res.city.zip"] + if code: + domain = [("name", "=", code)] + if country_code: + domain.append(("country_id.code", "=", country_code)) + city_zip = CityZip.search(domain, limit=1) + country = city_zip.country_id + if not country and country_code: + country = self.env["res.country"].search( + [("code", "=", country_code)], limit=1 + ) + return ( + city_zip.city_id.state_id.id, + country.id, + country.code, + city_zip.id, + city_zip.city_id.id, + ) + + def _validate_vat(self, vals, country_code): + ResPartner = self.env["res.partner"] + original_vat = vals.pop("vat", "") or "" + vat = original_vat + # Clean vat + vat = ( + vat.replace("-", "") + .replace(".", "") + .replace(" ", "") + .replace("*", "") + .upper() + ) + if not vat: + return vals + if not vat[1:2].isnumeric(): + country_code, vat = ResPartner._split_vat(vat) + if not country_code: + country_code = "ES" + full_vat = f"{country_code.upper()}{vat}" + if ResPartner.simple_vat_check(country_code.lower(), vat): + vals["vat"] = full_vat + else: + if vals.get("comment", False): + vals["comment"] += f"\nVAT: {original_vat}" + else: + vals["comment"] = f"VAT: {original_vat}" + return vals + + def generate_data_mapped_from_file(self, sheet_dic): + """ + param: + sheet_dic: Dictionary type {'sheet_name': { + 'odoo_col': 1, + 'source_col': 2, + }} + """ + if not self.data_mapper_file: + raise UserError(self.env._("Debe seleccionar un archivo para importar")) + xl_workbook = xlrd.open_workbook( + file_contents=base64.b64decode(self.data_mapper_file) + ) + data_dic = {} + for sheet_name, cols_dic in sheet_dic.items(): + xl_sheet = xl_workbook.sheet_by_name(sheet_name) + odoo_col = cols_dic["odoo_col"] + source_col = cols_dic["source_col"] + data_dic[sheet_name] = {} + for row_idx in range(1, xl_sheet.nrows): + if xl_sheet.cell_type(row_idx, source_col) == xlrd.XL_CELL_EMPTY: + continue + odoo_ref = xl_sheet.cell(row_idx, odoo_col).value + if xl_sheet.cell_type(row_idx, source_col) != xlrd.XL_CELL_TEXT: + vila_code = str(int(xl_sheet.cell(row_idx, 2).value)) + else: + vila_code = xl_sheet.cell(row_idx, source_col).value + if xl_sheet.cell_type(row_idx, odoo_col) != xlrd.XL_CELL_TEXT: + odoo_external = False + else: + odoo_external = self.env.ref(odoo_ref, raise_if_not_found=False) + data_dic[sheet_name][vila_code] = odoo_external + return data_dic + + def server_side_cursor_mysql(self, table, fields_sql, where, size=2000): + with self.connection_open() as conn: + query = text(f"SELECT {fields_sql} FROM {table} {where}") + result = conn.execution_options( + stream_results=True, + max_row_buffer=size, + ).execute(query) + for row in result: + yield dict(row._mapping) + + def server_side_cursor_postgresql(self, table, fields, where, size=2000): + # Opens a server side cursor to stream the content of the table in batches + with self.connection_open() as conn: + with conn.cursor( + name="odoo_import", cursor_factory=RealDictCursor + ) as cursor: + cursor.itersize = size + query = f"SELECT {fields} FROM {table} {where} ;" + cursor.execute(query) + yield from cursor + + def server_side_cursor(self, table, fields, where, size=2000): + # To be overwriten in downstream modules + method = self._get_adapter_method("server_side_cursor") + yield from method(table, fields, where, size) + + def background_server_cursor( + self, + table, + field, + where, + killswitch, + queue, + size=2000, + ): + gen = self.server_side_cursor(table, field, where, size) + try: + for row in gen: + if killswitch.is_set(): + break + queue.put(row) + queue.put(None) + except Exception as e: + queue.put({"_fetch_error": e}) + finally: + gen.close() + + def background_fetch(self, table, fields, where, size=2000): + queue = Queue() + killswitch = Event() + fetch_thread = Thread( + target=self.background_server_cursor, + args=(table, fields, where, killswitch, queue, size), + ) + _logger.info("Starting fetch thread...") + fetch_thread.start() + return BackgroundFetch(queue, killswitch, fetch_thread) + + def queue_iterator(self, queue): + while True: + row = queue.get() + if row is None: + yield None + break + if "_fetch_error" in row: + raise row["_fetch_error"] + yield row + + def background_fetch_iterator(self, fecht_data: BackgroundFetch): + queue, killswitch, fetch_thread = fecht_data + try: + yield from self.queue_iterator(queue) + except Exception as e: + killswitch.set() + _logger.critical(f"Error on process thread: {e}") + raise e + finally: + killswitch.set() + fetch_thread.join() + + +class DbSourceFieldsUpdate(models.Model): + _name = "base.external.dbsource.fields.update" + _description = "Base External Dbsource Fields To Update" + + db_source_id = fields.Many2one( + comodel_name="base.external.dbsource", string="External Db Source" + ) + model_id = fields.Many2one(comodel_name="ir.model", string="Model") + field_ids = fields.Many2many( + comodel_name="ir.model.fields", string="Fields To Update" + ) diff --git a/base_external_dbsource_importer/models/external_mixin.py b/base_external_dbsource_importer/models/external_mixin.py new file mode 100644 index 000000000..2127a5d97 --- /dev/null +++ b/base_external_dbsource_importer/models/external_mixin.py @@ -0,0 +1,142 @@ +# Copyright 2018 Tecnativa - Sergio Teruel +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from collections import defaultdict + +from psycopg2.extensions import AsIs + +from odoo import api, models + + +class DbsourceExternalMixin(models.AbstractModel): + """Provides utilities for identifying records using a unique external key.""" + + _name = "dbsource.external.mixin" + _description = "Mixin for models that need to map external records" + + @api.model + def search_external(self, key_value, field_key, mapped_model=False): + """ + Mapped model is an model to mapp external records to only one + """ + mapped_model = self.env.context.get("mapped_model", mapped_model) + if mapped_model: + domain = [("external_key", "=", key_value)] + if "record_model" in self.env[mapped_model]._fields: + domain.append(("record_model", "=", self._name)) + mapped = self.env[mapped_model].search(domain) + if mapped: + key_value = mapped.mapped_key + domain = [(field_key, "=", key_value)] + return self.with_context(active_test=False).search(domain) + + def create_bypassed(self, vals_list): # noqa: C901 + # From v12 create method + bad_names = {"id", "parent_path"} + if self._log_access: + bad_names.update(models.LOG_ACCESS_COLUMNS) + unknown_names = set() + data_list = [] + inversed_fields = set() + for vals in vals_list: + # add missing defaults + vals = self._add_missing_default_values(vals) + # distribute fields into sets for various purposes + data = {} + data["stored"] = stored = {} + data["inversed"] = inversed = {} + data["inherited"] = inherited = defaultdict(dict) + data["protected"] = protected = set() + for key, val in vals.items(): + if key in bad_names: + continue + field = self._fields.get(key) + if not field: + unknown_names.add(key) + continue + if field.store: + stored[key] = val + if field.inherited: + inherited[field.related_field.model_name][key] = val + elif field.inverse: + inversed[key] = val + inversed_fields.add(field) + protected.update(self._field_computed.get(field, [field])) + data_list.append(data) + + # From v12 _create method + # Create records from the stored field values in ``data_list``. + assert data_list + cr = self.env.cr + quote = '"{}"'.format + + # set boolean fields to False by default (avoid NULL in database) + for name, field in self._fields.items(): + if field.type == "boolean" and field.store: + for data in data_list: + data["stored"].setdefault(name, False) + + # insert rows + ids = [] # ids of created records + other_fields = set() # non-column fields + translated_fields = set() # translated fields + + # column names, formats and values (for common fields) + columns0 = [("id", "nextval(%s)", self._sequence)] + if self._log_access: + columns0.append(("create_uid", "%s", self._uid)) + columns0.append(("create_date", "%s", AsIs("(now() at time zone 'UTC')"))) + columns0.append(("write_uid", "%s", self._uid)) + columns0.append(("write_date", "%s", AsIs("(now() at time zone 'UTC')"))) + + for data in data_list: + # determine column values + columns = list(columns0) + for name, val in sorted(data["stored"].items()): + field = self._fields[name] + assert field.store + + if field.column_type: + col_val = field.convert_to_column(val, self, data["stored"]) + columns.append((name, field.column_format, col_val)) + if field.translate is True: + translated_fields.add(field) + else: + other_fields.add(field) + + # insert a row with the given columns + # pylint: disable=E8103 + query = "INSERT INTO {} ({}) VALUES ({}) RETURNING id".format( + quote(self._table), + ", ".join(quote(name) for name, fmt, val in columns), + ", ".join(fmt for name, fmt, val in columns), + ) + params = [val for name, fmt, val in columns] + cr.execute(query, params) + ids.append(cr.fetchone()[0]) + + # Compatibility v11 and v12 + if other_fields: + record = self.browse(ids[-1]) + protected_fields = protected + upd_todo = [field.name for field in other_fields] + + # From v11 _create method + with self.env.protecting(protected_fields, record): + # defaults in context must be removed when call a + # one2many or many2many + rel_context = { + key: val + for key, val in self._context.items() + if not key.startswith("default_") + } + # call the 'write' method of fields which are not columns + for name in sorted( + upd_todo, key=lambda name: self._fields[name]._sequence + ): + field_value = data["stored"].get(name, False) + if not field_value: + continue + field = self._fields[name] + field.write(record.with_context(**rel_context), field_value) + return self.browse(ids) diff --git a/base_external_dbsource_importer/models/mapped_mixin.py b/base_external_dbsource_importer/models/mapped_mixin.py new file mode 100644 index 000000000..51202b57b --- /dev/null +++ b/base_external_dbsource_importer/models/mapped_mixin.py @@ -0,0 +1,92 @@ +from odoo import api, fields, models +from odoo.exceptions import ValidationError + + +class DbsourceExternalMixin(models.AbstractModel): + """Provides utilities for mapping multiple external records to a single one""" + + _name = "dbsource.mapped.mixin" + _description = "Mixin for mapping models" + + _rec_names_search = ["external_name", "external_key"] + _check_company_auto = True + + external_name = fields.Char() + external_key = fields.Char(index="btree_not_null", required=True) + external_table = fields.Char(index=True) + mapped_key = fields.Char() + company_id = fields.Many2one( + comodel_name="res.company", + ondelete="cascade", + default=False, + index=True, + ) + record_model = fields.Char("Resource Model") + record_id = fields.Many2oneReference( + "Record ID", + model_field="record_model", + compute="_compute_record_id", + store=True, + ) + sql_constraints = [ + ( + "external_unique", + "UNIQUE(external_key, company_id)", + "External keys must be unique per company!", + ), + ] + + @api.constrains("external_key", "mapped_key", "record_model") + def _check_mapped_key_not_in_external_keys(self): + for record in self: + if self.search_count( + [ + ("external_key", "=", record.mapped_key), + ("id", "!=", record.id), + ("record_model", "=", record.record_model), + ] + ): + raise ValidationError( + self.env._( + "Target key %(key)s is the source of another mapping", + key=record.mapped_key, + ) + ) + + if self.search_count( + [ + ("mapped_key", "=", record.external_key), + ("id", "!=", record.id), + ("record_model", "=", record.record_model), + ] + ): + raise ValidationError( + self.env._( + "Source key %(key)s is the target of another mapping", + key=record.external_key, + ) + ) + + @api.depends("mapped_key", "record_model") + def _compute_record_id(self): + for record in self: + target = ( + self.env[record.record_model] + .with_context(active_test=False) + .search([("openbravo_key", "=", record.mapped_key)]) + ) + if len(target) != 1: + record.record_id = record.record_id + continue + record.record_id = target.id + + def ensure_mapping(self, vals): + existing = self.search( + [ + ("external_key", "=", vals["external_key"]), + ("record_model", "=", vals["record_model"]), + ] + ) + if existing: + return existing + return self.create(vals) diff --git a/base_external_dbsource_importer/pyproject.toml b/base_external_dbsource_importer/pyproject.toml new file mode 100644 index 000000000..4231d0ccc --- /dev/null +++ b/base_external_dbsource_importer/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/base_external_dbsource_importer/readme/CONFIGURE.md b/base_external_dbsource_importer/readme/CONFIGURE.md new file mode 100644 index 000000000..9a93e6daf --- /dev/null +++ b/base_external_dbsource_importer/readme/CONFIGURE.md @@ -0,0 +1,4 @@ +To configure this module, you need to: + +1. Database sources can be configured in Settings \> Technical \> + Database Structure \> Database sources. diff --git a/base_external_dbsource_importer/readme/CONTRIBUTORS.md b/base_external_dbsource_importer/readme/CONTRIBUTORS.md new file mode 100644 index 000000000..b0126250e --- /dev/null +++ b/base_external_dbsource_importer/readme/CONTRIBUTORS.md @@ -0,0 +1,4 @@ +- [Tecnativa](https://www.tecnativa.com): + - Sergio Teruel + - Carlos Dauden + - David Bañón Gil diff --git a/base_external_dbsource_importer/readme/DESCRIPTION.md b/base_external_dbsource_importer/readme/DESCRIPTION.md new file mode 100644 index 000000000..e2fa68387 --- /dev/null +++ b/base_external_dbsource_importer/readme/DESCRIPTION.md @@ -0,0 +1 @@ +This module extends the functionality of base_external_dbsource diff --git a/base_external_dbsource_importer/security/ir.model.access.csv b/base_external_dbsource_importer/security/ir.model.access.csv new file mode 100644 index 000000000..8f250349a --- /dev/null +++ b/base_external_dbsource_importer/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_base_external_dbsource_fields_update_group_system,bae_external_dbsource_fields_group_system,model_base_external_dbsource_fields_update,base.group_system,1,1,1,1 diff --git a/base_external_dbsource_importer/static/description/icon.png b/base_external_dbsource_importer/static/description/icon.png new file mode 100644 index 000000000..3a0328b51 Binary files /dev/null and b/base_external_dbsource_importer/static/description/icon.png differ diff --git a/base_external_dbsource_importer/static/description/index.html b/base_external_dbsource_importer/static/description/index.html new file mode 100644 index 000000000..e85eb2f3d --- /dev/null +++ b/base_external_dbsource_importer/static/description/index.html @@ -0,0 +1,443 @@ + + + + + +Base External Dbsource Importer + + + +
+

Base External Dbsource Importer

+ + +

Alpha License: AGPL-3 OCA/server-backend Translate me on Weblate Try me on Runboat

+

This module extends the functionality of base_external_dbsource

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Configuration

+

To configure this module, you need to:

+
    +
  1. Database sources can be configured in Settings > Technical > Database +Structure > Database sources.
    2. +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Tecnativa
    • +
+
+
+

Contributors

+
    +
  • Tecnativa:
      +
    • Sergio Teruel
      • +
    • Carlos Dauden
      • +
    • David Bañón Gil
      • +
    +
    • +
+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/server-backend project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/base_external_dbsource_importer/tests/__init__.py b/base_external_dbsource_importer/tests/__init__.py new file mode 100644 index 000000000..afd735628 --- /dev/null +++ b/base_external_dbsource_importer/tests/__init__.py @@ -0,0 +1 @@ +from . import test_external_dbsource_importer diff --git a/base_external_dbsource_importer/tests/models.py b/base_external_dbsource_importer/tests/models.py new file mode 100644 index 000000000..e1c014b42 --- /dev/null +++ b/base_external_dbsource_importer/tests/models.py @@ -0,0 +1,49 @@ +# Copyright 2025 Tecnativa - David Bañón Gil +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +from odoo import fields, models + +from odoo.addons.base_external_dbsource_importer.models.base_external_dbsource import ( + BaseExternalModelImporter, +) + + +class BaseExternalModelImporterTest(BaseExternalModelImporter): + _external_key = "test_ext_key" + data = [ + { + "id": 876, + "name": "John Test", + "email": "john@example.com", + "is_company": False, + }, + { + "id": 877, + "name": "Test INC.", + "email": "test@example.com", + "is_company": True, + }, + ] + + def _get_external_records(self, table_name, fields="*", where=""): + return self.data if table_name == "test_table" else [] + + +# pylint: disable=consider-merging-classes-inherited +class BaseExternalDbsourceTest(models.Model): + _inherit = "base.external.dbsource" + + connector = fields.Selection( + selection_add=[("test", "Test")], ondelete={"test": "cascade"} + ) + + @property + def test_importer(self): + return BaseExternalModelImporterTest(dbsource=self) + + +# pylint: enable=consider-merging-classes-inherited +class ResPartnerWithMixin(models.Model): + _inherit = ["res.partner", "dbsource.external.mixin"] + _name = "res.partner" + + test_ext_key = fields.Char(copy=False) diff --git a/base_external_dbsource_importer/tests/test_external_dbsource_importer.py b/base_external_dbsource_importer/tests/test_external_dbsource_importer.py new file mode 100644 index 000000000..85f8533aa --- /dev/null +++ b/base_external_dbsource_importer/tests/test_external_dbsource_importer.py @@ -0,0 +1,96 @@ +from odoo_test_helper import FakeModelLoader + +from odoo import Command +from odoo.tests import tagged + +from odoo.addons.base.tests.common import BaseCommon + + +@tagged("post_install", "-at_install") +class TestExternalDBSource(BaseCommon): + def setUp(self): + super().setUp() + # Load a test model using odoo_test_helper + self.loader = FakeModelLoader(self.env, self.__module__) + self.loader.backup_registry() + from .models import BaseExternalDbsourceTest, ResPartnerWithMixin + + self.loader.update_registry((BaseExternalDbsourceTest,)) + self.loader.update_registry((ResPartnerWithMixin,)) + self.dbsource = self.dbsource = self.env["base.external.dbsource"].create( + { + "connector": "test", + "name": "Test data loader", + } + ) + + def tearDown(self): + self.loader.restore_registry() + return super().tearDown() + + def _import_data(self): + ext_records, records, records_dic = self.dbsource.test_importer.load_data( + "res.partner", "test_table" + ) + for record in ext_records: + self.dbsource.test_importer.upsert( + str(record["id"]), + records, + records_dic, + { + "name": record["name"], + "email": record["email"], + "is_company": record["is_company"], + "test_ext_key": record["id"], + }, + ) + + def test_import(self): + self._import_data() + partner = self.env["res.partner"].search([("name", "=", "John Test")], limit=1) + self.assertTrue(partner, "Contact 'John Test' was not created in Odoo") + self.assertEqual(partner.email, self.dbsource.test_importer.data[0]["email"]) + # We now change source data + imported_partner_email = partner.email + imported_partner_id = partner.id + new_email = "new_email@example.com" + new_name = "John Doe Test" + self.dbsource.test_importer.data[0]["email"] = new_email + self.dbsource.test_importer.data[0]["name"] = new_name + # But only want want to update name, not email + self.dbsource.write( + { + "fields_to_update_ids": [ + Command.create( + { + "model_id": self.env.ref("base.model_res_partner").id, + "field_ids": [ + Command.set( + [ + self.env.ref("base.field_res_partner__name").id, + ], + ) + ], + }, + ), + ] + } + ) + self._import_data() + partner = self.env["res.partner"].browse(imported_partner_id) + partner2 = self.env["res.partner"].search([("name", "=", "Test INC.")]) + self.assertEqual( + len(partner2), 1, "Records should not be duplicated when importing again" + ) + + self.assertTrue(partner, "Partner should still exist") + self.assertEqual( + partner.email, + imported_partner_email, + "Email got updated when it wasn't in 'fields_to_update_ids'", + ) + self.assertEqual( + partner.name, + new_name, + "Name didn't get updated when it was in 'fields_to_update_ids'", + ) diff --git a/base_external_dbsource_importer/views/base_external_dbsource_view.xml b/base_external_dbsource_importer/views/base_external_dbsource_view.xml new file mode 100644 index 000000000..b4c3fa2c3 --- /dev/null +++ b/base_external_dbsource_importer/views/base_external_dbsource_view.xml @@ -0,0 +1,44 @@ + + + + + External Dbsource Importer View + base.external.dbsource + + + + + + + + + + + + + + +
Clear ormcache
+