diff --git a/connector_typesense/README.rst b/connector_typesense/README.rst new file mode 100644 index 00000000..244ee26f --- /dev/null +++ b/connector_typesense/README.rst @@ -0,0 +1,108 @@ +=================== +connector_typesense +=================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:f09633c3af59b153f0eba3f876a9835676c430a6f39dd0dfa3545a924c80bc11 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fsearch--engine-lightgray.png?logo=github + :target: https://github.com/OCA/search-engine/tree/16.0/connector_typesense + :alt: OCA/search-engine +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/search-engine-16-0/search-engine-16-0-connector_typesense + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/search-engine&target_branch=16.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This addon provides the bases to implement addons to export information to +Typesense_ indexes. + +.. _Typesense: https://typesense.org + +**Table of contents** + +.. contents:: + :local: + +Installation +============ + +This package requires a typesense search engine running. +Please read this for a [quick docker based setup](https://typesense.org/docs/guide/install-typesense.html#option-2-local-machine-self-hosting). + +Configuration +============= + +You have to configure (Host, Port, Protocol, Typesense API Key) in a new backend form view: + +Search Engine > Configuration > Backends + +Usage +===== + +a nice UI is also available here: https://github.com/bfritscher/typesense-dashboard/releases + + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* Derico +* Kencove + +Contributors +~~~~~~~~~~~~ + +* Maik Derstappen +* Mohamed Alkobrosli + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-Kencove| image:: https://github.com/Kencove.png?size=40px + :target: https://github.com/Kencove + :alt: Kencove + +Current `maintainer `__: + +|maintainer-Kencove| + +This module is part of the `OCA/search-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/connector_typesense/__init__.py b/connector_typesense/__init__.py new file mode 100644 index 00000000..738a2eec --- /dev/null +++ b/connector_typesense/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import tools diff --git a/connector_typesense/__manifest__.py b/connector_typesense/__manifest__.py new file mode 100644 index 00000000..77c785ca --- /dev/null +++ b/connector_typesense/__manifest__.py @@ -0,0 +1,20 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "connector_typesense", + "category": "Connector", + "summary": "Connector For Typesense Search Engine", + "version": "16.0.1.0.0", + "license": "AGPL-3", + "author": "Derico, Kencove, Odoo Community Association (OCA)", + "website": "https://github.com/OCA/search-engine", + "maintainers": ["Kencove"], + "depends": ["connector_search_engine"], + "data": [ + "views/ts_backend.xml", + ], + "demo": ["demo/se_index_config_demo.xml", "demo/backend_demo.xml"], + "external_dependencies": {"python": ["typesense==1.0.3", "requests"]}, + "installable": True, +} diff --git a/connector_typesense/demo/backend_demo.xml b/connector_typesense/demo/backend_demo.xml new file mode 100644 index 00000000..1cd56f01 --- /dev/null +++ b/connector_typesense/demo/backend_demo.xml @@ -0,0 +1,14 @@ + + + + + Demo Sale Channel Typesense + demo_sale_channel_typesense + typesense + localhost + 8108 + http + xyz + + diff --git a/connector_typesense/demo/se_index_config_demo.xml b/connector_typesense/demo/se_index_config_demo.xml new file mode 100644 index 00000000..fd714965 --- /dev/null +++ b/connector_typesense/demo/se_index_config_demo.xml @@ -0,0 +1,22 @@ + + + + TS Product Config + {} + +{ + "name": "ts_products_collection", + "fields": [ + { + "name": "id", + "type": "string" + }, + { + "name": "name", + "type": "string" + } + ] +} + + + diff --git a/connector_typesense/models/__init__.py b/connector_typesense/models/__init__.py new file mode 100644 index 00000000..b4cddc58 --- /dev/null +++ b/connector_typesense/models/__init__.py @@ -0,0 +1 @@ +from . import se_backend, se_index diff --git a/connector_typesense/models/se_backend.py b/connector_typesense/models/se_backend.py new file mode 100644 index 00000000..6fc263ae --- /dev/null +++ b/connector_typesense/models/se_backend.py @@ -0,0 +1,43 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + +from ..tools.adapter import TypesenseAdapter + + +class SeBackend(models.Model): + _inherit = "se.backend" + + backend_type = fields.Selection( + selection_add=[("typesense", "Typesense")], + ondelete={"typesense": "cascade"}, + string="Type", + required=True, + ) + ts_server_host = fields.Char( + string="Typesense host", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_server_port = fields.Char( + string="Typesense port", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_server_protocol = fields.Char( + string="Typesense protocol", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_server_timeout = fields.Integer( + string="Typesense server timeout", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_api_key = fields.Char( + help="Typesense Api Key", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + + def _get_adapter_class(self): + if self.backend_type == "typesense": + return TypesenseAdapter + else: + return super()._get_adapter_class() diff --git a/connector_typesense/models/se_index.py b/connector_typesense/models/se_index.py new file mode 100644 index 00000000..43d0c0dc --- /dev/null +++ b/connector_typesense/models/se_index.py @@ -0,0 +1,19 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + +from ..tools.serializer import TSJsonifySerializer + + +class SeIndex(models.Model): + + _inherit = "se.index" + + serializer_type = fields.Selection(selection_add=[("typesense", "Typesense")]) + + def _get_serializer(self): + if self.serializer_type == "typesense": + return TSJsonifySerializer() + else: + return super()._get_serializer() diff --git a/connector_typesense/readme/CONFIGURE.rst b/connector_typesense/readme/CONFIGURE.rst new file mode 100644 index 00000000..c1e532be --- /dev/null +++ b/connector_typesense/readme/CONFIGURE.rst @@ -0,0 +1,3 @@ +You have to configure (Host, Port, Protocol, Typesense API Key) in a new backend form view: + +Search Engine > Configuration > Backends diff --git a/connector_typesense/readme/CONTRIBUTORS.rst b/connector_typesense/readme/CONTRIBUTORS.rst new file mode 100644 index 00000000..38bcf55b --- /dev/null +++ b/connector_typesense/readme/CONTRIBUTORS.rst @@ -0,0 +1,2 @@ +* Maik Derstappen +* Mohamed Alkobrosli diff --git a/connector_typesense/readme/DESCRIPTION.rst b/connector_typesense/readme/DESCRIPTION.rst new file mode 100644 index 00000000..985e0128 --- /dev/null +++ b/connector_typesense/readme/DESCRIPTION.rst @@ -0,0 +1,4 @@ +This addon provides the bases to implement addons to export information to +Typesense_ indexes. + +.. _Typesense: https://typesense.org diff --git a/connector_typesense/readme/INSTALL.rst b/connector_typesense/readme/INSTALL.rst new file mode 100644 index 00000000..451f51b3 --- /dev/null +++ b/connector_typesense/readme/INSTALL.rst @@ -0,0 +1,2 @@ +This package requires a typesense search engine running. +Please read this for a [quick docker based setup](https://typesense.org/docs/guide/install-typesense.html#option-2-local-machine-self-hosting). diff --git a/connector_typesense/readme/USAGE.rst b/connector_typesense/readme/USAGE.rst new file mode 100644 index 00000000..60f089bd --- /dev/null +++ b/connector_typesense/readme/USAGE.rst @@ -0,0 +1,2 @@ +a nice UI is also available here: https://github.com/bfritscher/typesense-dashboard/releases + diff --git a/connector_typesense/static/description/icon.png b/connector_typesense/static/description/icon.png new file mode 100644 index 00000000..64e41565 Binary files /dev/null and b/connector_typesense/static/description/icon.png differ diff --git a/connector_typesense/static/description/index.html b/connector_typesense/static/description/index.html new file mode 100644 index 00000000..79fdbef6 --- /dev/null +++ b/connector_typesense/static/description/index.html @@ -0,0 +1,445 @@ + + + + + +connector_typesense + + + +
+

connector_typesense

+ + +

Beta License: AGPL-3 OCA/search-engine Translate me on Weblate Try me on Runboat

+

This addon provides the bases to implement addons to export information to +Typesense indexes.

+

Table of contents

+ +
+

Installation

+

This package requires a typesense search engine running. +Please read this for a [quick docker based setup](https://typesense.org/docs/guide/install-typesense.html#option-2-local-machine-self-hosting).

+
+
+

Configuration

+

You have to configure (Host, Port, Protocol, Typesense API Key) in a new backend form view:

+

Search Engine > Configuration > Backends

+
+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Derico
    • +
  • Kencove
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

Kencove

+

This module is part of the OCA/search-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/connector_typesense/tests/__init__.py b/connector_typesense/tests/__init__.py new file mode 100644 index 00000000..fb2e1046 --- /dev/null +++ b/connector_typesense/tests/__init__.py @@ -0,0 +1 @@ +from . import test_connector_typesense diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter.yaml new file mode 100644 index 00000000..7cbd0917 --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter.yaml @@ -0,0 +1,76 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"created_at":1747351443,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "41", "name": "Marty McFly"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '35' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '16' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete.yaml new file mode 100644 index 00000000..7b66ca6a --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete.yaml @@ -0,0 +1,297 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2 + response: + body: + string: '{"created_at":1747351443,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":4,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-1", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351545,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-1"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "1", "name": "Foo"} + + {"id": "2", "name": "Bar"} + + {"id": "3", "name": "Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/?filter_by=id%3A%5B1%2C+3%5D + response: + body: + string: '{"num_deleted":2}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '17' + content-type: + - application/json + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/search?q=%2A + response: + body: + string: '{"facet_counts":[],"found":1,"hits":[{"document":{"id":"2","name":"Bar"},"highlight":{},"highlights":[]}],"out_of":1,"page":1,"request_params":{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","per_page":10,"q":"*"},"search_cutoff":false,"search_time_ms":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '272' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete_nonexisting_documents.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete_nonexisting_documents.yaml new file mode 100644 index 00000000..bb4db01d --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete_nonexisting_documents.yaml @@ -0,0 +1,40 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/?filter_by=id%3A%5B%27donotexist%27%2C+%27donotexisteither%27%5D + response: + body: + string: '{"num_deleted":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '17' + content-type: + - application/json + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_iter.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_iter.yaml new file mode 100644 index 00000000..bf49437a --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_iter.yaml @@ -0,0 +1,295 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1 + response: + body: + string: '{"created_at":1747351545,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":1,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-1", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351549,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-1"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"created_at":1747351549,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "1", "name": "Foo"} + + {"id": "2", "name": "Bar"} + + {"id": "3", "name": "Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/search?q=%2A + response: + body: + string: '{"facet_counts":[],"found":3,"hits":[{"document":{"id":"3","name":"Baz"},"highlight":{},"highlights":[]},{"document":{"id":"2","name":"Bar"},"highlight":{},"highlights":[]},{"document":{"id":"1","name":"Foo"},"highlight":{},"highlights":[]}],"out_of":3,"page":1,"request_params":{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","per_page":10,"q":"*"},"search_cutoff":false,"search_time_ms":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '408' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_reindex.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_reindex.yaml new file mode 100644 index 00000000..296601a9 --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_reindex.yaml @@ -0,0 +1,628 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1 + response: + body: + string: '{"created_at":1747351549,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-1", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351551,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-1"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "1", "name": "Foo"} + + {"id": "2", "name": "Bar"} + + {"id": "3", "name": "Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1/documents/export + response: + body: + string: '{"id":"1","name":"Foo"} + + {"id":"2","name":"Bar"} + + {"id":"3","name":"Baz"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '71' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2 + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-2", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351553,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"id":"1","name":"Foo"} + + {"id":"2","name":"Bar"} + + {"id":"3","name":"Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '71' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2/documents/import?action=create + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2 + response: + body: + string: '{"created_at":1747351553,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-2"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1 + response: + body: + string: '{"created_at":1747351551,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/search?q=%2A + response: + body: + string: '{"facet_counts":[],"found":3,"hits":[{"document":{"id":"3","name":"Baz"},"highlight":{},"highlights":[]},{"document":{"id":"2","name":"Bar"},"highlight":{},"highlights":[]},{"document":{"id":"1","name":"Foo"},"highlight":{},"highlights":[]}],"out_of":3,"page":1,"request_params":{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","per_page":10,"q":"*"},"search_cutoff":false,"search_time_ms":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '408' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/docker-compose.typesense.example.yml b/connector_typesense/tests/docker-compose.typesense.example.yml new file mode 100644 index 00000000..3c5ffb02 --- /dev/null +++ b/connector_typesense/tests/docker-compose.typesense.example.yml @@ -0,0 +1,13 @@ +version: "3.8" + +services: + typesense-server: + image: typesense/typesense:0.25.2 + container_name: typesense-server + ports: + - "8108:8108" + volumes: + - /tmp/typesense-data:/data + command: > + --data-dir /data --api-key=xyz --listen-port=8108 --enable-cors + restart: unless-stopped diff --git a/connector_typesense/tests/test_connector_typesense.py b/connector_typesense/tests/test_connector_typesense.py new file mode 100644 index 00000000..299f9fa7 --- /dev/null +++ b/connector_typesense/tests/test_connector_typesense.py @@ -0,0 +1,141 @@ +# Copyright 2019 Kencove +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import json +from time import sleep + +from vcr_unittest import VCRMixin + +from odoo.addons.connector_search_engine.tests.test_all import TestBindingIndexBase + +from ..tools.adapter import TypesenseAdapter + + +class TestConnectorTypeSense(VCRMixin, TestBindingIndexBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend = cls.env.ref("connector_typesense.backend_1") + cls.se_config = cls.env.ref("connector_typesense.se_index_config_product") + cls.setup_records() + cls.adapter: TypesenseAdapter = cls.se_index.se_adapter + cls.demo_data = [ + {"id": "1", "name": "Foo"}, + {"id": "2", "name": "Bar"}, + {"id": "3", "name": "Baz"}, + ] + + def _get_vcr_kwargs(self, **kwargs): + return { + "record_mode": "one", + "match_on": ["method", "path", "query"], + "filter_headers": ["Authorization"], + "decode_compressed_response": True, + } + + @classmethod + def _prepare_index_values(cls, backend): + values = super()._prepare_index_values(backend) + values.update({"config_id": cls.se_config.id}) + return values + + def test_index_adapter(self): + # ts serialize type converts id type from integer into string + self.se_index.write({"serializer_type": "typesense"}) + # Export collection settings to cassette (typesense) + self.se_index.export_settings() + + data = self.partner_binding.data + self.assertFalse(data) + + # Recompute data to be sent + self.se_index.batch_recompute(force_export=True) + data = self.partner_binding.data + self.assertEqual(data["name"], "Marty McFly") + self.assertIsInstance(data["id"], str) + + # Set partner to be updated and patch data to cassette (typesense) + self.partner_binding.write({"state": "to_export"}) + self.se_index.batch_sync(force_export=True) + + # Ensure that calls have been recorded in cassette + self.assertTrue(self.cassette.all_played) + self.assertGreaterEqual(len(self.cassette.requests), 2) + + request_1 = self.cassette.requests[0] + self.assertEqual(request_1.method, "GET") + expected_path = f"/collections/{self.se_index.name}".lower() + self.assertEqual(self.parse_path(request_1.uri), expected_path) + response_1 = self.cassette.responses[0] + body = response_1.get("body").get("string").decode("utf-8") + lines = [line for line in filter(lambda line: line, body.split("\n"))] + # we must have 1 line for the index op + self.assertEqual(len(lines), 1) + index_action = json.loads(lines[0]) + self.assertEqual(index_action.get("name"), f"{self.se_index.name}-2".lower()) + + request_2 = self.cassette.requests[-1] + self.assertEqual(request_2.method, "POST") + body = request_2.body.decode("utf-8") + data = json.loads(body) + # Mock the preserved id in the cassette: 107 to new record id + data["id"] = self.partner_binding.data.get("id") + self.assertDictEqual(data, self.partner_binding.data) + + def test_index_config_as_str(self): + self.se_config.write({"body_str": '{"mappings": {"1":1}}'}) + self.assertDictEqual(self.se_config.body, {"mappings": {"1": 1}}) + self.assertEqual(self.se_config.body_str, '{"mappings": {"1":1}}') + + def test_index_adapter_iter(self): + data = self.demo_data + self.adapter.clear() + self.adapter.settings() + self.adapter.index(data) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + res = [x["document"] for x in self.adapter.each()] + res.sort(key=lambda d: d["id"]) + self.assertListEqual(res, data) + + def test_index_adapter_delete(self): + data = self.demo_data + self.adapter.clear() + self.adapter.index(data) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + self.adapter.delete([1, 3]) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + res = [x["document"] for x in self.adapter.each()] + res.sort(key=lambda d: d["id"]) + self.assertListEqual(res, [{"id": "2", "name": "Bar"}]) + + def test_index_adapter_delete_nonexisting_documents(self): + """We try to delete records that do not exist. + Because it does not matter, it is just ignored. No exception. + """ + self.adapter.delete(["donotexist", "donotexisteither"]) + + def test_index_adapter_reindex(self): + data = self.demo_data + self.adapter.clear() + self.adapter.index(data) + index_name = self.adapter._get_current_aliased_index_name() + next_index_name = self.adapter._get_next_aliased_index_name(index_name) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + self.adapter.reindex() + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + res = [x["document"] for x in self.adapter.each()] + res.sort(key=lambda d: d["id"]) + self.assertListEqual(res, data) + self.assertEqual( + self.adapter._get_current_aliased_index_name(), next_index_name + ) diff --git a/connector_typesense/tools/__init__.py b/connector_typesense/tools/__init__.py new file mode 100644 index 00000000..307aa310 --- /dev/null +++ b/connector_typesense/tools/__init__.py @@ -0,0 +1,2 @@ +from . import adapter +from . import serializer diff --git a/connector_typesense/tools/adapter.py b/connector_typesense/tools/adapter.py new file mode 100644 index 00000000..bb5ffcdf --- /dev/null +++ b/connector_typesense/tools/adapter.py @@ -0,0 +1,260 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import json +import logging +from typing import Any, Iterator + +import requests + +from odoo import _ +from odoo.exceptions import UserError + +from odoo.addons.connector_search_engine.tools.adapter import SearchEngineAdapter + +_logger = logging.getLogger(__name__) + + +try: + import typesense +except ImportError: + _logger.debug("Can not import typesense") + + +# def _is_delete_nonexistent_documents(elastic_exception): +# """True iff all errors in this exception are deleting a nonexisting document.""" +# b = lambda d: "delete" in d and d["delete"]["status"] == 404 # noqa +# return all(b(error) for error in elastic_exception.errors) + + +class TypesenseAdapter(SearchEngineAdapter): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__ts_client = None + + @property + def _index_name(self): + return self.index_record.name.lower() + + # @property + # def _ts_connection_class(self): + # return typesense.RequestsHttpConnection + + @property + def _ts_client(self): + if not self.__ts_client: + self.__ts_client = self._get_ts_client() + return self.__ts_client + + @property + def _index_config(self): + return self.index_record.config_id.body + + def _get_ts_client(self): + backend = self.backend_record + return typesense.Client( + { + "nodes": [ + { + "host": backend.ts_server_host, + "port": backend.ts_server_port, + "protocol": backend.ts_server_protocol, + } + ], + "api_key": backend.ts_api_key, + "connection_timeout_seconds": int(backend.ts_server_timeout) or 300, + } + ) + + def test_connection(self): + ts = self._ts_client + try: + ts.collections.retrieve() + except typesense.exceptions.ObjectNotFound as exc: + raise UserError( + _("Not Found - The requested resource is not found.") + ) from exc + except typesense.exceptions.RequestUnauthorized as exc: + raise UserError(_("Unauthorized - Your API key is wrong.")) from exc + except requests.exceptions.ConnectionError as exc: + raise UserError(_("Unable to connect :") + "\n\n" + repr(exc)) from exc + except requests.exceptions.InvalidURL as exc: + raise UserError( + _("Invalid URL - No host supplied") + "\n\n" + repr(exc) + ) from exc + + def index(self, records) -> None: + ts = self._ts_client + # Convert records to JSONL string (one JSON object per line) + jsonl_data = "\n".join([json.dumps(record) for record in records]) + try: + res = ts.collections[self._index_name].documents.import_( + jsonl_data, {"action": "upsert"} + ) + except typesense.exceptions.ObjectNotFound as e: + _logger.warning( + f"{self._index_name} not found, skip updating alias and " + f"creating a new index (collection)!\n\n{e}" + ) + self.reindex() + self.index(records) + # Validate number of successfully indexed documents + results = [json.loads(line) for line in res.splitlines()] + success_count = sum(1 for item in results if item["success"]) + if success_count != len(records): + self.clear() + self.index(records) + + def delete(self, binding_ids) -> None: + ts = self._ts_client + ts.collections[self._index_name].documents.delete( + {"filter_by": f"id:{binding_ids}"} + ) + + def clear(self) -> None: + ts = self._ts_client + index_name = self._get_current_aliased_index_name() or self._index_name + ts.collections[index_name].delete() + self.settings() + + def each(self) -> Iterator[dict[str, Any]]: + ts = self._ts_client + res = ts.collections[self._index_name].documents.search( + { + "q": "*", + } + ) + if not res: + # eg: empty index + return + hits = res["hits"] + for hit in hits: + yield hit + + def settings(self) -> None: + ts = self._ts_client + try: + ts.collections[self._index_name].retrieve() + except typesense.exceptions.ObjectNotFound: + client = self._ts_client + # To allow rolling updates, we work with index aliases + aliased_index_name = self._get_next_aliased_index_name() + # index_name / collection_name is part of the schema defined in + # self._index_config + index_config = self._index_config + index_config.update( + { + "name": aliased_index_name, + } + ) + _logger.info(f"Create aliased_index_name '{aliased_index_name}'...") + client.collections.create(index_config) + _logger.info( + f"Set collection alias '{self._index_name}' >> aliased_index_name " + f"'{aliased_index_name}'." + ) + client.aliases.upsert( + self._index_name, {"collection_name": aliased_index_name} + ) + + def _get_current_aliased_index_name(self) -> str: + """Get the current aliased index name if any""" + current_aliased_index_name = None + try: + alias = self._ts_client.aliases[self._index_name].retrieve() + if "collection_name" in alias: + current_aliased_index_name = alias["collection_name"] + except typesense.exceptions.ObjectNotFound as e: + _logger.warning( + f"current_aliased_index_name not found, skip updating alias and " + f"creating a new index (collection)!\n\n{e}" + ) + return current_aliased_index_name + + def _get_next_aliased_index_name( + self, aliased_index_name: str | None = None + ) -> str: + """Get the next aliased index name + + The next aliased index name is based on the current aliased index name. + It's the current aliased index name incremented by 1. + + :param aliased_index_name: the current aliased index name + :return: the next aliased index name + """ + next_version = 1 + if aliased_index_name: + next_version = int(aliased_index_name.split("-")[-1]) + 1 + return f"{self._index_name}-{next_version}" + + def reindex(self) -> None: + """Reindex records according to the current config + This method is useful to allows a rolling update of index + configuration. + This process is based on the following steps: + 1. export data from current aliased index + 2. create a new index (collection) with the current config + 3. import data into new aliased index (collection) + 4. Update the index alias to point to the new aliased index (collection) + 5. Drop the old index. + """ + client = self._ts_client + current_aliased_index_name = self._get_current_aliased_index_name() + if not current_aliased_index_name: + self.settings() + current_aliased_index_name = self._get_current_aliased_index_name() + data = client.collections[current_aliased_index_name].documents.export() + next_aliased_index_name = self._get_next_aliased_index_name( + current_aliased_index_name + ) + try: + client.collections[next_aliased_index_name].retrieve() + except typesense.exceptions.ObjectNotFound: + # To allow rolling updates, we work with index aliases + # index_name / collection_name is part of the schema defined + # in self._index_config + _logger.info( + f"Create new_aliased_index_name '{next_aliased_index_name}'..." + ) + index_config = self._index_config + index_config.update( + { + "name": next_aliased_index_name, + } + ) + client.collections.create(index_config) + _logger.info( + f"Import existing data into new_aliased_index_name " + f"'{next_aliased_index_name}'..." + ) + client.collections[next_aliased_index_name].documents.import_( + data.encode("utf-8"), {"action": "create"} + ) + + try: + client.collections[next_aliased_index_name].retrieve() + except typesense.exceptions.ObjectNotFound as e: + _logger.warning( + f"New aliased_index_name not found, skip updating alias and " + f"not removing old index (collection)!\n\n{e}" + ) + else: + _logger.info( + f"Set collection alias '{self._index_name}' >> " + f"new_aliased_index_name '{next_aliased_index_name}'." + ) + client.aliases.upsert( + self._index_name, {"collection_name": next_aliased_index_name} + ) + _logger.info( + f"Remove old aliased index (collection) " + f"'{current_aliased_index_name}'." + ) + client.collections[current_aliased_index_name].delete() + + else: + _logger.warning( + f"next_aliased_index_name '{next_aliased_index_name}' " + f"already exists, skip!", + self._index_name, + ) diff --git a/connector_typesense/tools/serializer.py b/connector_typesense/tools/serializer.py new file mode 100644 index 00000000..d305af89 --- /dev/null +++ b/connector_typesense/tools/serializer.py @@ -0,0 +1,17 @@ +# Copyright 2023 Kencove (https://kencove.com). +# @author Mohamed Alkobrosli +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo.addons.connector_search_engine.tools.serializer import ModelSerializer + + +class TSJsonifySerializer(ModelSerializer): + def serialize(self, record): + # Typesense SE requires id field to be string + # It also requires other intger field types to be searchable + data = {} + if record.id: + data["id"] = str(record.id) + if record.name: + data["name"] = record.name + return data diff --git a/connector_typesense/views/ts_backend.xml b/connector_typesense/views/ts_backend.xml new file mode 100644 index 00000000..9358d8b4 --- /dev/null +++ b/connector_typesense/views/ts_backend.xml @@ -0,0 +1,40 @@ + + + + se.backend + + + + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index 4ac43d30..b0a37f9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ elasticsearch>=7.0.0,<=7.13.4 pydantic requests +typesense==1.0.3 typing-extensions unidecode diff --git a/setup/connector_typesense/odoo/addons/connector_typesense b/setup/connector_typesense/odoo/addons/connector_typesense new file mode 120000 index 00000000..1ffbdd91 --- /dev/null +++ b/setup/connector_typesense/odoo/addons/connector_typesense @@ -0,0 +1 @@ +../../../../connector_typesense \ No newline at end of file diff --git a/setup/connector_typesense/setup.py b/setup/connector_typesense/setup.py new file mode 100644 index 00000000..28c57bb6 --- /dev/null +++ b/setup/connector_typesense/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)