From 6e59027e6d5f5be3736b56a7d81c87c1a5316a4a Mon Sep 17 00:00:00 2001 From: MrTango Date: Wed, 8 Nov 2023 13:14:04 +0100 Subject: [PATCH] [16.0][Add] connector_typesense: a connector to typsense search engine --- connector_typesense/README.rst | 108 +++ connector_typesense/__init__.py | 2 + connector_typesense/__manifest__.py | 20 + connector_typesense/demo/backend_demo.xml | 14 + .../demo/se_index_config_demo.xml | 22 + connector_typesense/models/__init__.py | 1 + connector_typesense/models/se_backend.py | 43 ++ connector_typesense/models/se_index.py | 19 + connector_typesense/readme/CONFIGURE.rst | 3 + connector_typesense/readme/CONTRIBUTORS.rst | 2 + connector_typesense/readme/DESCRIPTION.rst | 4 + connector_typesense/readme/INSTALL.rst | 2 + connector_typesense/readme/USAGE.rst | 2 + .../static/description/icon.png | Bin 0 -> 12321 bytes .../static/description/index.html | 445 +++++++++++++ connector_typesense/tests/__init__.py | 1 + ...ConnectorTypeSense.test_index_adapter.yaml | 76 +++ ...orTypeSense.test_index_adapter_delete.yaml | 297 +++++++++ ..._adapter_delete_nonexisting_documents.yaml | 40 ++ ...ctorTypeSense.test_index_adapter_iter.yaml | 295 ++++++++ ...rTypeSense.test_index_adapter_reindex.yaml | 628 ++++++++++++++++++ .../docker-compose.typesense.example.yml | 13 + .../tests/test_connector_typesense.py | 141 ++++ connector_typesense/tools/__init__.py | 2 + connector_typesense/tools/adapter.py | 260 ++++++++ connector_typesense/tools/serializer.py | 17 + connector_typesense/views/ts_backend.xml | 40 ++ requirements.txt | 1 + .../odoo/addons/connector_typesense | 1 + setup/connector_typesense/setup.py | 6 + 30 files changed, 2505 insertions(+) create mode 100644 connector_typesense/README.rst create mode 100644 connector_typesense/__init__.py create mode 100644 connector_typesense/__manifest__.py create mode 100644 connector_typesense/demo/backend_demo.xml create mode 100644 connector_typesense/demo/se_index_config_demo.xml create mode 100644 connector_typesense/models/__init__.py create mode 100644 connector_typesense/models/se_backend.py create mode 100644 connector_typesense/models/se_index.py create mode 100644 connector_typesense/readme/CONFIGURE.rst create mode 100644 connector_typesense/readme/CONTRIBUTORS.rst create mode 100644 connector_typesense/readme/DESCRIPTION.rst create mode 100644 connector_typesense/readme/INSTALL.rst create mode 100644 connector_typesense/readme/USAGE.rst create mode 100644 connector_typesense/static/description/icon.png create mode 100644 connector_typesense/static/description/index.html create mode 100644 connector_typesense/tests/__init__.py create mode 100644 connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter.yaml create mode 100644 connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete.yaml create mode 100644 connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete_nonexisting_documents.yaml create mode 100644 connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_iter.yaml create mode 100644 connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_reindex.yaml create mode 100644 connector_typesense/tests/docker-compose.typesense.example.yml create mode 100644 connector_typesense/tests/test_connector_typesense.py create mode 100644 connector_typesense/tools/__init__.py create mode 100644 connector_typesense/tools/adapter.py create mode 100644 connector_typesense/tools/serializer.py create mode 100644 connector_typesense/views/ts_backend.xml create mode 120000 setup/connector_typesense/odoo/addons/connector_typesense create mode 100644 setup/connector_typesense/setup.py diff --git a/connector_typesense/README.rst b/connector_typesense/README.rst new file mode 100644 index 00000000..244ee26f --- /dev/null +++ b/connector_typesense/README.rst @@ -0,0 +1,108 @@ +=================== +connector_typesense +=================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:f09633c3af59b153f0eba3f876a9835676c430a6f39dd0dfa3545a924c80bc11 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fsearch--engine-lightgray.png?logo=github + :target: https://github.com/OCA/search-engine/tree/16.0/connector_typesense + :alt: OCA/search-engine +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/search-engine-16-0/search-engine-16-0-connector_typesense + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/search-engine&target_branch=16.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This addon provides the bases to implement addons to export information to +Typesense_ indexes. + +.. _Typesense: https://typesense.org + +**Table of contents** + +.. contents:: + :local: + +Installation +============ + +This package requires a typesense search engine running. +Please read this for a [quick docker based setup](https://typesense.org/docs/guide/install-typesense.html#option-2-local-machine-self-hosting). + +Configuration +============= + +You have to configure (Host, Port, Protocol, Typesense API Key) in a new backend form view: + +Search Engine > Configuration > Backends + +Usage +===== + +a nice UI is also available here: https://github.com/bfritscher/typesense-dashboard/releases + + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* Derico +* Kencove + +Contributors +~~~~~~~~~~~~ + +* Maik Derstappen +* Mohamed Alkobrosli + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-Kencove| image:: https://github.com/Kencove.png?size=40px + :target: https://github.com/Kencove + :alt: Kencove + +Current `maintainer `__: + +|maintainer-Kencove| + +This module is part of the `OCA/search-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/connector_typesense/__init__.py b/connector_typesense/__init__.py new file mode 100644 index 00000000..738a2eec --- /dev/null +++ b/connector_typesense/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import tools diff --git a/connector_typesense/__manifest__.py b/connector_typesense/__manifest__.py new file mode 100644 index 00000000..77c785ca --- /dev/null +++ b/connector_typesense/__manifest__.py @@ -0,0 +1,20 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "connector_typesense", + "category": "Connector", + "summary": "Connector For Typesense Search Engine", + "version": "16.0.1.0.0", + "license": "AGPL-3", + "author": "Derico, Kencove, Odoo Community Association (OCA)", + "website": "https://github.com/OCA/search-engine", + "maintainers": ["Kencove"], + "depends": ["connector_search_engine"], + "data": [ + "views/ts_backend.xml", + ], + "demo": ["demo/se_index_config_demo.xml", "demo/backend_demo.xml"], + "external_dependencies": {"python": ["typesense==1.0.3", "requests"]}, + "installable": True, +} diff --git a/connector_typesense/demo/backend_demo.xml b/connector_typesense/demo/backend_demo.xml new file mode 100644 index 00000000..1cd56f01 --- /dev/null +++ b/connector_typesense/demo/backend_demo.xml @@ -0,0 +1,14 @@ + + + + + Demo Sale Channel Typesense + demo_sale_channel_typesense + typesense + localhost + 8108 + http + xyz + + diff --git a/connector_typesense/demo/se_index_config_demo.xml b/connector_typesense/demo/se_index_config_demo.xml new file mode 100644 index 00000000..fd714965 --- /dev/null +++ b/connector_typesense/demo/se_index_config_demo.xml @@ -0,0 +1,22 @@ + + + + TS Product Config + {} + +{ + "name": "ts_products_collection", + "fields": [ + { + "name": "id", + "type": "string" + }, + { + "name": "name", + "type": "string" + } + ] +} + + + diff --git a/connector_typesense/models/__init__.py b/connector_typesense/models/__init__.py new file mode 100644 index 00000000..b4cddc58 --- /dev/null +++ b/connector_typesense/models/__init__.py @@ -0,0 +1 @@ +from . import se_backend, se_index diff --git a/connector_typesense/models/se_backend.py b/connector_typesense/models/se_backend.py new file mode 100644 index 00000000..6fc263ae --- /dev/null +++ b/connector_typesense/models/se_backend.py @@ -0,0 +1,43 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + +from ..tools.adapter import TypesenseAdapter + + +class SeBackend(models.Model): + _inherit = "se.backend" + + backend_type = fields.Selection( + selection_add=[("typesense", "Typesense")], + ondelete={"typesense": "cascade"}, + string="Type", + required=True, + ) + ts_server_host = fields.Char( + string="Typesense host", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_server_port = fields.Char( + string="Typesense port", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_server_protocol = fields.Char( + string="Typesense protocol", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_server_timeout = fields.Integer( + string="Typesense server timeout", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + ts_api_key = fields.Char( + help="Typesense Api Key", + groups="connector_search_engine.group_connector_search_engine_manager", + ) + + def _get_adapter_class(self): + if self.backend_type == "typesense": + return TypesenseAdapter + else: + return super()._get_adapter_class() diff --git a/connector_typesense/models/se_index.py b/connector_typesense/models/se_index.py new file mode 100644 index 00000000..43d0c0dc --- /dev/null +++ b/connector_typesense/models/se_index.py @@ -0,0 +1,19 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + +from ..tools.serializer import TSJsonifySerializer + + +class SeIndex(models.Model): + + _inherit = "se.index" + + serializer_type = fields.Selection(selection_add=[("typesense", "Typesense")]) + + def _get_serializer(self): + if self.serializer_type == "typesense": + return TSJsonifySerializer() + else: + return super()._get_serializer() diff --git a/connector_typesense/readme/CONFIGURE.rst b/connector_typesense/readme/CONFIGURE.rst new file mode 100644 index 00000000..c1e532be --- /dev/null +++ b/connector_typesense/readme/CONFIGURE.rst @@ -0,0 +1,3 @@ +You have to configure (Host, Port, Protocol, Typesense API Key) in a new backend form view: + +Search Engine > Configuration > Backends diff --git a/connector_typesense/readme/CONTRIBUTORS.rst b/connector_typesense/readme/CONTRIBUTORS.rst new file mode 100644 index 00000000..38bcf55b --- /dev/null +++ b/connector_typesense/readme/CONTRIBUTORS.rst @@ -0,0 +1,2 @@ +* Maik Derstappen +* Mohamed Alkobrosli diff --git a/connector_typesense/readme/DESCRIPTION.rst b/connector_typesense/readme/DESCRIPTION.rst new file mode 100644 index 00000000..985e0128 --- /dev/null +++ b/connector_typesense/readme/DESCRIPTION.rst @@ -0,0 +1,4 @@ +This addon provides the bases to implement addons to export information to +Typesense_ indexes. + +.. _Typesense: https://typesense.org diff --git a/connector_typesense/readme/INSTALL.rst b/connector_typesense/readme/INSTALL.rst new file mode 100644 index 00000000..451f51b3 --- /dev/null +++ b/connector_typesense/readme/INSTALL.rst @@ -0,0 +1,2 @@ +This package requires a typesense search engine running. +Please read this for a [quick docker based setup](https://typesense.org/docs/guide/install-typesense.html#option-2-local-machine-self-hosting). diff --git a/connector_typesense/readme/USAGE.rst b/connector_typesense/readme/USAGE.rst new file mode 100644 index 00000000..60f089bd --- /dev/null +++ b/connector_typesense/readme/USAGE.rst @@ -0,0 +1,2 @@ +a nice UI is also available here: https://github.com/bfritscher/typesense-dashboard/releases + diff --git a/connector_typesense/static/description/icon.png b/connector_typesense/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..64e41565f310a40c1e472cbea0b05b910953446c GIT binary patch literal 12321 zcmb`NgLj68as_`+iGm9v28cj`+VQO;kVXkJ!@vonmhO0 zv(IPma7B3uBzQb{5D0`MB`Ky1JdgZ$hXn^7NqKojK_I4MDKTMHkIeHd7)_OY9k+ubWXDvX5|Rx^zFr^s)68A z`0Dw@tQC?6xVap@fUNGZ!bZo;7p}2?9MeUlV4> z&8_77@SdOs?&Xhdv>7`=C0-&oMSm@3fJ2AcT>VMyBI1a+maYm0x*i#`EKn6LUefT# zeU-f>DeAkuJp>UW>o)QE4s&6ZxUcg@%&my!2{I7>4yM`$p)wyMY2R1Xz3k%^io1Zh zml!^Wu-dL6AVQA|qL|yp#I9iRM&f4m3Hp}~j!NNay{TgkXWZ^fIGN%R4g^Fn$~iaf z4lH_gR{NV1dz|^>AR|#Zr<5RIC&=u+Dj%@A>U0uVbENSGNfj}!|0^fFFq0hUSo=V2rnXhzKbI7+#SwCbkor!`!YF~O>_q5i? z^E+!xjg&jn=33XeKl7;HoF5Hd3JV0HY3pM0PXXOE5&vNuovhFkl|F69(wMYG73_8$ zYzDVrg8s&ifiS$>xYG69fro}aZsu_r!Eu1*%(YdDNHUy3Z=-_rti5KP@FhU0^5sX{ zVuWXYXWdnFTFIra0KF}{#%ek5pS+jdcy+$)OGsHpG|1iuXpM|T)Lv(($U!oct`RKd zZCLCHtVlZInsr|NZ3&fotF7z|he?a2h2odzs*2j|FzU3s_s~5`*qt_=yvBvV&(^9g z_Ys^x;@tUO?RrWkWb~(5dz<5+HW>+ScDY8$3AAiJ<_SzU6@HD{kGGMxTDKiZnN!&< zv>WyQM-S4mHw{wt3;CQs)Rt3sMlRU1Lw8OdGgPSo>pt;8qZCTVK}K0<`)vV!982v; zUZRflp^|-H5JMok)ob+BRo#TgFqDR-|4}GiSGl@Z>Z67Kd32A8mAXHKrN(!Vc7Yf4 zjVW~WS8-@rJRhw)*(_j#3Q#(K%Zg^%dq(7xTB*>xo1@^)sI*Ca(3}Y{P>N!40qgH6 z>fc5^V0A8FlMs?mKc8PK7~R?^oyTnz8N(T78&!2qs;kO-ED9fXAb1xd7LYl~dXT^yCsvNF}ReNzws3}sgWQrj1WHFoA6I`k#iQM)o_4n^MHD$jyxBp*rZ4J8nEr;atRkVZ67AboT6a6BhhQf2L~{!`K+>iV&b$~dyE`H)bt3ocQg=dZ38 z@B5q+N!aSJi4+J0>RI^nFvC4wGB{?ec`|PCz%Hkc-@4V5j-3`^(8;txRr~3Sm7IaVtILJ!I3bDqPTL`L2X}MZ)bSfj5wI`+xwSH) zwaKy-KCXh0|8V;r+~kw*4G7X5I=3 z5jmVHKWYs1xG@AFGy^7s=bV&B!&Wtf-XvCKzPWRWE0JfJV(`DNBu}DpO?P z3l%BWYcX|aQ*pKN{Qi@xszIvlX7B8)pxav=G3=Y86Z(FfI-RRvx9`DnG4C+p0fv~r zx5?Nz&#>*f^NwUBcaW7=QN5%&LuQkNHCR|&2KH+Gs|Cft(? zKeFSD4@+40GE$goRsNIRn3w?@&(MCp6K{ zMv~{v(Bp4iLw4ZY2iJU$8pQ?!IUYZfMB#w@Ys<)86`52i5<~owk=~_8GT+^g6 zU}5?kU4U?_%+N5?>85okW5F5LQBI=U-03RK*dfup(E@jXyI3}Gl?{&s%lJ-pwQTfQ zRZ-u`l}2)1^8UEwG&3CQl?AJ?@A*1= z>(Gdhm{+L4hl?x**m-?sQNh*|FUA-nJX@HI!n0@Db7a>EEFtGf2?s=yRZMmvbjugB} zg7>q;^%O-gNTleVN;LZN0{i`XR>MdLxu0EA8J4ey6VHB}^tbalh;}NqpY2* z2eY|cpOVO5C%V46MU--2C9fs@!<(=}Gjh{nN>u%`)mvklL$~s7E zRUVtxa=EXrW4=&GBM|JJVY^P8VvUQB9&G>7WN|AQS0%hHsF({q20=}8TuPtZn|3kA zQ0AW~m*JR0l$k*=LrLe);KA#UAPLJCPYL#%XtB&{v9-G&x zr^L4C^UcrBa-(t;tDhQTXUI-rHFQc+wJd%o_q6*sht5plj*>XmEVAGOgAg)9O}8Ig zs-cf>T8;m!=%yrHybC9`p5`lO(CrZpNcQP?Scv23q#vF4@LK$66z}?@)mK^H9I98d z2$Osyz6mrbx17d{M4!gg<$P?F7Or`J8yIo&X;alxVjq}VP@d;=3g}i!3A~0g9_je| zZ4pWDm(iV+*4_JiU)m8$ldj;B=cVCYSB?X3actl4p7ExI0M?JEY3+`DbZDSutJpil z1F{HOf;x`+DIDSb@qM>a#jUpjO~IFa?0k3B&edTf!@?_AWVnVY&Du?80c+)KUIB?G zNP&5SE71r~o!2R^dvKKm8!o~;GSLGwCq%O;a0bj&RM zSGSs|HLHeXHU00M>uu+p80@Ec1^Wt{YIWQ9`t?!I;we;=WLgmD;lsA8MZ3~w3`=o4 zYEHE%OnxVUnmaOypi(%08{OGQQ@@a|WK9>rPxb~J%Mc_i$cm4m(H7azTa5&~U^(k|W24Wq6$cNsNXfl8CHaE5y;X4@N7 zSwfnlyq=o|^1+hThxZ>1UF&uyB$id0C(h)LOp)jD<^FFDb!o}WZH*N5i;)?T*W8_icS#JKH;iw~~&u=T-eFZG9d4B#6 zeK;xwD$ksKlG~ubo^L47V`ito8tQ_z87dU8O%20rhWWg`b4PZc+A2+<*|{4RgH(UA z^9D~JCLluRc6MD!>t{SPzmix*Os^M_z4d9~C0+iD3CY=Nm$pC~lEa|$Zi4Fgibfw6 zfmsR*2&d0|N9eyZhkZMTf3lF+DbWT3ih>ES$t>+q(T8{7Xpr4AZju#;r~hrVTvCz} z`R4!^QhzPpX%aCPZ>jXovyriyUu?o!WPFm7g|7`9uE&sj%zk3q|ad2(h@NnC67GE4A8Kyx=P}8xa)fQBoTlD8~wCUF*L$q4N z5G|CcRD|5#-JojL)QpRYk&b0`&Y5ibE{2OvOKw9rxL|sv9EDw7Qhla?gh(;^h$^^p zwK^-(XYZ8DSLKD3xzOLhgh&i$g51S@b}}oxz7v6iGC z9zuFu%1eb%G1$b&G0P5U-Rl@zJ=^#|o2Z2P;O2(~|EXO>*G=8@1h#4K&|BCb#Wm%g zPp|~q@q;LKTA-_rR}EV{r)%q2@W`j>`}SLJEuHeQgS+xRAp zfumD~EtZs{zl|w4gV&TEd<)N>2`rwzNx9^7KK)P;Gt0KBa}x<9ba78;^5a{4)K|3W zXz4@IKr%Zeb?kBC&t;4XV=6{i5Gc33@NCliZYE~%UMvL}GV6Pk^uumAHE&g3abw4M z{=abyE%V8!f4uz=WDmLT?=n+Z{;lKIG+)CaNKZ@(kckBvXL6FcrBt<-m9*BeP)EJ) zL&Y56h)*-i3BYW$7__gw-B`L1gM3ele1^cr?KwTNr6>~;k%)5Lgr6Y6ei07xmiSiO;I z0zLC9(ln~ka;4_X%jbcAWiqEwT#zkEq6>T|nLn@2=CUEkh4&j+Pg-{1B&h12*vQ%} z^Ilop9|9zT+O_f=!N2^J5{a=tcJru0CYKT7qsBd{nHe|gO_ue60qC)%r1#JB#HP4a zjXM(VK@lZR`%Bs6^=(*&8Kq zP9A2In$J6@_RM>wQ>kUs7~SgMiI8DL)UJg}wk9LwXp9Of-MW}XYuGYZL)wL5Z^hR( zPk>!LbwyVqV7<9Gh6gOkS`6foQ%Ovdi=4h`Wft>HzX4|SFi9@v; zt?>3HjyU;Ix&~{#@J^SwS`OW=+Vl)?;c=i{4 z!G;#6k(U8-_>VyjxJ15pUEV6AoeDI7WN;_M+`hax2`jmkXyAWe6|_W3?ZlC5^Q*qA zP{H`pLMDF$sk>?mCn4l_6~JcRQug9P&xEicj@Fx1A~Fhl*t=DM;!ql(a__ayPUH{{ zWdol5Gvm3@(}lw4up2$w$lo&xkP5-V9P@F(B2@g3&&O&V^Znv$dTExo`NYt#$TDiO z<%q8i5I#h>Q$5lBqEqY)l9u$ZzQp%1EN@t*s0!zN0&~i= zKOvDoh{ep;TQ`nM-_GTs_s$J%c!F5lAEd@#dmD11q`M!Jv332lyq;py0g=OrJc@|S z0+9KgFtzuTJvsRav>mwZqP?mi(nBAX`$?qH{GI@iS&_)=&aQ~JM&-(MaObsJpdOP` zc9R39-P6n;U*6T9#Pk*4l7w;jv>=CfO@?P+1;%A$DNuWoSM|}M1pSh0$`w*t8hVY1 zq-~l!t|1-QRf#BL|0wGftnI<^}R6jD{&rrkMJ|33Qs(Np-ZFCOSm8=qKdW0}-Hq3_pu8)`-$X5c>~}56i+@Ec~!O3dY}Qv3wz~J~5+}LHQ=-&&y_2 z6I&-atEDyPyF7}ZAyQRr*WR)xy6b}QUnen5_@wjKTbS#=nl-i%i$pr8t*^F|$q7dq zDTKY_RCbh5jzVyE~x!%CUL4AJ$CJPXd*#> zKBeeV@468|vO0zoGD+3u(ZIQTAS#u<+i=|kEB22J_+aCAj@GH>&S0$FxlX!^ntsab z65rPG1?-1n>o$mCp_rd<3uWO%=du|cnCgM3_4@*Z=sb%yi}OZ%#iTR1Yp(LgH7WY% zoIICF*%wb|DzDPfd)tGyX*WIfBgGjJ?O!+K)NU7F^sOrDCCV=oY#5r!ze84Ocl#__ z!N3c$EsGsHCa}xl^G2A+P5>C9YIJ&2G3^!5ZzBqiM7cxQ-*Oz|-u!!e#hZv0h)6Om%%p#v9YFg`z0s7GT zoe5SLM4!0xi7hEC$T^#+7wf@$VpA*b$`r2v=?{f!kqB2|4i0j)b8>vy(FA`1HW`&kxL$>x77L1ST>{HEb-xg0M^S(uLc6yYO1vxB?Zrb%EOmXPW2*T^NHK_lmYJjI3A(< z-y3x4&Rf-&(Xz%`W`zc+!oV81aO-nz>`|YToCy*CAy|-Xy0Nf^b9n-!&p}^_woy^P zC-$=m>xU_h3UuEjWOtcL5c#|yGb~C zuvb-_{64?-ReZpi>YL9nvfqc|-Oy*7-7i9Dx@77&0k`_&w_bA_tciFt=XbyNu(ZA?KKdJ}6ahT2+;HUUJcpBFt2TxZpfiJ2$07p|}o=7LW{l3C(g z+FR}qLMkljSSX|3X`_82A9Whqbp#lg~jPc&J1g z^RTtacvPF8crQ?1Zy%x-f29c~Y~u&;XSX>sFZ)y=WiTaUr60;WW>wsjXs`cs8QMGw z5BiK;=9gg-%-3 zfw^=<{V<^l*Z=6+VsZD8vD?|CJbE4H>shAB?=`Ca<>{@j?;5T`IlM_&fySYCfvnD+ z6vT4MKbjj{8-N)Kp?;NBYwiAGay8uJztA`w7%do^4yvFQFr7(%*InKt8j#mLRv;#I zgb%=&sy(>-I+Ze5kG8W+wZ!|NpRfEhWJ|38+QEvPq!R6vdJ|t(Diy6Ea(wKOs)Ym* zlPAAF6es#u27=A!z-kG+HnN3-FnRs@d_`dwS0Dfs7$#IPULQI4O;Al<2i@51-P-S! zt44Q98+Ar~%f=x<T%520=;LFDvS}?2v>&S z=)(4Cw=ELD9QB&w`P=j6ubiqQM#wl^y&dG;{TxtAMuEu$boANd7|QUEy&D|doBL1U zeP?_z-b$b@sykL-E%jkSD z@{Dhvu~k37r55K(#CF@gF@&^T2~^O?I9>g6=ZpPu%AFeJ*&N8^=A+}ctwFv7L75WN z|I3#domR)%+r6slU#y|IAgWhb@kn_`8wzxi!mz_k#a^eed$%bKjp=H#jiL7_2Q_suun8T(OFgeaQZ6{YA1t@ z7Rs#2{~Ka(7X*ThIS8V1jD1(DP(7B!TrWig1Ns(?xrmlLW{+qJI3QrR_OnvSR*n3NxWmegwS2W6gI?eE zStkQE$i!N_zg!vk-=(x{U@Y51;80XvzICyhwWsby>)u$PDw4UhXE%=g_;XyMkkg{B zj}|yd)!ps;ra~0$W%?a+q7L0T$O47C2&F9PMYMYfa=7;Ur^`t8A8JWVG55nX%k6eq zOUdRDRa1J40!r9zZ}ea=cD(u9)rk~E2r>q0Myi&D3vIq29k|dl2QpYLei3;xclvKpC%TPVAC)! zWx!XaU&11NiL)JmpwVAST+&G>%T$l651Sx}uZKg69Gm+lJpHIzI8(dajX)O{ax^lV z(P&V!OhgX9KI_;u=vWV*(WIW%zU{7dc{i25Xtz*@WqZUjb0=e<_MTqM^2Y7O5?9CA zbN`c=th5mM{AkzZ`80T~5Qb2F= z{3K7&z#h%brx8PE1Ufb2N>0MW2<7xIL{j}@TfE&Mo3%(CII5Th9oqV%f5G69we*cf>z_MM@kZ|#eTjd>IEot zjjHAJxm_S@KM17RwESc8qj#lWkddjmLK{x(qk3p9GI;^drYkQBUM8ti`^+mkVs@~g z6f(zZ;RP@O@}zqKAa^*Q*Oez*O4Iw2H1*I@AK(rCF59sS`&P;|h4PM$9m{@k6SsWh zW7$I6LCj${EM$Q(ERPPPB>p1@42Z})D4S?Z+O;J+?x8=ly}TPC(WjOsx)JO4NiDb zB9{NCyB}u-N25xys?M@%DTaT7oZWi${$o`EMg9DAJhWOnyB^2goG;_-=HH-q`5r&2 zzqP+*@@6qF_e{p}O3xU?&F`^=K-P%&&UxnV{abN? zL#Q~X(PB3n>ufr<+F;V*2%b9lHG`PrODK#)%pD$IIP%Yyf3?8REGWvB9D09zbTU_V z?dX!ubfDhbc36l}JIb~-?oy_01VF6znm&-Z3O=Q7`b%5s`l#Y!L$7@jW{R&M$r61& zuD!jbO3~d>G?vA3W&REZP^X1IZlxN0@j;Q&JEFnrN$>;-5{=0`{STykhm$=!!t<4% zv4ny)dpsMTxLYgw`$4vO9IskCAH`1dJ`?%6K>p<0gT`mEX#REyBOLF$J48Jp?DT-j zZyH%(SEaP>=|@(; zq-U$V7yac0Ui(ZA6U9EnUb8!I=ZJ52kn%F^@=f`EGsodK^;4oQOIk>`M`2M_p1)Ta z+n}=nB9+pM-4p;(&tD>Nrrz zPKfWKEqdQ{IYz40Be{4ng(sFZbiJ5EBX&!WLj@Z+IzIq{1q60!>%Jt3Q`ijm6kt4E zc@o!_s9XqbShg@Dssm_@wMc}owI#P)q2um#Y;kq-cc9-^OivDW3L`4?4E7&9F!0=- zSo69RE8s}r2P;`P-pceH8x=6#nB(h&ds_wQwv-veH2WX9@O_Bn8>sFtv#Q1A=>k&g?z6ZGACCo6f5L>oh zNQL+vU(9l9(EZ~T9W*N&RIGXk(Qn3CNi!73*+UBq?*H|HR+)?!T*jN3N(MyLV(rU= z&(UfQvSqU=WAKiGsdh2-((@Cl7L5;pZF|;M5}3b20fjvm1o()BG`kmDiC3W;F?h(F zZ#?u&=;49$)`)-!i0_7o;s)c%eTh@KKKu)a*$B5+Wo53+w;mqIfW=paiEWl$ARQqi zr~13#^`BfI4MpH!T(u-YER(6}$jEmCW8K*ipP>IGc>Ps=oit&JeH+^34TGyBLv%kZ z7lQhj%Z=zKLD=uFTyhg=ai6-rzDwGTUAZ?}fmnC9nKWV60OJ@a_5{p1ug;yf-ekDP z7o^LeW$h&!2aA*r2rh)hzZAXZ8hyjp_WEDjS;43BHo0}do?}sDc3_yq$Ztg?oW_}4 zH@UZRUZHk>rm8v^fsjByV}`o!Lbao4M!G9oU#@SdDcyTSXEbU2Kl9iKQ%|16E=}na7B@noqsmwZIugF{)In zg$jxV6XG?J33FRvx$(*+Q8|6F(8pKD217Ny_(6*ew4N0*wSK8Orfzbi=)J_1G%;%p zWt(EDdL!xTs}cpB=-z8|l{s@6;tF2X#dyiSqN#uHOVUX)CYucPCkbaja0#D`IkL`; zlkT1Lyp(Sj6s-o~`lX^;bD>KL)904GF0DqqU=d@~{=;@7GK1l&o^zq?*CaqIkzp1$ zkDu>5f4^lp>L3IUIh98!(MS8gXR?#Vmn`5h64x(|38!=}@MmS~V88)6PY zT0zO(jNlrU9wUK-1-~=+;)tA1f|v^x#UAV33>_Y5aBV$+U@%B^7>1ut1O0>i=tWcJ zR0J{^$Liel%-w#RVEW7Bj(`WHy$mm{BLi<|( zx6OPO1d^4Tv4`|)##31ge{RHhRM@M@&d&fuN zE^9)N?(mw*!iAf0sgB%#Zq|nQg>?S6t)YfV&E^02$(PSRAV5v@*_#8V=T6Ih&$TX) z-`mpVGkK#oC?v$i9n`=jCYRCB=$?3qS#BUfwN+)(16aQ5O1w8=6C^KUHlNaIqdp>A~45#M$G4OobN zEdYPid5Au~y=$~+L zWNVof!7%6%ReK;2Ah4Kw`~Ihwf + + + + +connector_typesense + + + +
+

connector_typesense

+ + +

Beta License: AGPL-3 OCA/search-engine Translate me on Weblate Try me on Runboat

+

This addon provides the bases to implement addons to export information to +Typesense indexes.

+

Table of contents

+ +
+

Installation

+

This package requires a typesense search engine running. +Please read this for a [quick docker based setup](https://typesense.org/docs/guide/install-typesense.html#option-2-local-machine-self-hosting).

+
+
+

Configuration

+

You have to configure (Host, Port, Protocol, Typesense API Key) in a new backend form view:

+

Search Engine > Configuration > Backends

+
+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Derico
    • +
  • Kencove
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

Kencove

+

This module is part of the OCA/search-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/connector_typesense/tests/__init__.py b/connector_typesense/tests/__init__.py new file mode 100644 index 00000000..fb2e1046 --- /dev/null +++ b/connector_typesense/tests/__init__.py @@ -0,0 +1 @@ +from . import test_connector_typesense diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter.yaml new file mode 100644 index 00000000..7cbd0917 --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter.yaml @@ -0,0 +1,76 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"created_at":1747351443,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "41", "name": "Marty McFly"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '35' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '16' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete.yaml new file mode 100644 index 00000000..7b66ca6a --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete.yaml @@ -0,0 +1,297 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2 + response: + body: + string: '{"created_at":1747351443,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":4,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-1", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351545,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-1"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "1", "name": "Foo"} + + {"id": "2", "name": "Bar"} + + {"id": "3", "name": "Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/?filter_by=id%3A%5B1%2C+3%5D + response: + body: + string: '{"num_deleted":2}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '17' + content-type: + - application/json + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/search?q=%2A + response: + body: + string: '{"facet_counts":[],"found":1,"hits":[{"document":{"id":"2","name":"Bar"},"highlight":{},"highlights":[]}],"out_of":1,"page":1,"request_params":{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","per_page":10,"q":"*"},"search_cutoff":false,"search_time_ms":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '272' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete_nonexisting_documents.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete_nonexisting_documents.yaml new file mode 100644 index 00000000..bb4db01d --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_delete_nonexisting_documents.yaml @@ -0,0 +1,40 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/?filter_by=id%3A%5B%27donotexist%27%2C+%27donotexisteither%27%5D + response: + body: + string: '{"num_deleted":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '17' + content-type: + - application/json + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_iter.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_iter.yaml new file mode 100644 index 00000000..bf49437a --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_iter.yaml @@ -0,0 +1,295 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1 + response: + body: + string: '{"created_at":1747351545,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":1,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-1", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351549,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-1"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"created_at":1747351549,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "1", "name": "Foo"} + + {"id": "2", "name": "Bar"} + + {"id": "3", "name": "Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/search?q=%2A + response: + body: + string: '{"facet_counts":[],"found":3,"hits":[{"document":{"id":"3","name":"Baz"},"highlight":{},"highlights":[]},{"document":{"id":"2","name":"Bar"},"highlight":{},"highlights":[]},{"document":{"id":"1","name":"Foo"},"highlight":{},"highlights":[]}],"out_of":3,"page":1,"request_params":{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","per_page":10,"q":"*"},"search_cutoff":false,"search_time_ms":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '408' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_reindex.yaml b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_reindex.yaml new file mode 100644 index 00000000..296601a9 --- /dev/null +++ b/connector_typesense/tests/cassettes/TestConnectorTypeSense.test_index_adapter_reindex.yaml @@ -0,0 +1,628 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1 + response: + body: + string: '{"created_at":1747351549,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-1", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351551,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-1"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"id": "1", "name": "Foo"} + + {"id": "2", "name": "Bar"} + + {"id": "3", "name": "Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/import?action=upsert + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-1","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1/documents/export + response: + body: + string: '{"id":"1","name":"Foo"} + + {"id":"2","name":"Bar"} + + {"id":"3","name":"Baz"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '71' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2 + response: + body: + string: '{"message": "Not Found"}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +- request: + body: '{"name": "demo_sale_channel_typesense_contact_en_us-2", "fields": [{"name": + "id", "type": "string"}, {"name": "name", "type": "string"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections + response: + body: + string: '{"created_at":1747351553,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":0,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + access-control-allow-origin: + - '*' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + status: + code: 201 + message: Created +- request: + body: '{"id":"1","name":"Foo"} + + {"id":"2","name":"Bar"} + + {"id":"3","name":"Baz"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '71' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: POST + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2/documents/import?action=create + response: + body: + string: '{"success":true} + + {"success":true} + + {"success":true}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '50' + content-type: + - text/plain; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-2 + response: + body: + string: '{"created_at":1747351553,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-2","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"collection_name": "demo_sale_channel_typesense_contact_en_us-2"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '66' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: PUT + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: DELETE + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us-1 + response: + body: + string: '{"created_at":1747351551,"default_sorting_field":"","enable_nested_fields":false,"fields":[{"facet":false,"index":true,"infix":false,"locale":"","name":"name","optional":false,"sort":false,"type":"string"}],"name":"demo_sale_channel_typesense_contact_en_us-1","num_documents":3,"symbols_to_index":[],"token_separators":[]}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '322' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/collections/demo_sale_channel_typesense_contact_en_us/documents/search?q=%2A + response: + body: + string: '{"facet_counts":[],"found":3,"hits":[{"document":{"id":"3","name":"Baz"},"highlight":{},"highlights":[]},{"document":{"id":"2","name":"Bar"},"highlight":{},"highlights":[]},{"document":{"id":"1","name":"Foo"},"highlight":{},"highlights":[]}],"out_of":3,"page":1,"request_params":{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","per_page":10,"q":"*"},"search_cutoff":false,"search_time_ms":0}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '408' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.25.1 + X-TYPESENSE-API-KEY: + - xyz + method: GET + uri: http://localhost:8108/aliases/demo_sale_channel_typesense_contact_en_us + response: + body: + string: '{"collection_name":"demo_sale_channel_typesense_contact_en_us-2","name":"demo_sale_channel_typesense_contact_en_us"}' + headers: + Connection: + - keep-alive + accept-ranges: + - none + access-control-allow-origin: + - '*' + content-length: + - '116' + content-type: + - application/json; charset=utf-8 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/connector_typesense/tests/docker-compose.typesense.example.yml b/connector_typesense/tests/docker-compose.typesense.example.yml new file mode 100644 index 00000000..3c5ffb02 --- /dev/null +++ b/connector_typesense/tests/docker-compose.typesense.example.yml @@ -0,0 +1,13 @@ +version: "3.8" + +services: + typesense-server: + image: typesense/typesense:0.25.2 + container_name: typesense-server + ports: + - "8108:8108" + volumes: + - /tmp/typesense-data:/data + command: > + --data-dir /data --api-key=xyz --listen-port=8108 --enable-cors + restart: unless-stopped diff --git a/connector_typesense/tests/test_connector_typesense.py b/connector_typesense/tests/test_connector_typesense.py new file mode 100644 index 00000000..299f9fa7 --- /dev/null +++ b/connector_typesense/tests/test_connector_typesense.py @@ -0,0 +1,141 @@ +# Copyright 2019 Kencove +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import json +from time import sleep + +from vcr_unittest import VCRMixin + +from odoo.addons.connector_search_engine.tests.test_all import TestBindingIndexBase + +from ..tools.adapter import TypesenseAdapter + + +class TestConnectorTypeSense(VCRMixin, TestBindingIndexBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend = cls.env.ref("connector_typesense.backend_1") + cls.se_config = cls.env.ref("connector_typesense.se_index_config_product") + cls.setup_records() + cls.adapter: TypesenseAdapter = cls.se_index.se_adapter + cls.demo_data = [ + {"id": "1", "name": "Foo"}, + {"id": "2", "name": "Bar"}, + {"id": "3", "name": "Baz"}, + ] + + def _get_vcr_kwargs(self, **kwargs): + return { + "record_mode": "one", + "match_on": ["method", "path", "query"], + "filter_headers": ["Authorization"], + "decode_compressed_response": True, + } + + @classmethod + def _prepare_index_values(cls, backend): + values = super()._prepare_index_values(backend) + values.update({"config_id": cls.se_config.id}) + return values + + def test_index_adapter(self): + # ts serialize type converts id type from integer into string + self.se_index.write({"serializer_type": "typesense"}) + # Export collection settings to cassette (typesense) + self.se_index.export_settings() + + data = self.partner_binding.data + self.assertFalse(data) + + # Recompute data to be sent + self.se_index.batch_recompute(force_export=True) + data = self.partner_binding.data + self.assertEqual(data["name"], "Marty McFly") + self.assertIsInstance(data["id"], str) + + # Set partner to be updated and patch data to cassette (typesense) + self.partner_binding.write({"state": "to_export"}) + self.se_index.batch_sync(force_export=True) + + # Ensure that calls have been recorded in cassette + self.assertTrue(self.cassette.all_played) + self.assertGreaterEqual(len(self.cassette.requests), 2) + + request_1 = self.cassette.requests[0] + self.assertEqual(request_1.method, "GET") + expected_path = f"/collections/{self.se_index.name}".lower() + self.assertEqual(self.parse_path(request_1.uri), expected_path) + response_1 = self.cassette.responses[0] + body = response_1.get("body").get("string").decode("utf-8") + lines = [line for line in filter(lambda line: line, body.split("\n"))] + # we must have 1 line for the index op + self.assertEqual(len(lines), 1) + index_action = json.loads(lines[0]) + self.assertEqual(index_action.get("name"), f"{self.se_index.name}-2".lower()) + + request_2 = self.cassette.requests[-1] + self.assertEqual(request_2.method, "POST") + body = request_2.body.decode("utf-8") + data = json.loads(body) + # Mock the preserved id in the cassette: 107 to new record id + data["id"] = self.partner_binding.data.get("id") + self.assertDictEqual(data, self.partner_binding.data) + + def test_index_config_as_str(self): + self.se_config.write({"body_str": '{"mappings": {"1":1}}'}) + self.assertDictEqual(self.se_config.body, {"mappings": {"1": 1}}) + self.assertEqual(self.se_config.body_str, '{"mappings": {"1":1}}') + + def test_index_adapter_iter(self): + data = self.demo_data + self.adapter.clear() + self.adapter.settings() + self.adapter.index(data) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + res = [x["document"] for x in self.adapter.each()] + res.sort(key=lambda d: d["id"]) + self.assertListEqual(res, data) + + def test_index_adapter_delete(self): + data = self.demo_data + self.adapter.clear() + self.adapter.index(data) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + self.adapter.delete([1, 3]) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + res = [x["document"] for x in self.adapter.each()] + res.sort(key=lambda d: d["id"]) + self.assertListEqual(res, [{"id": "2", "name": "Bar"}]) + + def test_index_adapter_delete_nonexisting_documents(self): + """We try to delete records that do not exist. + Because it does not matter, it is just ignored. No exception. + """ + self.adapter.delete(["donotexist", "donotexisteither"]) + + def test_index_adapter_reindex(self): + data = self.demo_data + self.adapter.clear() + self.adapter.index(data) + index_name = self.adapter._get_current_aliased_index_name() + next_index_name = self.adapter._get_next_aliased_index_name(index_name) + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + self.adapter.reindex() + if self.cassette.dirty: + # when we record the test we must wait for es + sleep(2) + res = [x["document"] for x in self.adapter.each()] + res.sort(key=lambda d: d["id"]) + self.assertListEqual(res, data) + self.assertEqual( + self.adapter._get_current_aliased_index_name(), next_index_name + ) diff --git a/connector_typesense/tools/__init__.py b/connector_typesense/tools/__init__.py new file mode 100644 index 00000000..307aa310 --- /dev/null +++ b/connector_typesense/tools/__init__.py @@ -0,0 +1,2 @@ +from . import adapter +from . import serializer diff --git a/connector_typesense/tools/adapter.py b/connector_typesense/tools/adapter.py new file mode 100644 index 00000000..bb5ffcdf --- /dev/null +++ b/connector_typesense/tools/adapter.py @@ -0,0 +1,260 @@ +# Copyright 2024 Derico +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import json +import logging +from typing import Any, Iterator + +import requests + +from odoo import _ +from odoo.exceptions import UserError + +from odoo.addons.connector_search_engine.tools.adapter import SearchEngineAdapter + +_logger = logging.getLogger(__name__) + + +try: + import typesense +except ImportError: + _logger.debug("Can not import typesense") + + +# def _is_delete_nonexistent_documents(elastic_exception): +# """True iff all errors in this exception are deleting a nonexisting document.""" +# b = lambda d: "delete" in d and d["delete"]["status"] == 404 # noqa +# return all(b(error) for error in elastic_exception.errors) + + +class TypesenseAdapter(SearchEngineAdapter): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__ts_client = None + + @property + def _index_name(self): + return self.index_record.name.lower() + + # @property + # def _ts_connection_class(self): + # return typesense.RequestsHttpConnection + + @property + def _ts_client(self): + if not self.__ts_client: + self.__ts_client = self._get_ts_client() + return self.__ts_client + + @property + def _index_config(self): + return self.index_record.config_id.body + + def _get_ts_client(self): + backend = self.backend_record + return typesense.Client( + { + "nodes": [ + { + "host": backend.ts_server_host, + "port": backend.ts_server_port, + "protocol": backend.ts_server_protocol, + } + ], + "api_key": backend.ts_api_key, + "connection_timeout_seconds": int(backend.ts_server_timeout) or 300, + } + ) + + def test_connection(self): + ts = self._ts_client + try: + ts.collections.retrieve() + except typesense.exceptions.ObjectNotFound as exc: + raise UserError( + _("Not Found - The requested resource is not found.") + ) from exc + except typesense.exceptions.RequestUnauthorized as exc: + raise UserError(_("Unauthorized - Your API key is wrong.")) from exc + except requests.exceptions.ConnectionError as exc: + raise UserError(_("Unable to connect :") + "\n\n" + repr(exc)) from exc + except requests.exceptions.InvalidURL as exc: + raise UserError( + _("Invalid URL - No host supplied") + "\n\n" + repr(exc) + ) from exc + + def index(self, records) -> None: + ts = self._ts_client + # Convert records to JSONL string (one JSON object per line) + jsonl_data = "\n".join([json.dumps(record) for record in records]) + try: + res = ts.collections[self._index_name].documents.import_( + jsonl_data, {"action": "upsert"} + ) + except typesense.exceptions.ObjectNotFound as e: + _logger.warning( + f"{self._index_name} not found, skip updating alias and " + f"creating a new index (collection)!\n\n{e}" + ) + self.reindex() + self.index(records) + # Validate number of successfully indexed documents + results = [json.loads(line) for line in res.splitlines()] + success_count = sum(1 for item in results if item["success"]) + if success_count != len(records): + self.clear() + self.index(records) + + def delete(self, binding_ids) -> None: + ts = self._ts_client + ts.collections[self._index_name].documents.delete( + {"filter_by": f"id:{binding_ids}"} + ) + + def clear(self) -> None: + ts = self._ts_client + index_name = self._get_current_aliased_index_name() or self._index_name + ts.collections[index_name].delete() + self.settings() + + def each(self) -> Iterator[dict[str, Any]]: + ts = self._ts_client + res = ts.collections[self._index_name].documents.search( + { + "q": "*", + } + ) + if not res: + # eg: empty index + return + hits = res["hits"] + for hit in hits: + yield hit + + def settings(self) -> None: + ts = self._ts_client + try: + ts.collections[self._index_name].retrieve() + except typesense.exceptions.ObjectNotFound: + client = self._ts_client + # To allow rolling updates, we work with index aliases + aliased_index_name = self._get_next_aliased_index_name() + # index_name / collection_name is part of the schema defined in + # self._index_config + index_config = self._index_config + index_config.update( + { + "name": aliased_index_name, + } + ) + _logger.info(f"Create aliased_index_name '{aliased_index_name}'...") + client.collections.create(index_config) + _logger.info( + f"Set collection alias '{self._index_name}' >> aliased_index_name " + f"'{aliased_index_name}'." + ) + client.aliases.upsert( + self._index_name, {"collection_name": aliased_index_name} + ) + + def _get_current_aliased_index_name(self) -> str: + """Get the current aliased index name if any""" + current_aliased_index_name = None + try: + alias = self._ts_client.aliases[self._index_name].retrieve() + if "collection_name" in alias: + current_aliased_index_name = alias["collection_name"] + except typesense.exceptions.ObjectNotFound as e: + _logger.warning( + f"current_aliased_index_name not found, skip updating alias and " + f"creating a new index (collection)!\n\n{e}" + ) + return current_aliased_index_name + + def _get_next_aliased_index_name( + self, aliased_index_name: str | None = None + ) -> str: + """Get the next aliased index name + + The next aliased index name is based on the current aliased index name. + It's the current aliased index name incremented by 1. + + :param aliased_index_name: the current aliased index name + :return: the next aliased index name + """ + next_version = 1 + if aliased_index_name: + next_version = int(aliased_index_name.split("-")[-1]) + 1 + return f"{self._index_name}-{next_version}" + + def reindex(self) -> None: + """Reindex records according to the current config + This method is useful to allows a rolling update of index + configuration. + This process is based on the following steps: + 1. export data from current aliased index + 2. create a new index (collection) with the current config + 3. import data into new aliased index (collection) + 4. Update the index alias to point to the new aliased index (collection) + 5. Drop the old index. + """ + client = self._ts_client + current_aliased_index_name = self._get_current_aliased_index_name() + if not current_aliased_index_name: + self.settings() + current_aliased_index_name = self._get_current_aliased_index_name() + data = client.collections[current_aliased_index_name].documents.export() + next_aliased_index_name = self._get_next_aliased_index_name( + current_aliased_index_name + ) + try: + client.collections[next_aliased_index_name].retrieve() + except typesense.exceptions.ObjectNotFound: + # To allow rolling updates, we work with index aliases + # index_name / collection_name is part of the schema defined + # in self._index_config + _logger.info( + f"Create new_aliased_index_name '{next_aliased_index_name}'..." + ) + index_config = self._index_config + index_config.update( + { + "name": next_aliased_index_name, + } + ) + client.collections.create(index_config) + _logger.info( + f"Import existing data into new_aliased_index_name " + f"'{next_aliased_index_name}'..." + ) + client.collections[next_aliased_index_name].documents.import_( + data.encode("utf-8"), {"action": "create"} + ) + + try: + client.collections[next_aliased_index_name].retrieve() + except typesense.exceptions.ObjectNotFound as e: + _logger.warning( + f"New aliased_index_name not found, skip updating alias and " + f"not removing old index (collection)!\n\n{e}" + ) + else: + _logger.info( + f"Set collection alias '{self._index_name}' >> " + f"new_aliased_index_name '{next_aliased_index_name}'." + ) + client.aliases.upsert( + self._index_name, {"collection_name": next_aliased_index_name} + ) + _logger.info( + f"Remove old aliased index (collection) " + f"'{current_aliased_index_name}'." + ) + client.collections[current_aliased_index_name].delete() + + else: + _logger.warning( + f"next_aliased_index_name '{next_aliased_index_name}' " + f"already exists, skip!", + self._index_name, + ) diff --git a/connector_typesense/tools/serializer.py b/connector_typesense/tools/serializer.py new file mode 100644 index 00000000..d305af89 --- /dev/null +++ b/connector_typesense/tools/serializer.py @@ -0,0 +1,17 @@ +# Copyright 2023 Kencove (https://kencove.com). +# @author Mohamed Alkobrosli +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo.addons.connector_search_engine.tools.serializer import ModelSerializer + + +class TSJsonifySerializer(ModelSerializer): + def serialize(self, record): + # Typesense SE requires id field to be string + # It also requires other intger field types to be searchable + data = {} + if record.id: + data["id"] = str(record.id) + if record.name: + data["name"] = record.name + return data diff --git a/connector_typesense/views/ts_backend.xml b/connector_typesense/views/ts_backend.xml new file mode 100644 index 00000000..9358d8b4 --- /dev/null +++ b/connector_typesense/views/ts_backend.xml @@ -0,0 +1,40 @@ + + + + se.backend + + + + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index 4ac43d30..b0a37f9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ elasticsearch>=7.0.0,<=7.13.4 pydantic requests +typesense==1.0.3 typing-extensions unidecode diff --git a/setup/connector_typesense/odoo/addons/connector_typesense b/setup/connector_typesense/odoo/addons/connector_typesense new file mode 120000 index 00000000..1ffbdd91 --- /dev/null +++ b/setup/connector_typesense/odoo/addons/connector_typesense @@ -0,0 +1 @@ +../../../../connector_typesense \ No newline at end of file diff --git a/setup/connector_typesense/setup.py b/setup/connector_typesense/setup.py new file mode 100644 index 00000000..28c57bb6 --- /dev/null +++ b/setup/connector_typesense/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)