Skip to content

Commit 1572758

Browse files
cre-oscre-os
authored
Add tests for xml validation (#10)
* Add tests for XML validation scenarios * Fix empty config bug * Reorganize imports
1 parent 8d1cc42 commit 1572758

13 files changed

Lines changed: 215 additions & 10 deletions

‎docs/how_it_works.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,11 @@ source code to see how the lower level steps are stitched together.
211211

212212
Extracting the data from the database and converting it back to XML follow similar steps, in reverse order.
213213

214+
!!! info
215+
Extracting the data from the database is not very optimized and is actually currently quite slow, mostly due to
216+
complex join queries to retrieve data based on a filter only on the top node. This feature is currently useful
217+
for roundtrip test, but has limited value otherwise, because of its poor performance compared to loading.
218+
214219
### Querying data from the database
215220

216221
Walking through the data model tree, we query all tables using a chain of joins to the root table, on which we apply the

‎src/xml2db/document.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import datetime
33
import logging
44
from io import BytesIO
5-
from typing import Union, TYPE_CHECKING, Dict
5+
from typing import Union, TYPE_CHECKING
66
from zoneinfo import ZoneInfo
77
from sqlalchemy import Column, Table, text, select
88
from sqlalchemy.engine import Connection
@@ -12,7 +12,6 @@
1212
if TYPE_CHECKING:
1313
from .model import DataModel
1414

15-
from .exceptions import DataModelConfigError
1615
from .xml_converter import XMLConverter
1716

1817
logger = logging.getLogger(__name__)

‎src/xml2db/model.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ def __init__(
7979
temp_prefix: str = None,
8080
):
8181
self.model_config = self._validate_config(model_config)
82-
self.tables_config = model_config.get("tables", {})
82+
self.tables_config = model_config.get("tables", {}) if model_config else {}
8383

8484
self.xml_schema = xmlschema.XMLSchema(
8585
os.path.basename(xsd_file) if base_url is None else xsd_file,

‎src/xml2db/xml_converter.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66
from io import BytesIO
77
from itertools import zip_longest
88

9-
from .exceptions import DataModelConfigError
10-
119

1210
if typing.TYPE_CHECKING:
1311
from .model import DataModel

‎tests/fixtures.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
import pytest
21
import os
32

3+
import pytest
4+
45
from xml2db import DataModel
56

67

‎tests/sample_models/models.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1+
import hashlib
12
import os.path
3+
24
import sqlalchemy
3-
import hashlib
45

56

67
def make_sample_index(table_name):

‎tests/sample_models/orders/invalid_xml/invalid.xml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<?xml version='1.0' encoding='utf-8'?>
2+
<orders xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" batch_id="4" xsi:noNamespaceSchemaLocation="../orders.xsd">
3+
<shiporder orderid="3" processed_at="2023-09-11T18:27:56.000+02:00">
4+
<orderperson>
5+
<name>Alice</name>
6+
<address>string</address>
7+
<city>string</city>
8+
<zip codingSystem="int">21093</zip>
9+
<country>US</country>
10+
</orderperson>
11+
<shiptoContact>
12+
<name>Bob</name>
13+
<address>string</address>
14+
<city>string</city>
15+
<zip codingSystem="us">102832</zip>
16+
<country>FR</country>
17+
<phoneNumber>+1732897354</phoneNumber>
18+
<phoneNumber>+1732323984</phoneNumber>
19+
<companyId>
20+
<bic>JIDAZIO786DAZH</bic>
21+
</companyId>
22+
</shiptoContact>
23+
<item>
24+
<product>
25+
<name>product 1</name>
26+
<version>premium</version>
27+
</product>
28+
<quantity>10</quantity>
29+
<price>12.3</price>
30+
</item>
31+
<item>
32+
<product>
33+
<name>product 1</name>
34+
<version>premium</version>
35+
</product>
36+
<note>string</note>
37+
<quantity>21</quantity>
38+
<price>70.4</price>
39+
</item>
40+
</shiporder>
41+
</orders>

‎tests/sample_models/orders/invalid_xml/malformed_no_recover.xml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?xml version='1.0' encoding='utf-8'?>
2+
<orders xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" batch_id="2" xsi:noNamespaceSchemaLocation="../orders.xsd">
3+
<shiporder orderid="2" processed_at="2023-09-10T09:39:45.000+02:00">
4+
<orderperson>
5+
<name>Alice</name>
6+
<address>string</address>
7+
<city>string</city>
8+
<zip codingSystem="int">21093</zip>ing</country>
9+
<compan
10+
<product>
11+
<name>product 2</name>
12+
<version>regular</version>
13+
</product>
14+
<quantity>43</quantity>
15+
<price>30.3</price>
16+
</item>
17+
<item>
18+
<prodgular</version>
19+
</product>
20+
<note>string</note>
21+
<quantity>21</quantity>
22+
<price>70.4</price>
23+
</item>
24+
<item>
25+
<product>
26+
<name>product 1</name>
27+
<version>premium</version>
28+
</product>
29+
<quantity>2</quantity>
30+
<price>12.2</price>
31+
</ite

‎tests/sample_models/orders/invalid_xml/malformed_recover.xml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?xml version='1.0' encoding='utf-8'?>
2+
<orders xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" batch_id="5" xsi:noNamespaceSchemaLocation="../orders.xsd">
3+
<shiporder orderid="1" processed_at="2023-09-10T09:37:00.000+02:00">
4+
<orderperson>
5+
<name>Bob</name>
6+
<address>string</address>
7+
<city>string</city>
8+
<zip codingSystem="us">102832</zip>
9+
<country>FR</country>
10+
<phoneNumber>+1732897354</phoneNumber>
11+
<phoneNumber>+1732323984</phoneNumber>
12+
<companyId>
13+
<bic>JIDAZIO786DAZH</bic>
14+
</companyId>
15+
</orderperson>
16+
<item>
17+
<product>
18+
<name>product 1</name>
19+
<version>regular</version>
20+
</product>
21+
<quantity>13</quantity>
22+
<price>340.23</price>
23+
</item>
24+
<item>
25+
<product>
26+
<name>product 2</name>
27+
<version>regular</version>
28+
</product>
29+
<note>string</note>
30+
<quantity>8</quantity>
31+
<price>56.2</price>
32+
</item>
33+
<item>
34+
<product>
35+
<name>product 1</name>
36+
<version>premium</version>
37+
</product>
38+
<quantity>25</quantity>
39+
<price>21.7</price>
40+
</item>
41+
</shiporder>
42+
</orders>
43+
Hello World!

‎tests/test_conversions.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import os
2+
23
import pytest
34
from lxml import etree
45

56
from xml2db import DataModel
67
from xml2db.xml_converter import XMLConverter, remove_record_hash
7-
88
from .sample_models import models
99

1010

0 commit comments

Comments
 (0)