Event: Supports mapping a Guidle tag to multiple tags

The import tagmap now reads repeated source rows as multiple target tags, so a single Guidle/JSON tag can expand into several onegov tags. Example: 'Brauchtum / Fest': ['Brauchtum', 'Fest']. A single tag in Guidle maps to multiple tags in onegov-cloud

TYPE: Feature
LINK: ogc-3208

Author: Tschuppi81

src/onegov/event/cli.py

@@ -51,6 +51,20 @@
 cli = command_group()
 
 
+def parse_tagmap(tagmap_file: TextIOWrapper) -> dict[str, list[str]]:
+    """ Reads a tagmap CSV, mapping each source tag to its target tags.
+
+    Repeating a source tag on multiple rows maps it to multiple target tags.
+
+    """
+    tagmap: dict[str, list[str]] = {}
+    for row in csvreader(tagmap_file):
+        if len(row) < 2:
+            continue
+        tagmap.setdefault(row[0], []).append(row[1])
+    return tagmap
+
+
 @cli.command('clear')
 @pass_group_context
 def clear(
@@ -185,7 +199,7 @@ def import_json(
 
     """
     if tagmap_file is not None:
-        tagmap = {row[0]: row[1] for row in csvreader(tagmap_file)}
+        tagmap = parse_tagmap(tagmap_file)
     else:
         tagmap = None
 
@@ -251,7 +265,11 @@ def _import_json(request: CoreRequest, app: Framework) -> None:
             tags = item['cat1']
             if tagmap and tags:
                 unknown_tags |= set(tags) - tagmap.keys()
-                tags = {tagmap[tag] for tag in tags if tag in tagmap}
+                tags = {
+                    target
+                    for tag in tags if tag in tagmap
+                    for target in tagmap[tag]
+                }
 
             coordinates = None
             if item['latitude'] and item['longitude']:
@@ -428,7 +446,7 @@ def import_guidle(
 
     """
     if tagmap_file is not None:
-        tagmap = {row[0]: row[1] for row in csvreader(tagmap_file)}
+        tagmap = parse_tagmap(tagmap_file)
     else:
         tagmap = None
 

src/onegov/event/utils/guidle.py

@@ -217,9 +217,13 @@ def pdf(self) -> tuple[str, str] | tuple[None, None]:
 
     def tags(
         self,
-        tagmap: Mapping[str, str] | None = None
+        tagmap: Mapping[str, Sequence[str]] | None = None
     ) -> tuple[set[str], set[str]]:
-        """ Returns a set of known and a set of unkonwn tags. """
+        """ Returns a set of known and a set of unkonwn tags.
+
+        A source tag in the tagmap may map to more than one target tag.
+
+        """
 
         tag_elements = self.find(
             'guidle:classifications/'
@@ -233,7 +237,11 @@ def tags(
         }
         if tagmap:
             return (
-                {tagmap[tag] for tag in tags if tag in tagmap},
+                {
+                    target
+                    for tag in tags if tag in tagmap
+                    for target in tagmap[tag]
+                },
                 tags - tagmap.keys()
             )
         return tags, set()

tests/onegov/event/fixtures/guidle.xml

@@ -94,6 +94,35 @@ Lorem ipsum dolor sit amet, consetetur <a href="lorem.de">sadipscing</a> elitr,
                     </guidle:classification>
                 </guidle:classifications>
             </guidle:offer>
+            <guidle:offer id="551262855">
+                <guidle:lastUpdateDate>2017-10-21T22:44:12.834+02:00</guidle:lastUpdateDate>
+                <guidle:offerDetail id="551262858" languageCode="de">
+                    <guidle:title>Vortrag ohne Tags</guidle:title>
+                </guidle:offerDetail>
+                <guidle:schedules>
+                    <guidle:date>
+                        <guidle:startDate>2018-10-26</guidle:startDate>
+                        <guidle:startTime>20:00:00</guidle:startTime>
+                    </guidle:date>
+                </guidle:schedules>
+            </guidle:offer>
+            <guidle:offer id="551262856">
+                <guidle:lastUpdateDate>2017-10-21T22:44:12.834+02:00</guidle:lastUpdateDate>
+                <guidle:offerDetail id="551262859" languageCode="de">
+                    <guidle:title>Lesung mit unbekanntem Tag</guidle:title>
+                </guidle:offerDetail>
+                <guidle:schedules>
+                    <guidle:date>
+                        <guidle:startDate>2018-10-26</guidle:startDate>
+                        <guidle:startTime>20:00:00</guidle:startTime>
+                    </guidle:date>
+                </guidle:schedules>
+                <guidle:classifications>
+                    <guidle:classification id="204591954" name="Veranstaltungskalender" type="PRIMARY">
+                        <guidle:tag id="1585" name="Schauspiel" subcategoryId="204592149" subcategoryName="Theater"/>
+                    </guidle:classification>
+                </guidle:classifications>
+            </guidle:offer>
         </guidle:group>
     </guidle:groupSet>
 </guidle:exportData>

tests/onegov/event/test_cli.py

@@ -5,11 +5,17 @@
 from click.testing import CliRunner
 from onegov.core.utils import module_path
 from onegov.event.cli import cli
+from onegov.event.models import Event
 from os import path
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
 
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from onegov.core.orm import SessionManager
+
+
 def test_import_ical(cfg_path: str, temporary_directory: str) -> None:
     runner = CliRunner()
 
@@ -140,6 +146,7 @@ def test_import_ical(cfg_path: str, temporary_directory: str) -> None:
 def test_import_guidle(
     cfg_path: str,
     temporary_directory: str,
+    session_manager: SessionManager,
     xml: str
 ) -> None:
 
@@ -157,7 +164,7 @@ def test_import_guidle(
         ])
     assert result.exit_code == 0
     assert "Events successfully imported" in result.output
-    assert "4 added, 0 updated, 0 deleted" in result.output
+    assert "6 added, 0 updated, 0 deleted" in result.output
 
     # Reimport, not changed due to last_update
     with patch('onegov.event.cli.niquests.get', return_value=response):
@@ -190,10 +197,14 @@ def test_import_guidle(
     ], 'y')
     assert result.exit_code == 0
 
-    # Create tagmap
+    # Create tagmap, mapping 'Konzert Pop / Rock / Jazz' to two tags
     tagmap = path.join(temporary_directory, 'tagmap.csv')
     with open(tagmap, 'w') as f:
-        f.write("Konzert Pop / Rock / Jazz,Konzert\nSport,Sport")
+        f.write(
+            "Konzert Pop / Rock / Jazz,Konzert\n"
+            "Konzert Pop / Rock / Jazz,Musik\n"
+            "Sport,Sport"
+        )
 
     # Re-import with tagmap
     with patch('onegov.event.cli.niquests.get', return_value=response):
@@ -205,8 +216,31 @@ def test_import_guidle(
         ])
     assert result.exit_code == 0
     assert "Events successfully imported" in result.output
-    assert "Tags not in tagmap: \"Kulinarik\"!"
-    assert "4 added, 0 updated, 0 deleted" in result.output
+    assert "Tags not in tagmap:" in result.output
+    assert '"Kulinarik"' in result.output
+    assert '"Theater"' in result.output
+    assert "6 added, 0 updated, 0 deleted" in result.output
+
+    # 'Konzert Pop / Rock / Jazz' maps to two tags, both of which are applied,
+    # while the unmapped 'Kulinarik' is dropped without dropping the event
+    session_manager.set_current_schema('foo-bar')
+    events = session_manager.session().query(Event).all()
+    assert len(events) == 6
+
+    tagged = [e for e in events if e.tags]
+    untagged = [e for e in events if not e.tags]
+
+    # the four schedules of the offer with a mapped tag keep both target tags
+    assert len(tagged) == 4
+    assert all(sorted(e.tags) == ['Konzert', 'Musik'] for e in tagged)
+
+    # the offer without any tags and the offer whose only tag is not in the
+    # tagmap are both imported anyway, just without any tags
+    assert len(untagged) == 2
+    assert {e.title for e in untagged} == {
+        'Vortrag ohne Tags',
+        'Lesung mit unbekanntem Tag',
+    }
 
 
 @pytest.mark.parametrize("xml", [

tests/onegov/event/test_utils.py

@@ -36,7 +36,7 @@ def tzdatetime(
 ])
 def test_import_guidle(session: Session, xml: str) -> None:
     offers = list(GuidleExportData(etree.parse(xml)).offers())
-    assert len(offers) == 1
+    assert len(offers) == 3
 
     assert offers[0].uid == '551262854'
     assert offers[0].title == "Theatervorstellung"
@@ -66,9 +66,28 @@ def test_import_guidle(session: Session, xml: str) -> None:
         {'Konzert Pop / Rock / Jazz', 'Kulinarik'},
         set()
     )
-    assert offers[0].tags({'Konzert Pop / Rock / Jazz': 'Konzert'}) == (
+    assert offers[0].tags({'Konzert Pop / Rock / Jazz': ['Konzert']}) == (
         {'Konzert'}, {'Kulinarik'}
     )
+    assert offers[0].tags({
+        'Konzert Pop / Rock / Jazz': ['Konzert', 'Musik'],
+        'Kulinarik': ['Kulinarik'],
+    }) == (
+        {'Konzert', 'Musik', 'Kulinarik'}, set()
+    )
+
+    # an offer without any tags yields no known and no unknown tags
+    assert offers[1].tags() == (set(), set())
+    assert offers[1].tags({'Konzert Pop / Rock / Jazz': ['Konzert']}) == (
+        set(), set()
+    )
+
+    # an offer whose tags are not in the tagmap yields no known tags, but
+    # reports all of them as unknown
+    assert offers[2].tags() == ({'Theater'}, set())
+    assert offers[2].tags({'Konzert Pop / Rock / Jazz': ['Konzert']}) == (
+        set(), {'Theater'}
+    )
 
     schedules = list(offers[0].schedules())
     assert len(schedules) == 4