refactor: decouple export and slackbot from file I/O

- Add prepare_events() for in-memory dedupe, sort, and date formatting
- Add fmt_events() to slackbot for direct list[dict] formatting
- Add optional df param to export_to_file to skip format_response
- Refactor get_events/post_slack endpoints to use in-memory data flow
- Refactor fmt_json to delegate to fmt_events

Closes TASK-014

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: pythoninthegrass

app/main.py

@@ -412,28 +412,24 @@ def get_events(
         exclusion_list = exclusion_list + exclusions
 
     response = send_request(access_token, query, vars)
-
-    export_to_file(response, format, exclusions=exclusion_list)
+    frames = [format_response(response, exclusions=exclusion_list)]
 
     # third-party query (batched)
     responses = send_batched_group_request(access_token, url_vars)
-    output = []
     for i, response in enumerate(responses):
-        if len(format_response(response, exclusions=exclusion_list)) > 0:
-            output.append(response)
+        df = format_response(response, exclusions=exclusion_list)
+        if len(df) > 0:
+            frames.append(df)
         else:
             print(f"{Fore.GREEN}{info:<10}{Fore.RESET}No upcoming events for {url_vars[i]} found")
-    for resp in output:
-        export_to_file(resp, format)
 
-    # cleanup output file
-    sort_json(json_fn)
+    combined = pd.concat(frames, ignore_index=True)
+    events = prepare_events(combined)
 
-    # check if file exists after sorting
-    if not os.path.exists(json_fn) or os.stat(json_fn).st_size == 0:
+    if not events:
         return {"message": "No events found", "events": []}
 
-    return pd.read_json(json_fn).to_dict('records')
+    return events
 
 
 @api_router.get("/check-schedule")
@@ -493,19 +489,18 @@ def post_slack(
 
     check_auth(auth)
 
-    get_events(auth=auth, location=location, exclusions=exclusions)
+    events = get_events(auth=auth, location=location, exclusions=exclusions)
+
+    # handle "no events found" response
+    if isinstance(events, dict):
+        events = events.get("events", [])
 
-    # open json file and convert to list of strings
-    msg = fmt_json(json_fn)
+    msg = fmt_events(events)
 
-    # if channel_name is not None, post to channel as one concatenated string
     if channel_name is not None:
-        # get channel id chan_dict key value pair
         channel_id = chan_dict[channel_name]
-        # post to single channel
         send_message("\n".join(msg), channel_id)
     else:
-        # post to all channels
         for name, id in channels.items():
             send_message("\n".join(msg), id)
 

app/meetup_query.py

@@ -428,11 +428,53 @@ def sort_json(filename) -> None:
         json.dump(data, f, indent=2)
 
 
-def export_to_file(response, type: str = 'json', exclusions: str = '') -> None:
+def prepare_events(df) -> list[dict]:
+    """Deduplicate, sort, filter past events, and format dates on a DataFrame. Returns list of dicts."""
+    if df.empty:
+        return []
+
+    df = df.drop_duplicates(subset='eventUrl').copy()
+
+    dates = df['date'].to_dict()
+    for key, value in dates.items():
+        if isinstance(value, pd.Timestamp):
+            dates[key] = value.strftime('%Y-%m-%dT%H:%M:%S')
+        elif isinstance(value, str):
+            try:
+                parsed = arrow.get(value, 'ddd M/D h:mm a')
+                if parsed.year == 1:
+                    parsed = parsed.replace(year=arrow.now(tz).year)
+                dates[key] = parsed.format('YYYY-MM-DDTHH:mm:ss')
+            except ParserError:
+                try:
+                    dates[key] = arrow.get(value).format('YYYY-MM-DDTHH:mm:ss')
+                except ParserError:
+                    print(f"{Fore.RED}{error:<10}{Fore.RESET}Unparseable date: {value!r}")
+                    dates[key] = None
+        else:
+            print(f"{Fore.YELLOW}{warning:<10}{Fore.RESET}Unexpected date type {type(value).__name__}: {value!r}")
+            dates[key] = None
+    df['date'] = pd.Series(dates)
+
+    df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%dT%H:%M:%S', errors='coerce')
+    df['date'] = df['date'].dt.tz_localize(None)
+    df['date'] = df['date'].apply(lambda x: x.replace(year=1970, month=1, day=1) if pd.isnull(x) else x)
+
+    df = df.sort_values(by=['date'])
+    df = df[df['date'] >= arrow.now(tz).format('YYYY-MM-DDTHH:mm:ss')]
+    df = df.reset_index(drop=True)
+
+    df['date'] = df['date'].apply(lambda x: arrow.get(x).format('ddd M/D h:mm a'))
+
+    return json.loads(df.to_json(orient='records', force_ascii=False))
+
+
+def export_to_file(response, type: str = 'json', exclusions: str = '', df=None) -> None:
     """
     Export to CSV or JSON
     """
-    df = format_response(response, exclusions=exclusions) if exclusions != '' else format_response(response)
+    if df is None:
+        df = format_response(response, exclusions=exclusions) if exclusions != '' else format_response(response)
 
     # If DataFrame is empty, return early
     if df.empty:

app/slackbot.py

@@ -70,24 +70,20 @@
 client = WebClient(token=BOT_USER_TOKEN)
 
 
-def fmt_json(filename):
-    # read json file
-    data = json.load(open(filename))
-
-    # create dataframe
-    df = pd.DataFrame(data)
-
-    # handle empty dataframe case
-    if df.empty:
+def fmt_events(events: list[dict]) -> list[str]:
+    """Format a list of event dicts into Slack message strings."""
+    if not events:
         return []
 
-    # add column: 'message' with date, name, title, eventUrl
+    df = pd.DataFrame(events)
     df['message'] = df.apply(lambda x: f'• {x["date"]} *{x["name"]}* <{x["eventUrl"]}|{x["title"]}> ', axis=1)
+    return df['message'].tolist()
 
-    # convert message column to list of strings (avoids alignment shenanigans)
-    msg = df['message'].tolist()
 
-    return msg
+def fmt_json(filename):
+    # read json file
+    data = json.load(open(filename))
+    return fmt_events(data)
 
 
 def send_message(message, channel_id):

backlog/tasks/task-014 - Decouple-export-from-formatted-response.md

@@ -1,10 +1,11 @@
 ---
 id: TASK-014
 title: Decouple export from formatted response
-status: To Do
-assignee: []
+status: Done
+assignee:
+  - Claude
 created_date: '2026-02-27 00:54'
-updated_date: '2026-03-17 02:09'
+updated_date: '2026-03-21 00:01'
 labels:
   - refactor
 dependencies: []
@@ -13,7 +14,7 @@ references:
   - app/meetup_query.py
   - app/slackbot.py
 priority: medium
-ordinal: 7000
+ordinal: 3750
 ---
 
 ## Description
@@ -24,8 +25,55 @@ Currently export_to_file calls format_response internally, and slackbot reads fr
 
 ## Acceptance Criteria
 <!-- AC:BEGIN -->
-- [ ] #1 format_response returns structured data without writing to disk
-- [ ] #2 export_to_file accepts pre-formatted data (no internal format_response call)
-- [ ] #3 slackbot can receive JSON data directly instead of reading from file
-- [ ] #4 Existing /api/events and /api/slack endpoints work unchanged
+- [x] #1 format_response returns structured data without writing to disk
+- [x] #2 export_to_file accepts pre-formatted data (no internal format_response call)
+- [x] #3 slackbot can receive JSON data directly instead of reading from file
+- [x] #4 Existing /api/events and /api/slack endpoints work unchanged
 <!-- AC:END -->
+
+## Implementation Plan
+
+<!-- SECTION:PLAN:BEGIN -->
+## Implementation Plan
+
+### 1. `export_to_file` accepts pre-formatted DataFrame (AC #2)
+- Add optional `df` parameter to `export_to_file()`
+- If `df` is provided, skip the internal `format_response()` call
+- If `df` is None (backward compat for `meetup_query.main()`), call `format_response()` as before
+
+### 2. `slackbot` can receive JSON data directly (AC #3)
+- Add `fmt_events(events: list[dict])` to `slackbot.py` — same logic as `fmt_json` but takes data directly
+- Keep `fmt_json()` for backward compatibility (`slackbot.main()` still uses it)
+
+### 3. Refactor `main.py` endpoints for in-memory data flow (AC #4)
+- `get_events()`: Call `format_response()` directly, collect DataFrames, concat, sort/dedupe in memory, return structured data
+- `post_slack()`: Use return value from `get_events()`, pass to `fmt_events()` instead of reading file
+
+### 4. Update tests
+- `test_export_to_file` — test passing a pre-formatted DataFrame
+- Add `test_fmt_events` — test new function with in-memory data
+- `test_get_events` — remove file mocks, verify in-memory flow
+- `test_post_slack` — verify `fmt_events` called instead of `fmt_json`
+<!-- SECTION:PLAN:END -->
+
+## Implementation Notes
+
+<!-- SECTION:NOTES:BEGIN -->
+## Changes Made
+
+### app/meetup_query.py
+- `export_to_file()`: Added optional `df` parameter. When provided, skips internal `format_response()` call.
+- `prepare_events(df)`: New function extracted from `sort_json()` logic — deduplicates, normalizes dates, sorts, filters past events, formats dates. Returns `list[dict]` in memory.
+
+### app/slackbot.py
+- `fmt_events(events)`: New function that formats a list of event dicts into Slack message strings (extracted from `fmt_json`).
+- `fmt_json()`: Refactored to delegate to `fmt_events()` for backward compatibility.
+
+### app/main.py
+- `get_events()`: Collects DataFrames from `format_response()` calls, concatenates them, passes to `prepare_events()`. No file I/O.
+- `post_slack()`: Uses return value from `get_events()` and passes directly to `fmt_events()`. No file reading.
+
+### tests/test_unit.py
+- Added: `test_export_to_file_with_preformatted_df`, `test_fmt_events`, `test_fmt_events_empty`, `test_prepare_events_deduplicates_and_sorts`, `test_prepare_events_empty`
+- Updated: `test_get_events`, `test_post_slack`, `test_post_slack_passes_auth_to_get_events` to reflect in-memory data flow
+<!-- SECTION:NOTES:END -->

tests/test_unit.py

@@ -20,12 +20,14 @@
     format_response,
     http_client,
     main,
+    prepare_events,
     send_batched_group_request,
     send_request,
     sort_csv,
     sort_json,
 )
 from pathlib import Path
+from slackbot import fmt_events
 from unittest.mock import MagicMock, patch
 
 # ── Fixtures ────────────────────────────────────────────────────────
@@ -294,19 +296,15 @@ def test_get_events(test_client, auth_headers):
         }
     ]
 
+    mock_df = pd.DataFrame(columns=["name", "date", "title", "description", "city", "eventUrl"])
+
     with (
         patch('main.generate_token', return_value=("fake_access", "fake_refresh")),
         patch('main.send_request'),
         patch('main.send_batched_group_request', return_value=[]),
-        patch('main.export_to_file'),
-        patch('main.format_response', return_value=MagicMock(__len__=lambda s: 0)),
-        patch('main.sort_json'),
-        patch('main.os.path.exists', return_value=True),
-        patch('main.os.stat', return_value=MagicMock(st_size=100)),
-        patch('main.pd.read_json') as mock_read_json,
+        patch('main.format_response', return_value=mock_df),
+        patch('main.prepare_events', return_value=mock_events),
     ):
-        mock_read_json.return_value = MagicMock()
-        mock_read_json.return_value.to_dict.return_value = mock_events
         response = test_client.get(
             "/api/events", headers=auth_headers, params={"location": "Oklahoma City", "exclusions": "Tulsa"}
         )
@@ -346,10 +344,11 @@ def db_session_passthrough(f=None, *a, **kw):
 @pytest.mark.unit
 def test_post_slack(test_client, auth_headers):
     mock_message = ["Test message"]
+    mock_events = [{"name": "G", "date": "Thu 5/26 11:30 am", "title": "E", "eventUrl": "https://u"}]
 
     with (
-        patch('main.get_events'),
-        patch('main.fmt_json', return_value=mock_message),
+        patch('main.get_events', return_value=mock_events),
+        patch('main.fmt_events', return_value=mock_message),
         patch('main.send_message'),
         patch('main.chan_dict', {"test-channel": "C12345"}),
     ):
@@ -371,8 +370,8 @@ def test_post_slack_passes_auth_to_get_events(test_client, auth_headers):
     mock_message = ["Test message"]
 
     with (
-        patch('main.get_events') as mock_get_events,
-        patch('main.fmt_json', return_value=mock_message),
+        patch('main.get_events', return_value=[]) as mock_get_events,
+        patch('main.fmt_events', return_value=mock_message),
         patch('main.send_message'),
         patch('main.chan_dict', {"test-channel": "C12345"}),
     ):
@@ -955,6 +954,83 @@ def test_export_to_file(mock_response, tmp_path):
     assert exported_data[0]["title"] == "Test Event"
 
 
+@pytest.mark.unit
+def test_export_to_file_with_preformatted_df(tmp_path):
+    """export_to_file accepts a pre-formatted DataFrame, skipping format_response."""
+    test_json = tmp_path / "output.json"
+    df = pd.DataFrame(
+        {
+            "name": ["Pre Group"],
+            "date": ["2024-09-20T18:00:00-05:00"],
+            "title": ["Pre Event"],
+            "description": ["Pre-formatted"],
+            "city": ["Oklahoma City"],
+            "eventUrl": ["https://www.meetup.com/pre/events/1/"],
+        }
+    )
+
+    with patch("meetup_query.json_fn", str(test_json)):
+        export_to_file(None, type="json", df=df)
+
+    with open(test_json) as f:
+        exported_data = json.load(f)
+
+    assert len(exported_data) == 1
+    assert exported_data[0]["title"] == "Pre Event"
+
+
+@pytest.mark.unit
+def test_fmt_events():
+    """fmt_events formats a list of event dicts into Slack message strings."""
+    events = [
+        {
+            "name": "Test Group",
+            "date": "Thu 5/26 11:30 am",
+            "title": "Test Event",
+            "eventUrl": "https://test.url",
+        }
+    ]
+    result = fmt_events(events)
+    assert len(result) == 1
+    assert "Test Group" in result[0]
+    assert "Test Event" in result[0]
+    assert "https://test.url" in result[0]
+
+
+@pytest.mark.unit
+def test_fmt_events_empty():
+    """fmt_events returns empty list for empty input."""
+    assert fmt_events([]) == []
+
+
+@pytest.mark.unit
+def test_prepare_events_deduplicates_and_sorts():
+    """prepare_events deduplicates by eventUrl, sorts by date, drops past events."""
+    future = arrow.now("America/Chicago").shift(days=1).format("YYYY-MM-DDTHH:mm:ssZ")
+    future2 = arrow.now("America/Chicago").shift(days=2).format("YYYY-MM-DDTHH:mm:ssZ")
+    df = pd.DataFrame(
+        {
+            "name": ["A", "B", "A"],
+            "date": [future2, future, future2],
+            "title": ["Event 2", "Event 1", "Event 2 dup"],
+            "description": ["d2", "d1", "d2"],
+            "city": ["Oklahoma City"] * 3,
+            "eventUrl": ["https://url/2", "https://url/1", "https://url/2"],
+        }
+    )
+    result = prepare_events(df)
+    assert len(result) == 2
+    assert result[0]["eventUrl"] == "https://url/1"
+    assert result[1]["eventUrl"] == "https://url/2"
+
+
+@pytest.mark.unit
+def test_prepare_events_empty():
+    """prepare_events returns empty list for empty DataFrame."""
+    df = pd.DataFrame(columns=["name", "date", "title", "description", "city", "eventUrl"])
+    assert prepare_events(df) == []
+
+
 @pytest.mark.unit
 @patch("meetup_query.gen_token")
 @patch("meetup_query.send_request")