Skip to content

Commit 3019c78

Browse files
authored
add narrative pipelline test (#2307)
* add narrative pipelline test * change filename * slight mocking adjustment * mock sentence transformer * better evoc * try massaging mock data again * more mocking * diff mock strategy * fix cov report * test 500 gen embed * syntax fixes * update sytax again * syntax fix again * attempt mock fix * another mock attempt * fix action * fix action again * actions fix * add another test * add another test * fix test
1 parent af27bba commit 3019c78

5 files changed

Lines changed: 376 additions & 1 deletion

File tree

.github/workflows/python-ci.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ jobs:
7474
docker compose -f docker-compose.test.yml cp delphi/real_data delphi:/app/real_data
7575
echo "Copying coverage script into container..."
7676
docker compose -f docker-compose.test.yml cp delphi/generate_coverage_md.py delphi:/app/generate_coverage_md.py
77+
echo "Copying script to be tested into container..."
78+
docker compose -f docker-compose.test.yml cp delphi/polismath/run_math_pipeline.py delphi:/app/run_math_pipeline.py
79+
docker compose -f docker-compose.test.yml cp delphi/umap_narrative delphi:/app/umap_narrative
7780
7881
echo "Running tests and generating coverage report..."
7982
docker compose \
@@ -94,7 +97,7 @@ jobs:
9497
python create_dynamodb_tables.py --region us-east-1; \
9598
echo '--- Running Pytest ---'; \
9699
export PYTHONPATH=\$PYTHONPATH:/app; \
97-
pytest --cov=polismath --cov-report=xml:/app/coverage.xml /app/tests --ignore=/app/tests/test_pakistan_conversation.py
100+
pytest --cov=polismath --cov=run_math_pipeline --cov=./umap_narrative --cov-report=xml:/app/coverage.xml /app/tests --ignore=/app/tests/test_pakistan_conversation.py
98101
echo '--- Generating Coverage Comment Text ---'; \
99102
python /app/generate_coverage_md.py > /app/coverage-comment.md \
100103
"
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import os
2+
import sys
3+
from unittest import mock
4+
import pytest
5+
import numpy as np
6+
import importlib
7+
8+
# Add the 'umap_narrative' directory to the Python path to allow the target script to be imported.
9+
umap_narrative_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'umap_narrative'))
10+
if umap_narrative_dir not in sys.path:
11+
sys.path.insert(0, umap_narrative_dir)
12+
13+
@pytest.fixture(autouse=True)
14+
def setup_and_teardown(tmp_path, monkeypatch):
15+
"""Fixture to set up a clean environment for each test."""
16+
monkeypatch.setenv("ANTHROPIC_API_KEY", "mock_key_for_testing")
17+
cwd = os.getcwd()
18+
os.chdir(tmp_path)
19+
yield
20+
os.chdir(cwd)
21+
22+
def test_pipeline_flow_with_mocks(tmp_path):
23+
"""
24+
Tests the control flow of the script's mock data path.
25+
26+
This test verifies that when using `--use-mock-data`, the script calls the
27+
correct subset of data processing and storage functions.
28+
"""
29+
zid = "98765"
30+
test_args = [
31+
"500_generate_embedding_umap_cluster.py",
32+
"--use-mock-data",
33+
"--zid", zid,
34+
]
35+
36+
num_comments = 100
37+
num_layers = 3
38+
39+
# Define a valid, predictable return value for the ML processing step.
40+
mock_process_comments_return_value = (
41+
np.random.rand(num_comments, 2), # document_map
42+
np.random.rand(num_comments, 32), # document_vectors
43+
[np.random.randint(0, 5, num_comments) for _ in range(num_layers)], # cluster_layers
44+
[f"comment text {i}" for i in range(num_comments)], # comment_texts
45+
[str(i) for i in range(num_comments)] # comment_ids
46+
)
47+
48+
# Import the module to be tested programmatically.
49+
generate_embedding_module = importlib.import_module("500_generate_embedding_umap_cluster")
50+
51+
# Patch the external dependencies.
52+
with mock.patch.object(generate_embedding_module, 'process_comments', return_value=mock_process_comments_return_value) as mock_process_comments, \
53+
mock.patch.object(generate_embedding_module, 'DataConverter') as MockDataConverter, \
54+
mock.patch.object(generate_embedding_module, 'DynamoDBStorage') as MockDynamoStorage:
55+
56+
# Configure mocks to return simple, non-empty data.
57+
MockDataConverter.create_conversation_meta.return_value = "mock_meta_model"
58+
MockDataConverter.batch_convert_cluster_characteristics.return_value = ["mock_char_model"]
59+
60+
mock_dynamo_instance = mock.MagicMock()
61+
MockDynamoStorage.return_value = mock_dynamo_instance
62+
63+
# Run the main function from the script.
64+
with mock.patch.object(sys, 'argv', test_args):
65+
try:
66+
generate_embedding_module.main()
67+
except SystemExit as e:
68+
pytest.fail(f"Script exited unexpectedly: {e}")
69+
70+
# Assert that the mocked functions were called as expected for the mock data path.
71+
mock_process_comments.assert_called_once()
72+
MockDynamoStorage.assert_called_once()
73+
74+
# Assert that the DataConverter was used for the methods called in the mock path.
75+
MockDataConverter.create_conversation_meta.assert_called_once()
76+
assert MockDataConverter.batch_convert_cluster_characteristics.call_count == num_layers
77+
78+
# Assert that the correct subset of DynamoDB methods were called.
79+
mock_dynamo_instance.create_conversation_meta.assert_called_with("mock_meta_model")
80+
assert mock_dynamo_instance.batch_create_cluster_characteristics.call_count == num_layers
81+
mock_dynamo_instance.batch_create_cluster_characteristics.assert_called_with(["mock_char_model"])
82+
83+
# Assert that methods NOT in the mock data path were NOT called.
84+
mock_dynamo_instance.batch_create_comment_embeddings.assert_not_called()
85+
mock_dynamo_instance.batch_create_graph_edges.assert_not_called()
86+
mock_dynamo_instance.batch_create_comment_clusters.assert_not_called()
87+
mock_dynamo_instance.batch_create_topics.assert_not_called()
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import sys
2+
import os
3+
from unittest import mock
4+
import pytest
5+
import importlib
6+
7+
# Add the 'umap_narrative' directory to the Python path to allow the script to be imported.
8+
umap_narrative_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'umap_narrative'))
9+
if umap_narrative_dir not in sys.path:
10+
sys.path.insert(0, umap_narrative_dir)
11+
12+
# Import the module and function to be tested using importlib
13+
extremity_module = importlib.import_module("501_calculate_comment_extremity")
14+
calculate_and_store_extremity = extremity_module.calculate_and_store_extremity
15+
16+
def test_calculate_and_store_extremity_with_mocks():
17+
"""
18+
Tests the main logic of calculate_and_store_extremity by mocking its dependencies.
19+
- Mocks GroupDataProcessor to avoid database calls.
20+
- Mocks check_existing_extremity_values to force recalculation.
21+
- Verifies that the function correctly processes the mock output.
22+
"""
23+
conversation_id = 12345
24+
25+
# 1. Define a mock return value for the GroupDataProcessor
26+
mock_export_data = {
27+
'comments': [
28+
{'comment_id': 101, 'comment_extremity': 0.85},
29+
{'comment_id': 102, 'comment_extremity': 0.25},
30+
{'comment_id': 103, 'comment_extremity': 0.50},
31+
# A comment that might be missing the extremity value
32+
{'comment_id': 104},
33+
]
34+
}
35+
36+
# 2. Patch the dependencies within the script's namespace
37+
with mock.patch.object(extremity_module, 'GroupDataProcessor') as MockGroupDataProcessor, \
38+
mock.patch.object(extremity_module, 'check_existing_extremity_values', return_value={}) as mock_check_existing:
39+
40+
# Configure the mock instance of GroupDataProcessor
41+
mock_processor_instance = mock.MagicMock()
42+
mock_processor_instance.get_export_data.return_value = mock_export_data
43+
MockGroupDataProcessor.return_value = mock_processor_instance
44+
45+
# 3. Call the actual function to be tested
46+
result = calculate_and_store_extremity(conversation_id, force_recalculation=True)
47+
48+
# 4. Assert the results
49+
# Assert that the function correctly extracted the extremity values from the mock data
50+
expected_result = {
51+
101: 0.85,
52+
102: 0.25,
53+
103: 0.50,
54+
104: 0, # Should default to 0 if key is missing
55+
}
56+
assert result == expected_result, "The returned extremity values do not match the expected output."
57+
58+
# Assert that the dependencies were called as expected
59+
mock_check_existing.assert_not_called() # Should not be called when force_recalculation is True
60+
MockGroupDataProcessor.assert_called_once()
61+
mock_processor_instance.get_export_data.assert_called_once_with(conversation_id, False)
62+
63+
def test_check_for_existing_values(monkeypatch):
64+
"""
65+
Tests that the main function returns existing values and skips recalculation
66+
if they are found and `force` is False.
67+
"""
68+
conversation_id = 54321
69+
existing_values = {201: 0.9, 202: 0.1}
70+
71+
# Patch the check function and the GroupDataProcessor class
72+
with mock.patch.object(extremity_module, 'check_existing_extremity_values', return_value=existing_values) as mock_check_existing, \
73+
mock.patch.object(extremity_module, 'GroupDataProcessor') as MockGroupDataProcessor:
74+
75+
# Configure the mock instance that the class will produce upon instantiation
76+
mock_processor_instance = mock.MagicMock()
77+
MockGroupDataProcessor.return_value = mock_processor_instance
78+
79+
# Call the function with force_recalculation=False
80+
result = calculate_and_store_extremity(conversation_id, force_recalculation=False)
81+
82+
# Assert that the function correctly returned the pre-existing values
83+
assert result == existing_values
84+
85+
# Assert that the check for existing values was performed
86+
mock_check_existing.assert_called_once_with(conversation_id)
87+
88+
# Assert that GroupDataProcessor was instantiated (due to the script's structure)
89+
MockGroupDataProcessor.assert_called_once()
90+
91+
# Crucially, assert that the expensive calculation method was NOT called on the instance
92+
mock_processor_instance.get_export_data.assert_not_called()
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import sys
2+
import os
3+
from unittest import mock
4+
import pytest
5+
6+
# Add the 'umap_narrative' directory to the Python path to allow the script to be imported
7+
umap_narrative_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'umap_narrative'))
8+
if umap_narrative_dir not in sys.path:
9+
sys.path.insert(0, umap_narrative_dir)
10+
11+
# Now we can import the main function from the script to be tested
12+
from reset_conversation import main as reset_conversation_main
13+
14+
@pytest.fixture
15+
def mock_boto3_resources():
16+
"""Mocks the get_boto_resource function to return mock objects for S3 and DynamoDB."""
17+
18+
# --- Create Mock for S3 ---
19+
mock_s3_resource = mock.MagicMock()
20+
mock_bucket = mock.MagicMock()
21+
mock_s3_object = mock.MagicMock()
22+
mock_s3_object.key = 'visualizations/test-rid/some_file.html'
23+
# Configure the mock bucket's objects.filter to return a list containing our mock object
24+
mock_bucket.objects.filter.return_value = [mock_s3_object]
25+
# The Bucket() method of the S3 resource will return our mock bucket
26+
mock_s3_resource.Bucket.return_value = mock_bucket
27+
28+
# --- Create Mock for DynamoDB ---
29+
mock_dynamodb_resource = mock.MagicMock()
30+
mock_table = mock.MagicMock()
31+
# Configure the query/scan methods to return one fake item to trigger the deletion logic
32+
mock_table.query.return_value = {'Items': [{'pk': 'some_key', 'sk': 'some_sort_key'}]}
33+
mock_table.scan.return_value = {'Items': [{'pk': 'some_key', 'sk': 'some_sort_key'}]}
34+
# The Table() method of the DynamoDB resource will return our mock table
35+
mock_dynamodb_resource.Table.return_value = mock_table
36+
37+
# --- Create the main mock that replaces get_boto_resource ---
38+
with mock.patch('reset_conversation.get_boto_resource') as mock_get_resource:
39+
# Define a side effect to return the correct mock based on service name
40+
def get_resource_side_effect(service_name):
41+
if service_name == 'dynamodb':
42+
return mock_dynamodb_resource
43+
if service_name == 's3':
44+
return mock_s3_resource
45+
return mock.MagicMock()
46+
47+
mock_get_resource.side_effect = get_resource_side_effect
48+
49+
# Yield the mocks to the test function
50+
yield {
51+
"get_resource": mock_get_resource,
52+
"dynamodb": mock_dynamodb_resource,
53+
"s3": mock_s3_resource,
54+
"table": mock_table,
55+
"bucket": mock_bucket
56+
}
57+
58+
def test_reset_conversation_calls_all_services(mock_boto3_resources):
59+
"""
60+
Tests that the main reset script calls both DynamoDB and S3 deletion logic.
61+
"""
62+
test_zid = "12345"
63+
test_rid = "r_test_12345"
64+
65+
# Run the main function with test arguments
66+
reset_conversation_main(zid=test_zid, rid=test_rid)
67+
68+
# 1. Assert that our main mock was called for both services
69+
mock_boto3_resources["get_resource"].assert_any_call("dynamodb")
70+
mock_boto3_resources["get_resource"].assert_any_call("s3")
71+
72+
# 2. Assert that the script tried to get a DynamoDB table
73+
# It will be called many times, so just check it was called at all
74+
mock_boto3_resources["dynamodb"].Table.assert_called()
75+
76+
# 3. Assert that a deletion was attempted on a table
77+
# This confirms that the query/scan + delete loop was entered
78+
mock_table = mock_boto3_resources["table"]
79+
# Check that batch_writer (for query results) or delete_item (for single items) was called
80+
assert mock_table.batch_writer.called or mock_table.delete_item.called
81+
82+
# 4. Assert that the script tried to access the S3 bucket
83+
mock_boto3_resources["s3"].Bucket.assert_called_with(mock.ANY) # bucket name is from env
84+
85+
# 5. Assert that the script attempted to delete S3 objects
86+
mock_bucket = mock_boto3_resources["bucket"]
87+
mock_bucket.delete_objects.assert_called_once()
88+
89+
def test_reset_conversation_skips_s3_if_no_rid(mock_boto3_resources):
90+
"""
91+
Tests that S3 deletion is skipped if no report_id (rid) is provided.
92+
"""
93+
test_zid = "54321"
94+
95+
# Run the main function without the 'rid' argument
96+
reset_conversation_main(zid=test_zid, rid=None)
97+
98+
# Assert that the DynamoDB deletion logic was still called
99+
mock_boto3_resources["get_resource"].assert_any_call("dynamodb")
100+
mock_boto3_resources["dynamodb"].Table.assert_called()
101+
102+
# Assert that the S3 logic was SKIPPED
103+
mock_bucket = mock_boto3_resources["bucket"]
104+
mock_bucket.delete_objects.assert_not_called()
105+
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import os
2+
import sys
3+
from unittest import mock
4+
import pytest
5+
import numpy as np
6+
7+
# Add the 'umap_narrative' directory to the Python path to import 'run_pipeline'
8+
umap_narrative_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'umap_narrative'))
9+
if umap_narrative_dir not in sys.path:
10+
sys.path.insert(0, umap_narrative_dir)
11+
12+
# Now we can import the main function from the script we want to test
13+
from run_pipeline import main as run_pipeline_main
14+
15+
@pytest.fixture(autouse=True)
16+
def setup_and_teardown(tmp_path, monkeypatch):
17+
"""
18+
This fixture will be used by all tests in this module.
19+
- It changes the current working directory to a temporary directory.
20+
- It restores the original working directory after the test.
21+
- It mocks the ANTHROPIC_API_KEY to avoid warnings.
22+
"""
23+
monkeypatch.setenv("ANTHROPIC_API_KEY", "mock_key_for_testing")
24+
cwd = os.getcwd()
25+
os.chdir(tmp_path)
26+
yield
27+
os.chdir(cwd)
28+
29+
def test_pipeline_calls_correct_functions(tmp_path):
30+
"""
31+
Tests the pipeline's control flow by mocking major functions and asserting
32+
that they are called correctly, instead of asserting on file creation.
33+
This avoids failures related to external library rendering issues.
34+
"""
35+
zid = "12345"
36+
test_args = [
37+
"run_pipeline.py",
38+
"--use-mock-data",
39+
"--zid", zid,
40+
"--no-dynamo",
41+
]
42+
43+
num_comments = 100
44+
mock_called = False
45+
46+
def process_comments_side_effect(*args, **kwargs):
47+
nonlocal mock_called
48+
mock_called = True
49+
return (
50+
np.random.rand(num_comments, 2),
51+
np.random.rand(num_comments, 32),
52+
[np.random.randint(0, 5, num_comments) for _ in range(3)],
53+
[f"comment text {i}" for i in range(num_comments)],
54+
[i for i in range(num_comments)]
55+
)
56+
57+
# Patch all major functions to test the control flow
58+
with mock.patch('run_pipeline.process_comments', side_effect=process_comments_side_effect), \
59+
mock.patch('run_pipeline.create_basic_layer_visualization') as mock_create_basic, \
60+
mock.patch('run_pipeline.create_named_layer_visualization') as mock_create_named, \
61+
mock.patch('run_pipeline.create_enhanced_multilayer_index') as mock_create_index:
62+
63+
# Ensure the mocked visualization function returns a mock file path
64+
mock_create_named.return_value = "mock/path/to/file.html"
65+
66+
with mock.patch.object(sys, 'argv', test_args):
67+
try:
68+
run_pipeline_main()
69+
except SystemExit as e:
70+
pytest.fail(f"run_pipeline.py exited unexpectedly: {e}")
71+
72+
# 1. Assert that our primary mock was called, confirming the setup is correct.
73+
assert mock_called, "The mock for run_pipeline.process_comments was not called."
74+
75+
# 2. Assert that the visualization functions were called for each of the 3 mock layers.
76+
assert mock_create_basic.call_count == 3, f"Expected basic visualization to be called 3 times, but was called {mock_create_basic.call_count} times."
77+
assert mock_create_named.call_count == 3, f"Expected named visualization to be called 3 times, but was called {mock_create_named.call_count} times."
78+
79+
# 3. Assert that the final index file creation was attempted.
80+
assert mock_create_index.call_count == 1, f"Expected index creation to be called once, but was called {mock_create_index.call_count} times."
81+
82+
# 4. Assert that the index function was called with the correct `zid` due to the known bug.
83+
# This confirms we are testing the actual behavior of the script.
84+
mock_create_index.assert_called_once()
85+
call_args, _ = mock_create_index.call_args
86+
# The call is create_enhanced_multilayer_index(output_dir, conversation_name, layer_files, layer_info)
87+
# We check the second argument, which should be the `conversation_id` (zid) because of the bug.
88+
assert call_args[1] == zid, f"Expected conversation_id '{zid}' to be passed to index creation, but got '{call_args[1]}'"

0 commit comments

Comments
 (0)