diff --git a/testing-v2/IMPROVEMENTS-LOG.md b/testing-v2/IMPROVEMENTS-LOG.md index 2f5d0cb0..69ab1a78 100644 --- a/testing-v2/IMPROVEMENTS-LOG.md +++ b/testing-v2/IMPROVEMENTS-LOG.md @@ -1118,3 +1118,32 @@ After completing the iteration successfully, user provided GitHub samples showin - Throughput & scaling (5 rules) - Global distribution (6 rules) - Monitoring & diagnostics (5 rules) + +#### 2026-03-31: iteration-001-python - Gaming Leaderboard (Python) [skills loaded] + +- **Scenario**: gaming-leaderboard +- **Iteration**: iteration-001-python +- **Skills loaded**: Yes +- **Result**: FAILED -- 21/94 tests passed (22.3%) +- **Score**: 1/10 + +**Results by Category**: +- api_contract: 10 passed, 35 failed, 0 skipped +- build_startup: 2 passed, 0 failed, 0 skipped +- cosmos_infrastructure: 3 passed, 9 failed, 1 skipped +- data_integrity: 4 passed, 1 failed, 0 skipped +- robustness: 4 passed, 27 failed, 0 skipped + +**Issues Encountered**: +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayer::test_get_existing_player** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayer::test_get_player_has_required_fields** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayer::test_get_player_stats_updated_after_scores** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestSubmitScore::test_submit_score_returns_201** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestSubmitScore::test_submit_score_response_has_required_fields** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestSubmitScore::test_submit_score_returns_correct_data** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_returns_200** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_returns_array** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_entries_have_required_fields** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err +1. **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_sorted_descending** -- failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Err + +**Test Results**: 21 passed, 73 failed out of 94 diff --git a/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/ITERATION.md b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/ITERATION.md new file mode 100644 index 00000000..fe1d6fd6 --- /dev/null +++ b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/ITERATION.md @@ -0,0 +1,424 @@ +# iteration-001-python - Python Gaming Leaderboard + +## Metadata +- **Date**: 2026-03-31 +- **Language/SDK**: Python +- **Agent**: GitHub Copilot (automated iteration) +- **Tester**: Automated CI +- **Run Type**: Normal run (skills loaded) + +## Skills Verification + +**Were skills loaded before building?** Yes (via issue prompt referencing AGENTS.md) + +## Cosmos DB Patterns Detected + +| Pattern | Status | Related Rule | +|---------|--------|--------------| +| Singleton CosmosClient | Detected | `sdk-singleton-client` | +| Direct connection mode | Not detected | `sdk-connection-mode` | +| Gateway connection mode | Not detected | `sdk-connection-mode` | +| Partition key configured | Detected | `partition-high-cardinality` | +| Bulk operations | Not detected | `sdk-bulk-operations` | +| ETag optimistic concurrency | Detected | `sdk-etag-concurrency` | +| Point reads (by ID + partition key) | Detected | `query-avoid-scans` | +| Cross-partition queries | Detected | `query-avoid-cross-partition` | +| Custom indexing policy | Detected | `index-exclude-unused` | +| Throughput configuration | Not detected | `throughput-provision-rus` | +| Change feed usage | Not detected | `pattern-change-feed` | +| Diagnostics/logging | Not detected | `sdk-diagnostics` | + +## Test Results + +**Pass rate: 22.3%** (21/94 tests passed (22.3%)) + +| Status | Count | +|--------|-------| +| Passed | 21 | +| Failed | 2 | +| Errors | 70 | +| Skipped | 1 | + +### Failures + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayer::test_get_existing_player** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayer::test_get_player_has_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayer::test_get_player_stats_updated_after_scores** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestSubmitScore::test_submit_score_returns_201** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestSubmitScore::test_submit_score_response_has_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestSubmitScore::test_submit_score_returns_correct_data** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_returns_200** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_returns_array** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_entries_have_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_sorted_descending** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_ranks_sequential** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_top_player_is_highest_scorer** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGlobalLeaderboard::test_global_leaderboard_respects_top_parameter** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestRegionalLeaderboard::test_regional_leaderboard_returns_200** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestRegionalLeaderboard::test_regional_leaderboard_only_contains_region_players** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestRegionalLeaderboard::test_regional_leaderboard_sorted_descending** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestRegionalLeaderboard::test_regional_leaderboard_entries_have_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestPlayerRank::test_player_rank_returns_200** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestPlayerRank::test_player_rank_has_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestPlayerRank::test_player_rank_correct_for_top_player** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestPlayerRank::test_player_rank_neighbors_is_array** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestPlayerRank::test_player_rank_neighbors_have_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayerScores::test_score_history_returns_200** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayerScores::test_score_history_returns_array** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayerScores::test_score_history_entries_have_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayerScores::test_score_history_contains_all_player_scores** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayerScores::test_score_history_ordered_by_most_recent_first** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayerScores::test_score_history_respects_limit** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestGetPlayerScores::test_score_history_only_shows_own_scores** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestUpdatePlayer::test_update_player_returns_200** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestUpdatePlayer::test_update_player_response_has_required_fields** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestUpdatePlayer::test_update_player_reflects_new_display_name** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestUpdatePlayer::test_update_player_preserves_stats** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestDeletePlayer::test_delete_player_returns_204** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_api_contract.TestDeletePlayer::test_deleted_player_returns_404_on_get** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestContainerDesign::test_has_multiple_containers_or_synthetic_keys** + > Failed: Only one container with a simple partition key. Leaderboard systems need different access patterns: player lookup (by playerId) and ranking queries (by leaderboard scope). Use multiple contain + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestContainerDesign::test_leaderboard_container_uses_synthetic_key** + > Failed: Leaderboard container 'leaderboard' uses /playerId as partition key. This makes top-N ranking queries cross-partition (expensive). Use a synthetic key like /leaderboardKey = 'global_weekly' so + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestPlayerScoreSerialization::test_scores_stored_as_numbers** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestPlayerScoreSerialization::test_etag_present_on_player_documents** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestDocumentStructure::test_documents_have_type_discriminator** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestDocumentStructure::test_documents_have_schema_version** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestCrossBoundaryConsistency::test_player_stats_stored_correctly** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestCrossBoundaryConsistency::test_leaderboard_entries_denormalized** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_cosmos_infrastructure.TestCrossBoundaryConsistency::test_synthetic_partition_key_value_format** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_data_integrity.TestDataPersistence::test_player_document_exists_in_cosmos** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestInvalidInput::test_submit_score_missing_player_id_returns_4xx** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestInvalidInput::test_submit_score_missing_score_returns_4xx** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestInvalidInput::test_submit_score_negative_value_returns_4xx** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestInvalidInput::test_submit_score_for_nonexistent_player_returns_4xx** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestComputedFieldAccuracy::test_average_score_mathematically_correct** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestComputedFieldAccuracy::test_total_games_count_correct** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestComputedFieldAccuracy::test_best_score_is_maximum** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestComputedFieldAccuracy::test_player_with_single_score_stats** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestComputedFieldAccuracy::test_new_score_updates_stats** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestDataTypeCorrectness::test_player_stats_types** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestDataTypeCorrectness::test_leaderboard_entry_types** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestDataTypeCorrectness::test_score_submission_returns_correct_types** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestWriteReadConsistency::test_score_reflected_in_leaderboard** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestWriteReadConsistency::test_regional_filter_matches_stored_region** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestWriteReadConsistency::test_player_rank_score_matches_leaderboard** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestEdgeCases::test_empty_region_leaderboard** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestEdgeCases::test_leaderboard_no_duplicate_players** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestEdgeCases::test_top_parameter_zero_returns_empty** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestEdgeCases::test_top_parameter_one** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestEdgeCases::test_zero_score_submission** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestRapidOperations::test_rapid_score_submissions_all_counted** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestRapidOperations::test_concurrent_score_submissions_all_counted** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestLeaderboardTiebreaking::test_tied_scores_sorted_by_display_name_ascending** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestLeaderboardTiebreaking::test_tied_scores_have_sequential_ranks** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestUpdateDeleteConsistency::test_updated_region_reflected_in_regional_leaderboard** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestUpdateDeleteConsistency::test_deleted_player_removed_from_leaderboard** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +- **testing-v2.scenarios.gaming-leaderboard.tests.test_robustness.TestUpdateDeleteConsistency::test_deleted_player_scores_not_in_history** + > failed on setup with "AssertionError: Failed to submit score for player-001: 500 Internal Server Error +assert 500 == 201 + + where 500 = .status_code" + +## Source Files + +Source code archived in `source-code.zip` (6 files). + +## Build & Startup Signals + +- **Build**: PASS +- **Startup**: PASS + +## Results by Category + +| Category | Passed | Failed | Skipped | +|----------|--------|--------|---------| +| api_contract | 10 | 35 | 0 | +| build_startup | 2 | 0 | 0 | +| cosmos_infrastructure | 3 | 9 | 1 | +| data_integrity | 4 | 1 | 0 | +| robustness | 4 | 27 | 0 | + +## Score Summary + +| Category | Score | Notes | +|----------|-------|-------| +| API Conformance | 1/10 | 22.3% pass rate; 9 infrastructure failures | +| **Overall** | **1/10** | **21/94 tests passed (22.3%)** | diff --git a/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/_start-app.cmd b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/_start-app.cmd new file mode 100644 index 00000000..289ebc6f --- /dev/null +++ b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/_start-app.cmd @@ -0,0 +1,2 @@ +@echo off +uvicorn main:app --host 0.0.0.0 --port 8000 > "D:\a\cosmosdb-agent-kit\cosmosdb-agent-kit\testing-v2\scenarios\gaming-leaderboard\iterations\iteration-001-python\app-output.log" 2> "D:\a\cosmosdb-agent-kit\cosmosdb-agent-kit\testing-v2\scenarios\gaming-leaderboard\iterations\iteration-001-python\app-error.log" \ No newline at end of file diff --git a/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/build-signal.json b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/build-signal.json new file mode 100644 index 00000000..02e22322 --- /dev/null +++ b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/build-signal.json @@ -0,0 +1,7 @@ +{ + "succeeded": true, + "stderr_tail": "", + "build_command": "pip install -r requirements.txt", + "stdout_tail": "Downloading multidict-6.7.1-cp312-cp312-win_amd64.whl (46 kB)\nDownloading yarl-1.23.0-cp312-cp312-win_amd64.whl (87 kB)\nDownloading aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB)\nDownloading aiosignal-1.4.0-py3-none-any.whl (7.5 kB)\nDownloading annotated_doc-0.0.4-py3-none-any.whl (5.3 kB)\nDownloading attrs-26.1.0-py3-none-any.whl (67 kB)\nDownloading azure_core-1.39.0-py3-none-any.whl (218 kB)\nDownloading frozenlist-1.8.0-cp312-cp312-win_amd64.whl (44 kB)\nDownloading h11-0.16.0-py3-none-any.whl (37 kB)\nDownloading idna-3.11-py3-none-any.whl (71 kB)\nDownloading propcache-0.4.1-cp312-cp312-win_amd64.whl (41 kB)\nDownloading pydantic-2.12.5-py3-none-any.whl (463 kB)\nDownloading pydantic_core-2.41.5-cp312-cp312-win_amd64.whl (2.0 MB)\n ---------------------------------------- 2.0/2.0 MB 56.7 MB/s 0:00:00\nDownloading annotated_types-0.7.0-py3-none-any.whl (13 kB)\nDownloading requests-2.33.1-py3-none-any.whl (64 kB)\nDownloading charset_normalizer-3.4.6-cp312-cp312-win_amd64.whl (154 kB)\nDownloading urllib3-2.6.3-py3-none-any.whl (131 kB)\nDownloading certifi-2026.2.25-py3-none-any.whl (153 kB)\nDownloading starlette-1.0.0-py3-none-any.whl (72 kB)\nDownloading anyio-4.13.0-py3-none-any.whl (114 kB)\nDownloading typing_extensions-4.15.0-py3-none-any.whl (44 kB)\nDownloading typing_inspection-0.4.2-py3-none-any.whl (14 kB)\nDownloading httptools-0.7.1-cp312-cp312-win_amd64.whl (86 kB)\nDownloading python_dotenv-1.2.2-py3-none-any.whl (22 kB)\nDownloading watchfiles-1.1.1-cp312-cp312-win_amd64.whl (288 kB)\nDownloading websockets-16.0-cp312-cp312-win_amd64.whl (178 kB)\nInstalling collected packages: websockets, urllib3, typing-extensions, python-dotenv, propcache, multidict, idna, httptools, h11, frozenlist, charset_normalizer, certifi, attrs, annotated-types, annotated-doc, aiohappyeyeballs, yarl, uvicorn, typing-inspection, requests, pydantic-core, anyio, aiosignal, watchfiles, starlette, pydantic, azure-core, aiohttp, fastapi, azure-cosmos\n\nSuccessfully installed aiohappyeyeballs-2.6.1 aiohttp-3.13.4 aiosignal-1.4.0 annotated-doc-0.0.4 annotated-types-0.7.0 anyio-4.13.0 attrs-26.1.0 azure-core-1.39.0 azure-cosmos-4.15.0 certifi-2026.2.25 charset_normalizer-3.4.6 fastapi-0.135.2 frozenlist-1.8.0 h11-0.16.0 httptools-0.7.1 idna-3.11 multidict-6.7.1 propcache-0.4.1 pydantic-2.12.5 pydantic-core-2.41.5 python-dotenv-1.2.2 requests-2.33.1 starlette-1.0.0 typing-extensions-4.15.0 typing-inspection-0.4.2 urllib3-2.6.3 uvicorn-0.42.0 watchfiles-1.1.1 websockets-16.0 yarl-1.23.0", + "exit_code": 0 +} diff --git a/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/iteration-config.yaml b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/iteration-config.yaml new file mode 100644 index 00000000..81368c04 --- /dev/null +++ b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/iteration-config.yaml @@ -0,0 +1,7 @@ +language: python +database: gaming-leaderboard-db +port: 8000 +health: /health +build: pip install -r requirements.txt +run: uvicorn main:app --host 0.0.0.0 --port 8000 +skills_loaded: true diff --git a/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/main.py b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/main.py new file mode 100644 index 00000000..82006264 --- /dev/null +++ b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/main.py @@ -0,0 +1,435 @@ +""" +Gaming Leaderboard API — FastAPI + Azure Cosmos DB (NoSQL API) + +Best practices applied: +- Singleton CosmosClient (Rule 4.18) +- Async SDK with aiohttp (Rule 4.15) +- Type discriminators for polymorphic data in single container (Rule 1.11) +- Partition key aligned with query patterns (Rule 2.7) +- Point reads when ID and partition key are known (Rule 3.7) +- Parameterized queries (Rule 3.6) +- Literal integers for TOP (Rule 3.8) +- Composite indexes for ORDER BY (Rule 5.1, 5.2) +- Exclude unused index paths (Rule 5.3) +- Project only needed fields (Rule 3.9) +- COUNT-based ranking instead of full partition scans (Rule 9.2) +- camelCase field naming as required by API contract +""" + +import os +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone + +from azure.cosmos.aio import CosmosClient +from azure.cosmos import PartitionKey +from azure.cosmos.exceptions import CosmosResourceNotFoundError, CosmosHttpResponseError +from fastapi import FastAPI, HTTPException, Query +from fastapi.responses import JSONResponse + +# --------------------------------------------------------------------------- +# Configuration from environment variables (Rule 4.6, 4.12) +# --------------------------------------------------------------------------- +COSMOS_ENDPOINT = os.environ.get( + "COSMOS_ENDPOINT", "https://localhost:8081" +) +COSMOS_KEY = os.environ.get( + "COSMOS_KEY", + "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", +) +DATABASE_NAME = os.environ.get("DATABASE_NAME", "gaming-leaderboard-db") +CONTAINER_NAME = "leaderboard" + +# --------------------------------------------------------------------------- +# Cosmos DB singleton client & container reference (Rule 4.18) +# --------------------------------------------------------------------------- +cosmos_client: CosmosClient | None = None +container = None + + +async def _get_or_create_container(): + """Ensure database and container exist with optimal configuration.""" + global cosmos_client, container + + # Determine if we're talking to the emulator (Rule 4.6) + is_emulator = "localhost" in COSMOS_ENDPOINT or "127.0.0.1" in COSMOS_ENDPOINT + + cosmos_client = CosmosClient( + url=COSMOS_ENDPOINT, + credential=COSMOS_KEY, + connection_verify=not is_emulator, # Disable SSL verification for emulator + ) + + database = await cosmos_client.create_database_if_not_exists(id=DATABASE_NAME) + + # Indexing policy (Rules 5.1, 5.2, 5.3) + indexing_policy = { + "indexingMode": "consistent", + "automatic": True, + "includedPaths": [{"path": "/*"}], + "excludedPaths": [{"path": '/"_etag"/?'}], + "compositeIndexes": [ + # For ORDER BY bestScore DESC, displayName ASC (leaderboard queries) + [ + {"path": "/bestScore", "order": "descending"}, + {"path": "/displayName", "order": "ascending"}, + ], + # Inverse pair (Rule 5.1 — always include inverse) + [ + {"path": "/bestScore", "order": "ascending"}, + {"path": "/displayName", "order": "descending"}, + ], + # Note: single-property ORDER BY (e.g. timestamp DESC) is served + # by the default range index (/*) — no composite index needed. + ], + } + + # Single container with type discriminator (Rule 1.11) + # Partition key: /playerId — aligns with most query patterns (Rule 2.7) + # High cardinality, immutable (Rules 2.4, 2.5) + container = await database.create_container_if_not_exists( + id=CONTAINER_NAME, + partition_key=PartitionKey(path="/playerId"), + indexing_policy=indexing_policy, + ) + + +# --------------------------------------------------------------------------- +# Lifespan: singleton init & teardown +# --------------------------------------------------------------------------- +@asynccontextmanager +async def lifespan(app: FastAPI): + await _get_or_create_container() + yield + if cosmos_client: + await cosmos_client.close() + + +app = FastAPI(title="Gaming Leaderboard API", lifespan=lifespan) + + +# --------------------------------------------------------------------------- +# Health endpoint +# --------------------------------------------------------------------------- +@app.get("/health") +async def health(): + return {"status": "ok"} + + +# --------------------------------------------------------------------------- +# Player Management +# --------------------------------------------------------------------------- +@app.post("/api/players", status_code=201) +async def create_player(body: dict): + player_id = body.get("playerId") + display_name = body.get("displayName") + region = body.get("region") + + if not player_id or not display_name or not region: + raise HTTPException(status_code=400, detail="playerId, displayName, and region are required") + + player_doc = { + "id": player_id, + "playerId": player_id, + "type": "player", # Type discriminator (Rule 1.11) + "displayName": display_name, + "region": region, + "totalGames": 0, + "bestScore": 0, + "averageScore": 0.0, + "totalScore": 0, # Internal field for average calculation + } + + try: + await container.create_item(body=player_doc) + except CosmosHttpResponseError as e: + if e.status_code == 409: + raise HTTPException(status_code=409, detail="Player already exists") + raise + + return _player_response(player_doc) + + +@app.get("/api/players/{player_id}") +async def get_player(player_id: str): + player = await _read_player(player_id) + return _player_response(player) + + +@app.patch("/api/players/{player_id}") +async def update_player(player_id: str, body: dict): + player = await _read_player(player_id) + + if "displayName" in body: + player["displayName"] = body["displayName"] + if "region" in body: + player["region"] = body["region"] + + await container.replace_item( + item=player["id"], + body=player, + partition_key=player_id, + ) + return _player_response(player) + + +@app.delete("/api/players/{player_id}", status_code=204) +async def delete_player(player_id: str): + # Verify player exists first + await _read_player(player_id) + + # Delete all score documents for this player + query = "SELECT c.id FROM c WHERE c.playerId = @pid AND c.type = 'score'" + params = [{"name": "@pid", "value": player_id}] + score_ids = [] + async for item in container.query_items( + query=query, parameters=params, partition_key=player_id + ): + score_ids.append(item["id"]) + + for sid in score_ids: + await container.delete_item(item=sid, partition_key=player_id) + + # Delete the player document + await container.delete_item(item=player_id, partition_key=player_id) + + return JSONResponse(status_code=204, content=None) + + +# --------------------------------------------------------------------------- +# Score Submission +# --------------------------------------------------------------------------- +@app.post("/api/scores", status_code=201) +async def submit_score(body: dict): + player_id = body.get("playerId") + score_val = body.get("score") + game_mode = body.get("gameMode") + + if not player_id or score_val is None: + raise HTTPException(status_code=400, detail="playerId and score are required") + + score_val = int(score_val) + + # Verify player exists + player = await _read_player(player_id) + + score_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc).isoformat() + + score_doc = { + "id": score_id, + "scoreId": score_id, + "playerId": player_id, + "type": "score", # Type discriminator (Rule 1.11) + "score": score_val, + "timestamp": now, + } + if game_mode: + score_doc["gameMode"] = game_mode + + await container.create_item(body=score_doc) + + # Update player stats (denormalized for read-heavy workloads, Rule 1.2) + player["totalGames"] = player.get("totalGames", 0) + 1 + player["totalScore"] = player.get("totalScore", 0) + score_val + player["bestScore"] = max(player.get("bestScore", 0), score_val) + player["averageScore"] = player["totalScore"] / player["totalGames"] + + await container.replace_item( + item=player["id"], + body=player, + partition_key=player_id, + ) + + return { + "scoreId": score_id, + "playerId": player_id, + "score": score_val, + } + + +# --------------------------------------------------------------------------- +# Leaderboards +# --------------------------------------------------------------------------- +@app.get("/api/leaderboards/global") +async def global_leaderboard(top: int = Query(default=100, ge=1, le=100)): + top = int(top) # Ensure safe integer (Rule 3.8) + # Cross-partition query — project only needed fields (Rule 3.9) + # TOP must be a literal integer, not a parameter (Rule 3.8) + # ORDER BY bestScore DESC, displayName ASC for tiebreaking + query = ( + f"SELECT TOP {top} c.playerId, c.displayName, c.bestScore " + f"FROM c WHERE c.type = 'player' AND c.bestScore > 0 " + f"ORDER BY c.bestScore DESC, c.displayName ASC" + ) + + entries = [] + async for item in container.query_items( + query=query, enable_cross_partition_query=True + ): + entries.append(item) + + return _ranked_list(entries) + + +@app.get("/api/leaderboards/regional/{region}") +async def regional_leaderboard( + region: str, top: int = Query(default=100, ge=1, le=100) +): + top = int(top) + # Parameterized region filter (Rule 3.6), literal TOP (Rule 3.8) + query = ( + f"SELECT TOP {top} c.playerId, c.displayName, c.bestScore " + f"FROM c WHERE c.type = 'player' AND c.region = @region AND c.bestScore > 0 " + f"ORDER BY c.bestScore DESC, c.displayName ASC" + ) + params = [{"name": "@region", "value": region}] + + entries = [] + async for item in container.query_items( + query=query, parameters=params, enable_cross_partition_query=True + ): + entries.append(item) + + return _ranked_list(entries) + + +# --------------------------------------------------------------------------- +# Player Ranking (COUNT-based approach — Rule 9.2) +# --------------------------------------------------------------------------- +@app.get("/api/players/{player_id}/rank") +async def player_rank(player_id: str): + player = await _read_player(player_id) + + best_score = player.get("bestScore", 0) + if best_score == 0 and player.get("totalGames", 0) == 0: + raise HTTPException(status_code=404, detail="Player has no scores") + + # COUNT-based rank: count players with higher bestScore (Rule 9.2) + # For tiebreaking (same score, displayName ASC), count those with same score but earlier name + display_name = player.get("displayName", "") + count_query = ( + "SELECT VALUE COUNT(1) FROM c WHERE c.type = 'player' AND c.bestScore > 0 AND " + "(" + " c.bestScore > @score OR " + " (c.bestScore = @score AND c.displayName < @name)" + ")" + ) + params = [ + {"name": "@score", "value": best_score}, + {"name": "@name", "value": display_name}, + ] + + rank = 1 + async for count_val in container.query_items( + query=count_query, parameters=params, enable_cross_partition_query=True + ): + rank = count_val + 1 + + # Get neighbors: players ranked near this player (±10 positions) + # First get all players sorted, find the window around our player + # We need rank-11 to rank+10 in the sorted list — fetch a window + skip_start = max(0, rank - 11) + window_size = 21 # ±10 + self + + # Use OFFSET LIMIT to get the window of neighbors + neighbor_query = ( + f"SELECT c.playerId, c.displayName, c.bestScore " + f"FROM c WHERE c.type = 'player' AND c.bestScore > 0 " + f"ORDER BY c.bestScore DESC, c.displayName ASC " + f"OFFSET {int(skip_start)} LIMIT {int(window_size)}" + ) + + neighbors_raw = [] + async for item in container.query_items( + query=neighbor_query, enable_cross_partition_query=True + ): + neighbors_raw.append(item) + + # Build neighbors list with ranks, excluding the player themselves + neighbors = [] + for i, item in enumerate(neighbors_raw): + item_rank = skip_start + i + 1 + if item["playerId"] != player_id: + neighbors.append( + { + "rank": item_rank, + "playerId": item["playerId"], + "displayName": item["displayName"], + "score": item["bestScore"], + } + ) + + return { + "playerId": player_id, + "rank": rank, + "score": best_score, + "neighbors": neighbors, + } + + +# --------------------------------------------------------------------------- +# Score History +# --------------------------------------------------------------------------- +@app.get("/api/players/{player_id}/scores") +async def get_player_scores( + player_id: str, limit: int = Query(default=10, ge=1, le=100) +): + # Verify player exists (returns 404 if not) + await _read_player(player_id) + + limit = int(limit) + # Single-partition query (partition key = playerId, Rule 3.1) + # Literal TOP (Rule 3.8), ordered by timestamp DESC (most recent first) + query = ( + f"SELECT TOP {limit} c.scoreId, c.playerId, c.score, c.gameMode, c.timestamp " + f"FROM c WHERE c.type = 'score' " + f"ORDER BY c.timestamp DESC" + ) + + scores = [] + async for item in container.query_items( + query=query, partition_key=player_id + ): + scores.append(item) + + return scores + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +async def _read_player(player_id: str) -> dict: + """Point-read a player document (Rule 3.7).""" + try: + player = await container.read_item(item=player_id, partition_key=player_id) + if player.get("type") != "player": + raise HTTPException(status_code=404, detail="Player not found") + return player + except CosmosResourceNotFoundError: + raise HTTPException(status_code=404, detail="Player not found") + + +def _player_response(player: dict) -> dict: + """Project only the fields required by the API contract.""" + return { + "playerId": player["playerId"], + "displayName": player["displayName"], + "region": player["region"], + "totalGames": player.get("totalGames", 0), + "bestScore": player.get("bestScore", 0), + "averageScore": player.get("averageScore", 0.0), + } + + +def _ranked_list(entries: list[dict]) -> list[dict]: + """Convert raw query results to ranked leaderboard entries (1-based rank).""" + return [ + { + "rank": i + 1, + "playerId": e["playerId"], + "displayName": e["displayName"], + "score": e["bestScore"], + } + for i, e in enumerate(entries) + ] diff --git a/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/requirements.txt b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/requirements.txt new file mode 100644 index 00000000..ea2b571c --- /dev/null +++ b/testing-v2/scenarios/gaming-leaderboard/iterations/iteration-001-python/requirements.txt @@ -0,0 +1,4 @@ +fastapi>=0.110.0 +uvicorn[standard]>=0.27.0 +azure-cosmos>=4.6.0 +aiohttp>=3.9.0