|
20 | 20 | kmeans_sklearn, |
21 | 21 | calculate_silhouette_sklearn |
22 | 22 | ) |
23 | | -from polismath.pca_kmeans_rep.repness import conv_repness, participant_stats |
| 23 | +from polismath.pca_kmeans_rep.repness import conv_repness |
24 | 24 | from polismath.pca_kmeans_rep.corr import compute_correlation |
25 | 25 |
|
26 | 26 |
|
@@ -792,114 +792,118 @@ def _compute_participant_info_optimized(self, vote_matrix: pd.DataFrame, group_c |
792 | 792 |
|
793 | 793 | # OPTIMIZATION 3: Precompute group vote matrices and average votes |
794 | 794 |
|
795 | | - # Precompute group vote matrices and their valid comment masks |
796 | | - group_vote_matrices = {} |
| 795 | + # Precompute group average votes and valid comment masks |
797 | 796 | group_avg_votes = {} |
798 | 797 | group_valid_masks = {} |
799 | | - |
| 798 | + |
800 | 799 | for group_id, member_indices in group_member_indices.items(): |
801 | 800 | if len(member_indices) >= 3: # Only calculate for groups with enough members |
802 | 801 | # Extract the group vote matrix |
803 | 802 | group_vote_matrix = matrix_values[member_indices, :] |
804 | | - group_vote_matrices[group_id] = group_vote_matrix |
805 | | - |
| 803 | + |
806 | 804 | # Calculate average votes per comment for this group |
807 | 805 | group_avg_votes[group_id] = np.mean(group_vote_matrix, axis=0) |
808 | 806 |
|
809 | 807 | # Precompute which comments have at least 3 votes from this group |
810 | 808 | group_valid_masks[group_id] = np.sum(group_vote_matrix != 0, axis=0) >= 3 |
811 | 809 |
|
812 | | - # OPTIMIZATION 4: Use vectorized operations for participant stats |
813 | | - |
| 810 | + # VECTORIZED: Compute vote counts for ALL participants at once |
| 811 | + |
814 | 812 | process_start = time.time() |
815 | | - batch_start = time.time() |
816 | | - |
817 | | - for p_idx, participant_id in enumerate(vote_matrix.index): |
818 | | - if p_idx >= matrix_values.shape[0]: |
| 813 | + |
| 814 | + n_agree_all = np.sum(matrix_values > 0, axis=1) # (N,) |
| 815 | + n_disagree_all = np.sum(matrix_values < 0, axis=1) # (N,) |
| 816 | + n_pass_all = np.sum(matrix_values == 0, axis=1) # (N,) |
| 817 | + n_votes_all = n_agree_all + n_disagree_all # (N,) |
| 818 | + |
| 819 | + # Mask: participants with at least one real vote |
| 820 | + has_votes = n_votes_all > 0 # (N,) bool |
| 821 | + |
| 822 | + # VECTORIZED: Compute per-group correlations for ALL participants at once |
| 823 | + # Store as {group_id: corr_array} where corr_array is (N,) |
| 824 | + group_corr_arrays = {} |
| 825 | + |
| 826 | + for group_id, member_indices in group_member_indices.items(): |
| 827 | + if len(member_indices) < 3 or group_id not in group_avg_votes: |
| 828 | + # All correlations default to 0.0 |
| 829 | + group_corr_arrays[group_id] = np.zeros(participant_count) |
819 | 830 | continue |
820 | | - |
821 | | - # Print progress for large participant sets |
822 | | - if participant_count > 100 and p_idx % 100 == 0: |
823 | | - now = time.time() |
824 | | - elapsed = now - process_start |
825 | | - batch_time = now - batch_start |
826 | | - batch_start = now |
827 | | - percent = (p_idx / participant_count) * 100 |
828 | | - logger.info(f"Processed {p_idx}/{participant_count} participants ({percent:.1f}%) - " + |
829 | | - f"Elapsed: {elapsed:.2f}s, Batch: {batch_time:.4f}s") |
830 | | - |
831 | | - # Get participant votes |
832 | | - participant_votes = matrix_values[p_idx, :] |
833 | | - |
834 | | - # Count votes using vectorized operations |
835 | | - n_agree = np.sum(participant_votes > 0) |
836 | | - n_disagree = np.sum(participant_votes < 0) |
837 | | - n_pass = np.sum(participant_votes == 0) |
838 | | - n_votes = n_agree + n_disagree |
839 | | - |
840 | | - # Skip participants with no votes |
841 | | - if n_votes == 0: |
| 831 | + |
| 832 | + valid_mask = group_valid_masks[group_id] |
| 833 | + n_valid = int(np.sum(valid_mask)) |
| 834 | + |
| 835 | + if n_valid < 3: |
| 836 | + group_corr_arrays[group_id] = np.zeros(participant_count) |
842 | 837 | continue |
843 | | - |
844 | | - # Find participant's group using precomputed mapping |
845 | | - participant_group = ptpt_group_map.get(participant_id) |
846 | | - |
847 | | - # OPTIMIZATION 5: Efficient group correlation calculation |
848 | | - |
849 | | - # Calculate agreement with each group - optimized version |
850 | | - group_agreements = {} |
851 | | - |
852 | | - for group_id, member_indices in group_member_indices.items(): |
853 | | - if len(member_indices) < 3: |
854 | | - # Skip groups with too few members |
855 | | - group_agreements[group_id] = 0.0 |
856 | | - continue |
857 | | - |
858 | | - if group_id not in group_avg_votes or group_id not in group_valid_masks: |
859 | | - group_agreements[group_id] = 0.0 |
860 | | - continue |
861 | | - |
862 | | - # Use precomputed data |
863 | | - g_votes = group_avg_votes[group_id] |
864 | | - valid_mask = group_valid_masks[group_id] |
865 | | - |
866 | | - if np.sum(valid_mask) >= 3: # At least 3 valid comments |
867 | | - # Extract only valid comment votes |
868 | | - p_votes = participant_votes[valid_mask] |
869 | | - g_votes_valid = g_votes[valid_mask] |
870 | | - |
871 | | - # Fast correlation calculation |
872 | | - p_std = np.std(p_votes) |
873 | | - g_std = np.std(g_votes_valid) |
874 | | - |
875 | | - if p_std > 0 and g_std > 0: |
876 | | - # Use numpy's built-in correlation (faster and more numerically stable) |
877 | | - correlation = np.corrcoef(p_votes, g_votes_valid)[0, 1] |
878 | | - |
879 | | - if not np.isnan(correlation): |
880 | | - group_agreements[group_id] = correlation |
881 | | - else: |
882 | | - group_agreements[group_id] = 0.0 |
883 | | - else: |
884 | | - group_agreements[group_id] = 0.0 |
885 | | - else: |
886 | | - group_agreements[group_id] = 0.0 |
887 | | - |
888 | | - # Store participant stats |
| 838 | + |
| 839 | + # P: all participants' votes on valid comments — (N, n_valid) |
| 840 | + P = matrix_values[:, valid_mask] |
| 841 | + # g: group average on valid comments — (n_valid,) |
| 842 | + g = group_avg_votes[group_id][valid_mask] |
| 843 | + |
| 844 | + # Note: ddof=0 (biased estimator) is used throughout — that matches |
| 845 | + # numpy.corrcoef's internal convention, so this implementation |
| 846 | + # produces the same correlation values as `np.corrcoef(p, g)[0, 1]` |
| 847 | + # would on each (participant, group) pair. Do NOT switch to ddof=1 |
| 848 | + # ("sample" std) without changing both numerator and denominator |
| 849 | + # consistently; otherwise the correlation values drift. |
| 850 | + p_mean = P.mean(axis=1) # (N,) |
| 851 | + g_mean = g.mean() # scalar |
| 852 | + p_std = P.std(axis=1) # (N,), ddof=0 |
| 853 | + g_std = g.std() # scalar, ddof=0 |
| 854 | + |
| 855 | + if g_std == 0: |
| 856 | + group_corr_arrays[group_id] = np.zeros(participant_count) |
| 857 | + continue |
| 858 | + |
| 859 | + # Pearson correlation, two-pass (centered) formula: |
| 860 | + # corr = mean((P - p_mean)(g - g_mean)) / (p_std * g_std) |
| 861 | + # We use the centered formula rather than the algebraically |
| 862 | + # equivalent one-pass form `(mean(P*g) - p_mean*g_mean) / |
| 863 | + # (p_std*g_std)` because the one-pass form suffers from |
| 864 | + # catastrophic cancellation when the true correlation is small |
| 865 | + # (E[XY] ≈ E[X]*E[Y]), which is the common case in Polis data. |
| 866 | + # This matches numpy.corrcoef's numerical approach (and is what |
| 867 | + # tests/test_participant_info.py::test_matches_numpy_corrcoef |
| 868 | + # validates). |
| 869 | + P_centered = P - p_mean[:, None] # (N, n_valid) |
| 870 | + g_centered = g - g_mean # (n_valid,) |
| 871 | + cov = (P_centered @ g_centered) / n_valid # (N,) |
| 872 | + |
| 873 | + # np.where evaluates both branches; suppress divide-by-zero for p_std==0 |
| 874 | + with np.errstate(invalid='ignore', divide='ignore'): |
| 875 | + corr = np.where( |
| 876 | + p_std > 0, |
| 877 | + cov / (p_std * g_std), |
| 878 | + 0.0, |
| 879 | + ) |
| 880 | + corr = np.nan_to_num(corr, nan=0.0) |
| 881 | + group_corr_arrays[group_id] = corr |
| 882 | + |
| 883 | + # Assemble result dicts (zero computation — just indexing) |
| 884 | + group_ids = list(group_member_indices.keys()) |
| 885 | + |
| 886 | + for p_idx, participant_id in enumerate(vote_matrix.index): |
| 887 | + if not has_votes[p_idx]: |
| 888 | + continue |
| 889 | + |
889 | 890 | result['stats'][participant_id] = { |
890 | | - 'n_agree': int(n_agree), |
891 | | - 'n_disagree': int(n_disagree), |
892 | | - 'n_pass': int(n_pass), |
893 | | - 'n_votes': int(n_votes), |
894 | | - 'group': participant_group, |
895 | | - 'group_correlations': group_agreements |
| 891 | + 'n_agree': int(n_agree_all[p_idx]), |
| 892 | + 'n_disagree': int(n_disagree_all[p_idx]), |
| 893 | + 'n_pass': int(n_pass_all[p_idx]), |
| 894 | + 'n_votes': int(n_votes_all[p_idx]), |
| 895 | + 'group': ptpt_group_map.get(participant_id), |
| 896 | + 'group_correlations': { |
| 897 | + gid: float(group_corr_arrays[gid][p_idx]) |
| 898 | + for gid in group_ids |
| 899 | + } |
896 | 900 | } |
897 | | - |
| 901 | + |
898 | 902 | total_time = time.time() - start_time |
899 | 903 | process_time = time.time() - process_start |
900 | 904 | logger.info(f"Participant stats completed in {total_time:.2f}s (preparation: {prep_time:.2f}s, processing: {process_time:.2f}s)") |
901 | 905 | logger.info(f"Processed {len(result['stats'])} participants with {len(group_clusters)} groups") |
902 | | - |
| 906 | + |
903 | 907 | return result |
904 | 908 |
|
905 | 909 | def _compute_participant_info(self) -> None: |
|
0 commit comments