Skip to content

Commit 9d17934

Browse files
ArtOfCode-ArtOfCode-
authored
Merge pull request #2049 from codidact/art/2000/pii-correlation
Add PII correlation tool
2 parents e30df32 + 2d76aa6 commit 9d17934

20 files changed

Lines changed: 430 additions & 0 deletions

‎.rubocop.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ AllCops:
1313
- 'bin/**/*'
1414
- 'lib/namespaced_env_cache.rb'
1515
- 'vendor/bundle/**/*'
16+
- 'docker/**/*'
1617
NewCops: enable
1718
SuggestExtensions: false
1819

‎Gemfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ source 'https://rubygems.org'
22
ruby '>= 3.3', '< 3.5'
33

44
# Essential gems: servers, adapters, Rails + Rails requirements
5+
gem 'bcrypt', '~> 3.1'
56
gem 'coffee-rails', '~> 5.0.0'
67
gem 'connection_pool', '< 3.0' # mperham/connection_pool#210
78
gem 'counter_culture', '~> 3.2'
@@ -38,6 +39,7 @@ gem 'groupdate', '~> 6.1'
3839

3940
# View stuff.
4041
gem 'diffy', '~> 3.4'
42+
gem 'ipaddress', '~> 0.8'
4143
gem 'jbuilder', '~> 2.11'
4244
gem 'rqrcode', '~> 2.1'
4345
gem 'will_paginate', '~> 3.3'

‎Gemfile.lock‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ GEM
187187
mini_magick (>= 4.9.5, < 6)
188188
ruby-vips (>= 2.0.17, < 3)
189189
io-console (0.8.2)
190+
ipaddress (0.8.3)
190191
irb (1.17.0)
191192
pp (>= 0.6.0)
192193
prism (>= 1.3.0)
@@ -512,6 +513,7 @@ DEPENDENCIES
512513
aws-sdk-s3 (~> 1.208)
513514
aws-sdk-sns (~> 1.72)
514515
aws-ses-v4
516+
bcrypt (~> 3.1)
515517
byebug (~> 11.1)
516518
capybara (~> 3.38)
517519
chartkick (~> 4.2)
@@ -527,6 +529,7 @@ DEPENDENCIES
527529
flamegraph (~> 0.9)
528530
groupdate (~> 6.1)
529531
image_processing (~> 1.12)
532+
ipaddress (~> 0.8)
530533
jbuilder (~> 2.11)
531534
jquery-rails (~> 4.5.0)
532535
letter_opener_web (~> 2.0)

‎app/assets/javascripts/moderator.js‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,14 @@ $(() => {
4747
checkbox.checked = action === 'all';
4848
});
4949
});
50+
51+
QPixel.DOM.addSelectorListener('submit', '#pii-correlation-form', async (ev) => {
52+
ev.preventDefault();
53+
54+
const targetId = /** @type {HTMLInputElement}*/(document.querySelector('input[name="target_id"]')).value;
55+
const resp = await QPixel.fetch(`${location.pathname}?format=template&target_id=${targetId}`);
56+
const html = await resp.text();
57+
58+
document.querySelector('.js-correlation-container').innerHTML = html;
59+
});
5060
});

‎app/controllers/moderator_controller.rb‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,16 @@ def handle_spammy_users
8989
redirect_to mod_spammers_path
9090
end
9191

92+
def pii_correlation
93+
@user = User.find(params[:id])
94+
respond_to do |format|
95+
format.html
96+
format.template do
97+
@target = User.find_by(id: params[:target_id])
98+
end
99+
end
100+
end
101+
92102
private
93103

94104
def set_post

‎app/helpers/moderator_helper.rb‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,29 @@ def text_bg(cls, content = nil, **opts, &block)
1616
tag.span content, class: ["has-background-color-#{cls}", opts[:class]].join(' ')
1717
end
1818
end
19+
20+
##
21+
# Split an IP address into an array of hashed octets (well, hexadecets for IPv6).
22+
# @param ip [String] The IP address to process.
23+
# @param salting_user [User] A user from which to source a salt for hashing. For hashes to be directly comparable, you
24+
# must use the same user for each IP address you wish to compare, even if sourced from a different user.
25+
# @return [[String, [String?]]] The IP address family, and an array of hashed octets.
26+
def split_hash_ip(ip, salting_user)
27+
begin
28+
addr = IPAddress.parse(ip)
29+
rescue ArgumentError
30+
return ['', []]
31+
end
32+
splat = if addr.ipv6?
33+
addr.hexs
34+
else
35+
addr.octets
36+
end
37+
salt = BCrypt::Password.new(salting_user.encrypted_password).salt
38+
splat = splat.map { |p| Digest::SHA2.hexdigest(salt + p.to_s) }
39+
[
40+
addr.ipv6? ? 'IPv6' : 'IPv4',
41+
splat
42+
]
43+
end
1944
end

‎app/jobs/update_user_stats_job.rb‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
class UpdateUserStatsJob < ApplicationJob
2+
queue_as :default
3+
4+
def perform(*)
5+
domains = User.all.select(:email)
6+
.group_by { |u| u.email&.split('@')&.[](1) }
7+
.transform_values(&:size)
8+
.reject { |d, _u| d.include? 'localhost' }
9+
Rails.cache.hmset('user_email_domains', domains)
10+
end
11+
end

‎app/views/moderator/pii_correlation.html.erb‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<h2>PII correlation for <%= user_link @user %></h2>
2+
<p>
3+
This tool displays correlations between personally-identifying information across user accounts. This includes email
4+
address and IP addresses. Please select a user against whom to compare.
5+
</p>
6+
7+
<noscript>JavaScript is required to use this tool.</noscript>
8+
<form id="pii-correlation-form" action="#" class="form-horizontal">
9+
<div class="form-group-horizontal">
10+
<div class="form-group">
11+
<label for="target-id" class="form-element">Target user ID</label>
12+
<input type="number" id="target-id" name="target_id" class="form-element" required />
13+
</div>
14+
<div class="actions">
15+
<button type="submit" class="button is-primary is-filled">Compare</button>
16+
</div>
17+
</div>
18+
</form>
19+
20+
<p>
21+
Information is hashed to protect users' privacy.
22+
Text <span class="has-background-color-red">highlighted in red</span> indicates matching data.
23+
</p>
24+
25+
<div class="js-correlation-container"></div>

‎app/views/moderator/pii_correlation.template.erb‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
<h3>Comparing with <%= user_link @target %></h3>
2+
3+
<h4>Email address</h4>
4+
<table class="table is-full-width is-striped is-with-hover">
5+
<thead>
6+
<tr>
7+
<th></th>
8+
<th>Handle</th>
9+
<th>Domain</th>
10+
</tr>
11+
</thead>
12+
<tbody>
13+
<tr>
14+
<td><strong><%= @user.rtl_safe_username %></strong></td>
15+
<% user_handle = Digest::SHA2.hexdigest(@user.email.split('@')[0]) %>
16+
<% target_handle = Digest::SHA2.hexdigest(@target.email.split('@')[0]) %>
17+
<% user_domain = Digest::SHA2.hexdigest(@user.email.split('@')[1]) %>
18+
<% target_domain = Digest::SHA2.hexdigest(@target.email.split('@')[1]) %>
19+
<td>
20+
<code class="<%= 'has-background-color-red' if user_handle == target_handle %>">
21+
<%= user_handle[0..7] %>
22+
</code>
23+
</td>
24+
<td>
25+
<code class="<%= 'has-background-color-red' if user_domain == target_domain %>">
26+
<%= user_domain[0..7] %>
27+
</code><br/>
28+
<% domain_users = Rails.cache.hget 'user_email_domains', @user.email.split('@')[1] %>
29+
<% if domain_users.nil? %>
30+
(unknown number of users)
31+
<% else %>
32+
Used by <%= pluralize(domain_users, 'user') %>
33+
<% end %>
34+
</td>
35+
</tr>
36+
<tr>
37+
<td><strong><%= @target.rtl_safe_username %></strong></td>
38+
<td>
39+
<code class="<%= 'has-background-color-red' if user_handle == target_handle %>">
40+
<%= target_handle[0..7] %>
41+
</code>
42+
</td>
43+
<td>
44+
<code class="<%= 'has-background-color-red' if user_domain == target_domain %>">
45+
<%= target_domain[0..7] %>
46+
</code><br/>
47+
<% domain_users = Rails.cache.hget 'user_email_domains', @target.email.split('@')[1] %>
48+
<% if domain_users.nil? %>
49+
(unknown number of users)
50+
<% else %>
51+
Used by <%= pluralize(domain_users, 'user') %>
52+
<% end %>
53+
</td>
54+
</tr>
55+
</tbody>
56+
</table><br/>
57+
58+
<h4>IP addresses</h4>
59+
60+
<% user_current_family, user_current_ip = split_hash_ip(@user.current_sign_in_ip, @user) %>
61+
<% user_joiner = user_current_family == 'IPv4' ? '.' : ':' %>
62+
<% target_current_family, target_current_ip = split_hash_ip(@target.current_sign_in_ip, @user) %>
63+
<% target_joiner = target_current_family == 'IPv4' ? '.' : ':' %>
64+
<strong>Current sign-in</strong><br/>
65+
<table class="table is-striped is-with-hover">
66+
<thead>
67+
<tr>
68+
<th>User</th>
69+
<th>Family</th>
70+
<th>Address</th>
71+
</tr>
72+
</thead>
73+
<tbody>
74+
<tr>
75+
<td><%= @user.rtl_safe_username %></td>
76+
<td><%= user_current_family %></td>
77+
<td>
78+
<% user_current_ip.map.with_index do |p, i| %>
79+
<code class="<%= 'has-background-color-red' if p == target_current_ip[i] %>">
80+
<%= p[0..3] %></code><%= user_joiner if i < user_current_ip.length - 1 %>
81+
<% end %>
82+
</td>
83+
</tr>
84+
<tr>
85+
<td><%= @target.rtl_safe_username %></td>
86+
<td><%= target_current_family %></td>
87+
<td>
88+
<% target_current_ip.map.with_index do |p, i| %>
89+
<code class="<%= 'has-background-color-red' if p == user_current_ip[i] %>">
90+
<%= p[0..3] %></code><%= target_joiner if i < target_current_ip.length - 1 %>
91+
<% end %>
92+
</td>
93+
</tr>
94+
</tbody>
95+
</table><br/>
96+
97+
<% user_last_family, user_last_ip = split_hash_ip(@user.last_sign_in_ip, @user) %>
98+
<% user_joiner = user_last_family == 'IPv4' ? '.' : ':' %>
99+
<% target_last_family, target_last_ip = split_hash_ip(@target.last_sign_in_ip, @user) %>
100+
<% target_joiner = target_last_family == 'IPv4' ? '.' : ':' %>
101+
<strong>Last sign-in</strong><br/>
102+
<table class="table is-striped is-with-hover">
103+
<thead>
104+
<tr>
105+
<th>User</th>
106+
<th>Family</th>
107+
<th>Address</th>
108+
</tr>
109+
</thead>
110+
<tbody>
111+
<tr>
112+
<td><%= @user.rtl_safe_username %></td>
113+
<td><%= user_last_family %></td>
114+
<td>
115+
<% user_last_ip.map.with_index do |p, i| %>
116+
<code class="<%= 'has-background-color-red' if p == target_last_ip[i] %>">
117+
<%= p[0..3] %></code><%= user_joiner if i < user_last_ip.length - 1 %>
118+
<% end %>
119+
</td>
120+
</tr>
121+
<tr>
122+
<td><%= @target.rtl_safe_username %></td>
123+
<td><%= target_last_family %></td>
124+
<td>
125+
<% target_last_ip.map.with_index do |p, i| %>
126+
<code class="<%= 'has-background-color-red' if p == user_last_ip[i] %>">
127+
<%= p[0..3] %></code><%= target_joiner if i < target_last_ip.length - 1 %>
128+
<% end %>
129+
</td>
130+
</tr>
131+
</tbody>
132+
</table>

‎app/views/users/mod.html.erb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<li><a href="/warning/log/<%= @user.id %>">warnings and suspensions sent to user</a> <% if @user.community_user.suspended? %>(includes lifting the suspension)<% end %></li>
1313
<li><a href="/warning/new/<%= @user.id %>">warn or suspend user</a></li>
1414
<li><%= link_to 'vote summary', mod_vote_summary_path(@user) %></li>
15+
<li><%= link_to 'compare PII', mod_pii_correlation_path(@user) %></li>
1516
<% if current_user.developer %>
1617
<li><%= link_to 'impersonate', start_impersonating_path(@user), class: 'is-yellow' %></li>
1718
<% end %>

0 commit comments

Comments
 (0)