Skip to content

Commit 7531d44

Browse files
fbacallfbacall
authored
Merge pull request #1201 from pan-training/ingestion_filters
Ingestion filters. Fixes #1192
2 parents c123125 + e56e1b4 commit 7531d44

22 files changed

Lines changed: 781 additions & 59 deletions

‎app/assets/javascripts/source_filters.js‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
var SourceFilters = {
2+
lastId: function () {
3+
var existing_list_item_ids = $(".source-filter-form").map(function (i, c) { return $(c).data("id-in-filter-list") });
4+
if (existing_list_item_ids.length == 0) return 0;
5+
return Math.max.apply(null, existing_list_item_ids) + 1;
6+
},
7+
8+
add: function () {
9+
var new_form = $($('#source-filter-template').clone().html().replace(/REPLACE_ME/g, SourceFilters.lastId()));
10+
new_form.appendTo('#source-filter-list');
11+
12+
return false; // Stop form being submitted
13+
},
14+
15+
addBlockFilter: function () {
16+
var new_form = $($('#source-filter-template').clone().html().replace(/REPLACE_ME/g, SourceFilters.lastId()).replace(/allow/, 'block'));
17+
new_form.appendTo('#source-block-list');
18+
19+
return false; // Stop form being submitted
20+
},
21+
22+
delete: function () {
23+
$(this).parents('.source-filter-form').fadeOut().find("input[name$='[_destroy]']").val("true");
24+
}
25+
};
26+
27+
document.addEventListener("turbolinks:load", function () {
28+
$('#source-filters')
29+
.on('click', '#add-source-filter-btn', SourceFilters.add)
30+
.on('click', '#add-source-filter-btn-label', SourceFilters.add)
31+
.on('click', '.delete-source-filter-btn', SourceFilters.delete);
32+
$('#source-block-filters')
33+
.on('click', '#add-source-block-filter-btn', SourceFilters.addBlockFilter)
34+
.on('click', '#add-source-block-filter-btn-label', SourceFilters.addBlockFilter)
35+
.on('click', '.delete-source-filter-btn', SourceFilters.delete);
36+
});

‎app/assets/stylesheets/application.scss‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
@import "curators";
3131
@import "events";
3232
@import "external-resources";
33+
@import "sources";
3334
@import "footer";
3435
@import "forms";
3536
@import "headers";

‎app/assets/stylesheets/sources.scss‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.source-filter-form {
2+
display: flex;
3+
gap: 1em;
4+
margin-bottom: 4px;
5+
6+
label {
7+
margin-right: 4px;
8+
}
9+
}

‎app/controllers/sources_controller.rb‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
class SourcesController < ApplicationController
2-
before_action :set_source, except: [:index, :new, :create, :check_exists]
3-
before_action :set_content_provider, except: [:index, :check_exists]
2+
before_action :set_source, except: %i[index new create check_exists]
3+
before_action :set_content_provider, except: %i[index check_exists]
44
before_action :set_breadcrumbs
55

66
include SearchableIndex
@@ -65,8 +65,8 @@ def check_exists
6565
end
6666
else
6767
respond_to do |format|
68-
format.html { render :nothing => true, :status => 200, :content_type => 'text/html' }
69-
format.json { render json: {}, :status => 200, :content_type => 'application/json' }
68+
format.html { render nothing: true, status: 200, content_type: 'text/html' }
69+
format.json { render json: {}, status: 200, content_type: 'application/json' }
7070
end
7171
end
7272
end
@@ -75,6 +75,7 @@ def check_exists
7575
# PATCH/PUT /sources/1.json
7676
def update
7777
authorize @source
78+
7879
respond_to do |format|
7980
if @source.update(source_params)
8081
@source.create_activity(:update, owner: current_user) if @source.log_update_activity?
@@ -94,8 +95,10 @@ def destroy
9495
@source.create_activity :destroy, owner: current_user
9596
@source.destroy
9697
respond_to do |format|
97-
format.html { redirect_to policy(Source).index? ? sources_path : content_provider_path(@content_provider),
98-
notice: 'Source was successfully deleted.' }
98+
format.html do
99+
redirect_to policy(Source).index? ? sources_path : content_provider_path(@content_provider),
100+
notice: 'Source was successfully deleted.'
101+
end
99102
format.json { head :no_content }
100103
end
101104
end
@@ -106,7 +109,7 @@ def test
106109
@source.test_job_id = job_id
107110

108111
respond_to do |format|
109-
format.json { render json: { id: job_id }}
112+
format.json { render json: { id: job_id } }
110113
end
111114
end
112115

@@ -150,11 +153,11 @@ def set_content_provider
150153

151154
# Never trust parameters from the scary internet, only allow the white list through.
152155
def source_params
153-
permitted = [:url, :method, :token, :default_language, :enabled]
156+
permitted = %i[url method token default_language enabled source_filters]
154157
permitted << :approval_status if policy(@source || Source).approve?
155158
permitted << :content_provider_id if policy(Source).index?
156159

157-
params.require(:source).permit(permitted)
160+
params.require(:source).permit(permitted, source_filters_attributes: %i[id mode property value _destroy])
158161
end
159162

160163
def set_breadcrumbs
@@ -164,7 +167,7 @@ def set_breadcrumbs
164167
add_breadcrumb 'Sources', content_provider_path(@content_provider, anchor: 'sources')
165168

166169
if params[:id]
167-
add_breadcrumb @source.title, content_provider_source_path(@content_provider, @source) if (@source && !@source.new_record?)
170+
add_breadcrumb @source.title, content_provider_source_path(@content_provider, @source) if @source && !@source.new_record?
168171
add_breadcrumb action_name.capitalize.humanize, request.path unless action_name == 'show'
169172
elsif action_name != 'index'
170173
add_breadcrumb action_name.capitalize.humanize, request.path
@@ -173,5 +176,4 @@ def set_breadcrumbs
173176
super
174177
end
175178
end
176-
177179
end

‎app/models/source.rb‎

Lines changed: 34 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@ class Source < ApplicationRecord
1717

1818
belongs_to :user
1919
belongs_to :content_provider
20+
has_many :source_filters, dependent: :destroy
2021

2122
validates :url, :method, presence: true
2223
validates :url, url: true
2324
validates :approval_status, inclusion: { in: APPROVAL_STATUS.values }
24-
validates :method, inclusion: { in: -> (_) { TeSS::Config.user_ingestion_methods } },
25-
unless: -> { User.current_user&.is_admin? || User.current_user&.has_role?(:scraper_user) }
25+
validates :method, inclusion: { in: ->(_) { TeSS::Config.user_ingestion_methods } },
26+
unless: -> { User.current_user&.is_admin? || User.current_user&.has_role?(:scraper_user) }
2627
validates :default_language, controlled_vocabulary: { dictionary: 'LanguageDictionary',
2728
allow_blank: true }
2829
validate :check_method
@@ -31,6 +32,8 @@ class Source < ApplicationRecord
3132
before_update :log_approval_status_change
3233
before_update :reset_approval_status
3334

35+
accepts_nested_attributes_for :source_filters, allow_destroy: true
36+
3437
if TeSS::Config.solr_enabled
3538
# :nocov:
3639
searchable do
@@ -45,7 +48,7 @@ class Source < ApplicationRecord
4548
ingestor_title
4649
end
4750
string :content_provider do
48-
self.content_provider.try(:title)
51+
content_provider.try(:title)
4952
end
5053
string :node, multiple: true do
5154
associated_nodes.pluck(:name)
@@ -79,12 +82,10 @@ def self.facet_fields
7982
end
8083

8184
def self.check_exists(source_params)
82-
given_source = self.new(source_params)
85+
given_source = new(source_params)
8386
source = nil
8487

85-
if given_source.url.present?
86-
source = self.find_by_url(given_source.url)
87-
end
88+
source = find_by_url(given_source.url) if given_source.url.present?
8889

8990
source
9091
end
@@ -135,30 +136,41 @@ def self.approval_required?
135136
TeSS::Config.feature['user_source_creation'] && !User.current_user&.is_admin?
136137
end
137138

139+
def passes_filter?(item)
140+
source_filters.each do |filter|
141+
if filter.allow?
142+
return false unless filter.match(item)
143+
elsif filter.block?
144+
return false if filter.match(item)
145+
end
146+
end
147+
148+
true
149+
end
150+
138151
private
139152

140153
def set_approval_status
141-
if self.class.approval_required?
142-
self.approval_status = :not_approved
143-
else
144-
self.approval_status = :approved
145-
end
154+
self.approval_status = if self.class.approval_required?
155+
:not_approved
156+
else
157+
:approved
158+
end
146159
end
147160

148161
def reset_approval_status
149-
if self.class.approval_required?
150-
if method_changed? || url_changed?
151-
self.approval_status = :not_approved
152-
end
153-
end
162+
return unless self.class.approval_required?
163+
return unless method_changed? || url_changed?
164+
165+
self.approval_status = :not_approved
154166
end
155167

156168
def log_approval_status_change
157-
if approval_status_changed?
158-
old = (APPROVAL_STATUS[approval_status_before_last_save.to_i] || APPROVAL_STATUS[0]).to_s
159-
new = approval_status.to_s
160-
create_activity(:approval_status_changed, owner: User.current_user, parameters: { old: old, new: new })
161-
end
169+
return unless approval_status_changed?
170+
171+
old = (APPROVAL_STATUS[approval_status_before_last_save.to_i] || APPROVAL_STATUS[0]).to_s
172+
new = approval_status.to_s
173+
create_activity(:approval_status_changed, owner: User.current_user, parameters: { old:, new: })
162174
end
163175

164176
def loggable_changes

‎app/models/source_filter.rb‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
2+
module FilterComparisons
3+
def self.string_match(content_value, filter_value)
4+
content_value.to_s.casecmp?(filter_value)
5+
end
6+
7+
def self.prefix_string_match(content_value, filter_value)
8+
content_value.to_s.downcase.start_with?(filter_value.downcase)
9+
end
10+
11+
def self.contains_string_match(content_value, filter_value)
12+
content_value.to_s.downcase.include?(filter_value.downcase)
13+
end
14+
15+
def self.array_string_match(content_value, filter_value)
16+
return false if content_value.nil?
17+
content_value.any? { |i| i.to_s.casecmp?(filter_value) }
18+
end
19+
end
20+
21+
22+
FILTER_DEFINITIONS = {
23+
'target_audience' => {
24+
comparison: FilterComparisons.method(:array_string_match)
25+
},
26+
27+
'keyword' => {
28+
comparison: FilterComparisons.method(:array_string_match),
29+
filter_property: 'keywords'
30+
},
31+
32+
'title' => {
33+
comparison: FilterComparisons.method(:string_match)
34+
},
35+
36+
'title_contains' => {
37+
comparison: FilterComparisons.method(:contains_string_match),
38+
filter_property: 'title'
39+
},
40+
41+
'description' => {
42+
comparison: FilterComparisons.method(:string_match)
43+
},
44+
45+
'description_contains' => {
46+
comparison: FilterComparisons.method(:contains_string_match),
47+
filter_property: 'description'
48+
},
49+
50+
'url' => {
51+
comparison: FilterComparisons.method(:string_match)
52+
},
53+
54+
'url_prefix' => {
55+
comparison: FilterComparisons.method(:prefix_string_match),
56+
filter_property: 'url'
57+
},
58+
59+
'doi' => {
60+
comparison: FilterComparisons.method(:string_match)
61+
},
62+
63+
'license' => {
64+
comparison: FilterComparisons.method(:string_match),
65+
filter_property: 'licence'
66+
},
67+
68+
'difficulty_level' => {
69+
comparison: FilterComparisons.method(:string_match)
70+
},
71+
72+
'resource_type' => {
73+
comparison: FilterComparisons.method(:array_string_match)
74+
},
75+
76+
'prerequisites_contains' => {
77+
comparison: FilterComparisons.method(:contains_string_match),
78+
filter_property: 'prerequisites'
79+
},
80+
81+
'learning_objectives_contains' => {
82+
comparison: FilterComparisons.method(:contains_string_match),
83+
filter_property: 'learning_objectives'
84+
},
85+
86+
'subtitle' => {
87+
comparison: FilterComparisons.method(:string_match)
88+
},
89+
90+
'subtitle_contains' => {
91+
comparison: FilterComparisons.method(:contains_string_match),
92+
filter_property: 'subtitle'
93+
},
94+
95+
'city' => {
96+
comparison: FilterComparisons.method(:string_match)
97+
},
98+
99+
'country' => {
100+
comparison: FilterComparisons.method(:string_match)
101+
},
102+
103+
'event_type' => {
104+
comparison: FilterComparisons.method(:array_string_match),
105+
filter_property: 'event_types'
106+
},
107+
108+
'timezone' => {
109+
comparison: FilterComparisons.method(:string_match)
110+
}
111+
}.freeze
112+
113+
114+
class SourceFilter < ApplicationRecord
115+
belongs_to :source
116+
117+
auto_strip_attributes :value
118+
validates :mode, :property, presence: true
119+
120+
enum :property, FILTER_DEFINITIONS.keys.index_with(&:itself)
121+
122+
enum :mode, {
123+
allow: 'allow',
124+
block: 'block'
125+
}
126+
127+
def match(item)
128+
val = nil
129+
val = item.send(filter_property) if item.respond_to?(filter_property)
130+
filter_definition[:comparison].call(val, value.to_s)
131+
end
132+
133+
def filter_definition
134+
FILTER_DEFINITIONS[property]
135+
end
136+
137+
def filter_property
138+
filter_definition[:filter_property] || property
139+
end
140+
end

0 commit comments

Comments
 (0)