Ingestion filters. Fixes #1192

app/assets/javascripts/source_filters.js

@@ -0,0 +1,36 @@
+var SourceFilters = {
+    lastId: function () {
+        var existing_list_item_ids = $(".source-filter-form").map(function (i, c) { return $(c).data("id-in-filter-list") });
+        if (existing_list_item_ids.length == 0) return 0;
+        return Math.max.apply(null, existing_list_item_ids) + 1;
+    },
+
+    add: function () {
+        var new_form = $($('#source-filter-template').clone().html().replace(/REPLACE_ME/g, SourceFilters.lastId()));
+        new_form.appendTo('#source-filter-list');
+
+        return false; // Stop form being submitted
+    },
+
+    addBlockFilter: function () {
+        var new_form = $($('#source-filter-template').clone().html().replace(/REPLACE_ME/g, SourceFilters.lastId()).replace(/allow/, 'block'));
+        new_form.appendTo('#source-block-list');
+
+        return false; // Stop form being submitted
+    },
+
+    delete: function () {
+        $(this).parents('.source-filter-form').fadeOut().find("input[name$='[_destroy]']").val("true");
+    }
+};
+
+document.addEventListener("turbolinks:load", function () {
+    $('#source-filters')
+        .on('click', '#add-source-filter-btn', SourceFilters.add)
+        .on('click', '#add-source-filter-btn-label', SourceFilters.add)
+        .on('click', '.delete-source-filter-btn', SourceFilters.delete);
+    $('#source-block-filters')
+        .on('click', '#add-source-block-filter-btn', SourceFilters.addBlockFilter)
+        .on('click', '#add-source-block-filter-btn-label', SourceFilters.addBlockFilter)
+        .on('click', '.delete-source-filter-btn', SourceFilters.delete);
+});

app/assets/stylesheets/sources.scss

@@ -0,0 +1,9 @@
+.source-filter-form {
+    display: flex;
+    gap: 1em;
+    margin-bottom: 4px;
+
+    label {
+        margin-right: 4px;
+    }
+}

app/controllers/sources_controller.rb

@@ -1,6 +1,6 @@
 class SourcesController < ApplicationController
-  before_action :set_source, except: [:index, :new, :create, :check_exists]
-  before_action :set_content_provider, except: [:index, :check_exists]
+  before_action :set_source, except: %i[index new create check_exists]
+  before_action :set_content_provider, except: %i[index check_exists]
   before_action :set_breadcrumbs
 
   include SearchableIndex
@@ -65,8 +65,8 @@ def check_exists
       end
     else
       respond_to do |format|
-        format.html { render :nothing => true, :status => 200, :content_type => 'text/html' }
-        format.json { render json: {}, :status => 200, :content_type => 'application/json' }
+        format.html { render nothing: true, status: 200, content_type: 'text/html' }
+        format.json { render json: {}, status: 200, content_type: 'application/json' }
       end
     end
   end
@@ -75,6 +75,7 @@ def check_exists
   # PATCH/PUT /sources/1.json
   def update
     authorize @source
+
     respond_to do |format|
       if @source.update(source_params)
         @source.create_activity(:update, owner: current_user) if @source.log_update_activity?
@@ -94,8 +95,10 @@ def destroy
     @source.create_activity :destroy, owner: current_user
     @source.destroy
     respond_to do |format|
-      format.html { redirect_to policy(Source).index? ? sources_path : content_provider_path(@content_provider),
-                                notice: 'Source was successfully deleted.' }
+      format.html do
+        redirect_to policy(Source).index? ? sources_path : content_provider_path(@content_provider),
+                    notice: 'Source was successfully deleted.'
+      end
       format.json { head :no_content }
     end
   end
@@ -106,7 +109,7 @@ def test
     @source.test_job_id = job_id
 
     respond_to do |format|
-      format.json { render json: { id: job_id }}
+      format.json { render json: { id: job_id } }
     end
   end
 
@@ -150,11 +153,11 @@ def set_content_provider
 
   # Never trust parameters from the scary internet, only allow the white list through.
   def source_params
-    permitted = [:url, :method, :token, :default_language, :enabled]
+    permitted = %i[url method token default_language enabled source_filters]
     permitted << :approval_status if policy(@source || Source).approve?
     permitted << :content_provider_id if policy(Source).index?
 
-    params.require(:source).permit(permitted)
+    params.require(:source).permit(permitted, source_filters_attributes: %i[id filter_mode filter_by filter_value _destroy])
   end
 
   def set_breadcrumbs
@@ -164,7 +167,7 @@ def set_breadcrumbs
       add_breadcrumb 'Sources', content_provider_path(@content_provider, anchor: 'sources')
 
       if params[:id]
-        add_breadcrumb @source.title, content_provider_source_path(@content_provider, @source) if (@source && !@source.new_record?)
+        add_breadcrumb @source.title, content_provider_source_path(@content_provider, @source) if @source && !@source.new_record?
         add_breadcrumb action_name.capitalize.humanize, request.path unless action_name == 'show'
       elsif action_name != 'index'
         add_breadcrumb action_name.capitalize.humanize, request.path
@@ -173,5 +176,4 @@ def set_breadcrumbs
       super
     end
   end
-
 end

app/models/source.rb

@@ -17,12 +17,13 @@ class Source < ApplicationRecord
 
   belongs_to :user
   belongs_to :content_provider
+  has_many :source_filters, dependent: :destroy
 
   validates :url, :method, presence: true
   validates :url, url: true
   validates :approval_status, inclusion: { in: APPROVAL_STATUS.values }
-  validates :method, inclusion: { in: -> (_) { TeSS::Config.user_ingestion_methods } },
-            unless: -> { User.current_user&.is_admin? || User.current_user&.has_role?(:scraper_user) }
+  validates :method, inclusion: { in: ->(_) { TeSS::Config.user_ingestion_methods } },
+                     unless: -> { User.current_user&.is_admin? || User.current_user&.has_role?(:scraper_user) }
   validates :default_language, controlled_vocabulary: { dictionary: 'LanguageDictionary',
                                                         allow_blank: true }
   validate :check_method
@@ -31,6 +32,8 @@ class Source < ApplicationRecord
   before_update :log_approval_status_change
   before_update :reset_approval_status
 
+  accepts_nested_attributes_for :source_filters, allow_destroy: true
+
   if TeSS::Config.solr_enabled
     # :nocov:
     searchable do
@@ -45,7 +48,7 @@ class Source < ApplicationRecord
         ingestor_title
       end
       string :content_provider do
-        self.content_provider.try(:title)
+        content_provider.try(:title)
       end
       string :node, multiple: true do
         associated_nodes.pluck(:name)
@@ -73,18 +76,16 @@ def ingestor_class
   end
 
   def self.facet_fields
-    field_list = %w( content_provider node method enabled approval_status )
+    field_list = %w[content_provider node method enabled approval_status]
     field_list.delete('node') unless TeSS::Config.feature['nodes']
     field_list
   end
 
   def self.check_exists(source_params)
-    given_source = self.new(source_params)
+    given_source = new(source_params)
     source = nil
 
-    if given_source.url.present?
-      source = self.find_by_url(given_source.url)
-    end
+    source = find_by_url(given_source.url) if given_source.url.present?
 
     source
   end
@@ -135,30 +136,45 @@ def self.approval_required?
     TeSS::Config.feature['user_source_creation'] && !User.current_user&.is_admin?
   end
 
+  def passes_filter?(item)
+    passes = false
+    allow_all = true
+
+    source_filters.each do |filter|
+      if filter.allow?
+        allow_all = false
+        passes ||= filter.match(item)
+      elsif filter.block? && filter.match(item)
+        return false
+      end
+    end
+
+    passes || allow_all
+  end
+
   private
 
   def set_approval_status
-    if self.class.approval_required?
-      self.approval_status = :not_approved
-    else
-      self.approval_status = :approved
-    end
+    self.approval_status = if self.class.approval_required?
+                             :not_approved
+                           else
+                             :approved
+                           end
   end
 
   def reset_approval_status
-    if self.class.approval_required?
-      if method_changed? || url_changed?
-        self.approval_status = :not_approved
-      end
-    end
+    return unless self.class.approval_required?
+    return unless method_changed? || url_changed?
+
+    self.approval_status = :not_approved
   end
 
   def log_approval_status_change
-    if approval_status_changed?
-      old = (APPROVAL_STATUS[approval_status_before_last_save.to_i] || APPROVAL_STATUS[0]).to_s
-      new = approval_status.to_s
-      create_activity(:approval_status_changed, owner: User.current_user, parameters: { old: old, new: new })
-    end
+    return unless approval_status_changed?
+
+    old = (APPROVAL_STATUS[approval_status_before_last_save.to_i] || APPROVAL_STATUS[0]).to_s
+    new = approval_status.to_s
+    create_activity(:approval_status_changed, owner: User.current_user, parameters: { old:, new: })
   end
 
   def loggable_changes

app/models/source_filter.rb

@@ -0,0 +1,68 @@
+class SourceFilter < ApplicationRecord
+  belongs_to :source
+
+  auto_strip_attributes :filter_value
+  validates :filter_mode, :filter_by, presence: true
+
+  enum :filter_by, {
+    target_audience: 'target_audience',
+    keyword: 'keyword',
+    title: 'title',
+    description: 'description',
+    description_contains: 'description_contains',
+    url: 'url',
+    url_prefix: 'url_prefix',
+    doi: 'doi',
+    license: 'license',
+    difficulty_level: 'difficulty_level',
+    resource_type: 'resource_type',
+    prerequisites_contains: 'prerequisites_contains',
+    learning_objectives_contains: 'learning_objectives_contains',
+    subtitle: 'subtitle',
+    subtitle_contains: 'subtitle_contains',
+    city: 'city',
+    country: 'country',
+    event_type: 'event_type',
+    timezone: 'timezone'
+  }
+
+  enum :filter_mode, {
+    allow: 'allow',
+    block: 'block'
+  }
+
+  def match(item)
+    return false unless item.respond_to?(filter_property)
+
+    val = item.send(filter_property)
+
+    # string match
+    if %w[title url doi description license difficulty_level subtitle city country timezone].include?(filter_by)
+      val.to_s.casecmp?(filter_value)
+    # prefix string match
+    elsif %w[url_prefix].include?(filter_by)
+      val.to_s.downcase.start_with?(filter_value.downcase)
+    # contains string match
+    elsif %w[description_contains prerequisites_contains learning_objectives_contains subtitle_contains].include?(filter_by)
+      val.to_s.downcase.include?(filter_value.downcase)
+    # array string match
+    elsif %w[target_audience keyword resource_type event_type].include?(filter_by)
+      val.any? { |i| i.to_s.casecmp?(filter_value) }
+    else
+      false
+    end
+  end
+
+  def filter_property
+    {
+      'event_type' => 'event_types',
+      'keyword' => 'keywords',
+      'url_prefix' => 'url',
+      'description_contains' => 'description',
+      'prerequisites_contains' => 'prerequisites',
+      'learning_objectives_contains' => 'learning_objectives',
+      'subtitle_contains' => 'subtitle',
+      'license' => 'licence'
+    }.fetch(filter_by, filter_by)
+  end
+end

app/views/sources/_form.html.erb

@@ -36,10 +36,53 @@
                 include_blank: false %>
   <% end %>
 
+  <h3 style="margin-top: 1em; margin-bottom: .5em;"><%= t('sources.headings.filters') %></h3>
+
+  <h4><%= t('sources.headings.allow_list') %></h4>
+  <span class="help-block"><%= t('sources.hints.allow_list') %></span>
+  <div class="form-group" id="source-filters">
+    <div id="source-filter-list">
+      <% f.object.source_filters.allow.each do |filter| %>
+        <%= f.simple_fields_for :source_filters, filter, child_index: filter.id do |ff| %>
+          <%= render partial: 'source_filter_form', locals: { f: ff } %>
+        <% end %>
+      <% end %>
+    </div>
+
+    <a href="#" id="add-source-filter-btn" class="btn btn-icon">
+      <i class="icon icon-h4 plus-icon"></i>
+    </a>
+    <span id="add-source-filter-btn-label" class="help-inline-block help-block"><%= t('sources.hints.add_filter') %></span>
+  </div>
+
+  <h4><%= t('sources.headings.block_list') %></h4>
+  <span class="help-block"><%= t('sources.hints.block_list') %></span>
+  <div class="form-group" id="source-block-filters">
+    <div id="source-block-list">
+      <% f.object.source_filters.block.each do |filter| %>
+        <%= f.simple_fields_for :source_filters, filter, child_index: filter.id do |ff| %>
+          <%= render partial: 'source_filter_form', locals: { f: ff } %>
+        <% end %>
+      <% end %>
+    </div>
+
+    <a href="#" id="add-source-block-filter-btn" class="btn btn-icon">
+      <i class="icon icon-h4 plus-icon"></i>
+    </a>
+    <span id="add-source-block-filter-btn-label" class="help-inline-block help-block"><%= t('sources.hints.add_filter') %></span>
+  </div>
+
+
   <div class="form-group actions">
     <%= f.submit(class: 'btn btn-primary') %>
     <%= link_to t('.cancel', default: t("helpers.links.cancel")),
                 sources_path, class: 'btn btn-default' %>
   </div>
 
+  <template id="source-filter-template">
+    <%= f.simple_fields_for :source_filters, SourceFilter.new(filter_mode: 'allow'), child_index: "REPLACE_ME" do |ff| %>
+      <%= render partial: 'source_filter_form', locals: { f: ff } %>
+    <% end %>
+  </template>
+
 <% end %>

app/views/sources/_source_filter_form.html.erb

@@ -0,0 +1,11 @@
+<div class="form-inline source-filter-form" data-id-in-filter-list="<%= f.options[:child_index] %>">
+  <%= f.input :filter_by, collection: SourceFilter.filter_bies.keys.map { |t| [t.humanize, t] }, include_blank: false, required: false %>
+
+  <%= f.input :filter_value %>
+
+  <%= f.input :filter_mode, collection: SourceFilter.filter_modes.keys.map { |m| [m.humanize, m] }, include_blank: false, as: :hidden %>
+
+  <%= f.input :_destroy, as: :hidden %>
+
+  <button type="button" class="btn btn-danger delete-source-filter-btn">Remove</button>
+</div>