Merge pull request #3434 from matthewbadeau/feature/ignore_files

Configure files / directories that should be ignored during scan

Gemfile

@@ -171,3 +171,5 @@ gem "pghero", "~> 3.6"
 gem "pg_query", "~> 6.0"
 
 gem "get_process_mem", "~> 1.0"
+
+gem "to_regexp", "~> 0.2.1"

Gemfile.lock

@@ -730,6 +730,7 @@ GEM
     text (1.3.1)
     thor (1.3.2)
     timeout (0.4.3)
+    to_regexp (0.2.1)
     translation (1.41)
       gettext (~> 3.2, >= 3.2.5, <= 3.4.9)
     treetop (1.6.12)
@@ -869,6 +870,7 @@ DEPENDENCIES
   stopwords-filter2
   string-similarity (~> 2.1)
   sys-filesystem (~> 1.5)
+  to_regexp (~> 0.2.1)
   translation (~> 1.41)
   tus-server (~> 2.3)
   tzinfo-data
@@ -879,4 +881,4 @@ RUBY VERSION
    ruby 3.4.1p0
 
 BUNDLED WITH
-   2.5.23
+   2.6.2

app/controllers/settings_controller.rb

@@ -4,6 +4,7 @@ class SettingsController < ApplicationController
   def update
     # Save site-wide settings if user is an admin
     update_folder_settings(params[:folders])
+    update_file_settings(params[:files])
     update_tagging_settings(params[:model_tags])
     update_multiuser_settings(params[:multiuser])
     update_analysis_settings(params[:analysis])
@@ -20,6 +21,13 @@ def update_folder_settings(settings)
     SiteSettings.safe_folder_names = settings[:safe_folder_names]
   end
 
+  def update_file_settings(settings)
+    return unless settings
+    unless settings[:model_ignored_files].split("\n").any? { |p| p.to_regexp.nil? }
+      SiteSettings.model_ignored_files = settings[:model_ignored_files]
+    end
+  end
+
   def update_tagging_settings(settings)
     return unless settings
     SiteSettings.model_tags_filter_stop_words = settings[:filter_stop_words] == "1"

app/models/site_settings.rb

@@ -8,13 +8,20 @@ class SiteSettings < RailsSettings::Base
   field :model_tags_custom_stop_words, type: :array, default: SupportedMimeTypes.indexable_extensions
   field :model_tags_auto_tag_new, type: :string, default: "!new"
   field :model_path_template, type: :string, default: "{tags}/{modelName}{modelId}"
+  field :model_ignored_files, type: :array, default: [
+    /^\.[^\.]+/, # Hidden files starting with .
+    /.*\/@eaDir\/.*/, # Synology temp files
+    /__MACOSX/ # MACOS resource forks
+  ]
   field :parse_metadata_from_path, type: :boolean, default: true
   field :safe_folder_names, type: :boolean, default: true
   field :analyse_manifold, type: :boolean, default: false
   field :anonymous_usage_id, type: :string, default: nil
   field :default_viewer_role, type: :string, default: "member"
   field :approve_signups, type: :boolean, default: false
 
+  validates :model_ignored_files, regex_array: {strict: true}
+
   def self.registration_enabled?
     Rails.application.config.manyfold_features[:registration]
   end
@@ -52,13 +59,9 @@ def self.default_user
   end
 
   def self.ignored_file?(pathname)
-    @@patterns ||= [
-      /^\.[^\.]+/, # Hidden files starting with .
-      /.*\/@eaDir\/.*/, # Synology temp files
-      /__MACOSX/ # MACOS resource forks
-    ]
+    patterns ||= model_ignored_files
     (File.split(pathname) - ["."]).any? do |path_component|
-      @@patterns.any? { |pattern| path_component =~ pattern }
+      patterns.any? { |pattern| path_component =~ pattern.to_regexp }
     end
   end
 

app/validators/regex_array_validator.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class RegexArrayValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    return unless value.any? { |pattern| pattern.to_regexp.nil? }
+    record.errors.add(attribute, :invalid)
+  end
+end

app/views/settings/_file_settings.html.erb

@@ -0,0 +1,13 @@
+<h3><%= t(".title") %></h3>
+<p class='lead'>
+  <%= t ".summary" %>
+</p>
+<div class="row mb-2">
+  <p>
+    <%= t ".custom_ignore_filters.details" %>
+  </p>
+  <%= form.label nil, t(".custom_ignore_filters.label"), for: "files[model_ignored_files]", class: "col-sm-4 col-form-label" %>
+  <div class="col-sm-8 form-check form-switch">
+    <%= form.text_area "files[model_ignored_files]", value: SiteSettings.model_ignored_files.join("\n").gsub(/^\[|\]$/, ""), class: "form-control" %>
+  </div>
+</div>

app/views/settings/show.html.erb

@@ -1,6 +1,8 @@
 <%= form_with url: settings_path, method: :patch do |form| %>
   <%= render "folder_settings", form: form %>
   <hr>
+  <%= render "file_settings", form: form %>
+  <hr>
   <%= render "tag_settings", form: form %>
 
   <%= render "submit", form: form %>

config/locales/settings/en.yml

@@ -20,6 +20,12 @@ en:
       new:
         submit: Block domain
         title: New Domain Block
+    file_settings:
+      custom_ignore_filters:
+        details: Filters are in regex format separated by a new line.
+        label: Ignored files regex filters.
+      summary: Change file settings such as ignored files.
+      title: File settings
     folder_settings:
       details: Folder structure follows a template that you define using tokens. You can also include other text in the template (such as folder separators) and it will be included as-is.
       model_path_template:

spec/validators/regex_array_validator_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe RegexArrayValidator do
+  subject(:validator) { described_class.new({attributes: {any: true}}) }
+
+  let(:errors) { ActiveModel::Errors.new(subject) }
+
+  let(:record) { instance_double(ActiveModel::Validations, errors: errors) }
+
+  let(:array_of_regexes) {
+    %w[
+      /^\.[^\.]+/
+      /.*\/@eaDir\/.*/
+      /__MACOSX/
+    ]
+  }
+
+  describe "#validate_each(record, attribute, value)" do
+    it "adds error to invalid record when the array has an invalid regex" do
+      array_of_regexes.push "INVALID_REGEX"
+      expect {
+        validator.validate_each(record, :model_ignore_files, array_of_regexes)
+      }.to change(record.errors, :count)
+        .and change { record.errors.first&.type }.to eq(:invalid)
+    end
+
+    it "does not add error to invalid record when array contains only regexes" do
+      puts array_of_regexes.inspect
+      expect {
+        validator.validate_each(record, :model_ignore_files, array_of_regexes)
+      }.not_to change(record.errors, :count)
+    end
+  end
+end