diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 90f48cb53..7891e85fa 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/deployment-checks.yml b/.github/workflows/deployment-checks.yml index e360af9f4..966cf6744 100644 --- a/.github/workflows/deployment-checks.yml +++ b/.github/workflows/deployment-checks.yml @@ -36,7 +36,7 @@ jobs: ports: - 6379:6379 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 name: Set up Ruby - uses: ruby/setup-ruby@v1 with: @@ -49,7 +49,7 @@ jobs: cp config/ingestion.example.yml config/ingestion.yml bundle exec rake db:setup - name: Set up Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: '20.x' cache: 'yarn' diff --git a/.github/workflows/docker-prod-test.yml b/.github/workflows/docker-prod-test.yml index d60e16a42..7470afb86 100644 --- a/.github/workflows/docker-prod-test.yml +++ b/.github/workflows/docker-prod-test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup run: | cp env.sample .env diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index e19ec6921..dbdf61479 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup run: | cp env.sample .env diff --git a/.github/workflows/migration-test.yml b/.github/workflows/migration-test.yml index b16fec19a..8eac72cd1 100644 --- a/.github/workflows/migration-test.yml +++ b/.github/workflows/migration-test.yml @@ -49,7 +49,7 @@ jobs: sudo apt-get update sudo apt-get install imagemagick - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install Ruby & gems uses: ruby/setup-ruby@v1 with: @@ -76,7 +76,7 @@ jobs: rm -rf db/migrate mv db/migrate.bkp db/migrate - name: Set up Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: '20.x' cache: 'yarn' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 578947491..501914cc0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,11 +42,13 @@ jobs: sudo apt-get update sudo apt-get install imagemagick - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install Ruby & gems uses: ruby/setup-ruby@v1 with: bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Audit gems + run: bundle exec bundle-audit check --update - name: Configure and initialize database run: | cp test/config/test_tess.yml config/tess.yml @@ -54,7 +56,7 @@ jobs: cp config/ingestion.example.yml config/ingestion.yml bundle exec rake db:test:prepare - name: Set up Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: '20.x' cache: 'yarn' diff --git a/.ruby-version b/.ruby-version index f0f3ba8b6..61a52c914 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-3.3.10 \ No newline at end of file +ruby-3.4.9 diff --git a/Dockerfile b/Dockerfile index 7124e862e..e41954a87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION=3.3.10 +ARG RUBY_VERSION=3.4.9 #use ruby base image FROM ruby:$RUBY_VERSION-slim AS base diff --git a/Gemfile b/Gemfile index 98b174175..e84c1ef21 100644 --- a/Gemfile +++ b/Gemfile @@ -1,8 +1,8 @@ # frozen_string_literal: true -source 'https://rubygems.org' +source 'https://rubygems.org', cooldown: 14 -gem 'rails', '7.2.3' +gem 'rails', '8.1.3' gem 'active_model_serializers' gem 'activerecord-session_store' @@ -37,7 +37,6 @@ gem 'maxmind-db' gem 'money-rails' gem 'oai' gem 'omniauth_openid_connect' -gem 'omniauth-rails_csrf_protection' gem 'pg' gem 'private_address_check' gem 'public_activity' @@ -51,6 +50,7 @@ gem 'redis' gem 'rest-client' gem 'reverse_markdown' gem 'rss' +gem 'ruby-openai' gem 'sassc-rails' gem 'sass-rails' gem 'sentry-rails' @@ -60,7 +60,7 @@ gem 'sidekiq' gem 'sidekiq-status' gem 'simple_calendar', '~> 2.4' gem 'simple_form' -gem 'simple_token_authentication' +gem 'simple_token_authentication', git: 'https://github.com/gonzalo-bulnes/simple_token_authentication.git', ref: 'f1cba4e' # Rails 8 support not yet released on Rubygems gem 'sitemap_generator' gem 'sitemap-parser' gem 'slim' @@ -75,6 +75,7 @@ gem 'whenever', require: false # Work around https://github.com/javan/whenever/i gem 'will_paginate' group :development, :test do + gem 'bundle-audit' gem 'byebug' gem 'pry-byebug' gem 'rubocop' @@ -100,5 +101,3 @@ group :test do gem 'vcr' gem 'webmock' end - -gem 'ruby-openai' diff --git a/Gemfile.lock b/Gemfile.lock index 0f8b3d923..d469a5335 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,97 +6,108 @@ GIT tess_rdf_extractors (1.2.0) linkeddata (~> 3.2.0) +GIT + remote: https://github.com/gonzalo-bulnes/simple_token_authentication.git + revision: f1cba4e3e614932ffcb6dfd2d8bf03acd64cb1ec + ref: f1cba4e + specs: + simple_token_authentication (1.18.1) + actionmailer (>= 3.2.6, < 9) + actionpack (>= 3.2.6, < 9) + devise (>= 3.2, < 6) + GEM remote: https://rubygems.org/ specs: - actioncable (7.2.3) - actionpack (= 7.2.3) - activesupport (= 7.2.3) + action_text-trix (2.1.19) + railties + actioncable (8.1.3) + actionpack (= 8.1.3) + activesupport (= 8.1.3) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.2.3) - actionpack (= 7.2.3) - activejob (= 7.2.3) - activerecord (= 7.2.3) - activestorage (= 7.2.3) - activesupport (= 7.2.3) + actionmailbox (8.1.3) + actionpack (= 8.1.3) + activejob (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) mail (>= 2.8.0) - actionmailer (7.2.3) - actionpack (= 7.2.3) - actionview (= 7.2.3) - activejob (= 7.2.3) - activesupport (= 7.2.3) + actionmailer (8.1.3) + actionpack (= 8.1.3) + actionview (= 8.1.3) + activejob (= 8.1.3) + activesupport (= 8.1.3) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.2.3) - actionview (= 7.2.3) - activesupport (= 7.2.3) - cgi + actionpack (8.1.3) + actionview (= 8.1.3) + activesupport (= 8.1.3) nokogiri (>= 1.8.5) - racc - rack (>= 2.2.4, < 3.3) + rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (7.2.3) - actionpack (= 7.2.3) - activerecord (= 7.2.3) - activestorage (= 7.2.3) - activesupport (= 7.2.3) + actiontext (8.1.3) + action_text-trix (~> 2.1.15) + actionpack (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.2.3) - activesupport (= 7.2.3) + actionview (8.1.3) + activesupport (= 8.1.3) builder (~> 3.1) - cgi erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - active_model_serializers (0.10.14) + active_model_serializers (0.10.16) actionpack (>= 4.1) activemodel (>= 4.1) case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) - activejob (7.2.3) - activesupport (= 7.2.3) + activejob (8.1.3) + activesupport (= 8.1.3) globalid (>= 0.3.6) - activemodel (7.2.3) - activesupport (= 7.2.3) + activemodel (8.1.3) + activesupport (= 8.1.3) activemodel-serializers-xml (1.0.3) activemodel (>= 5.0.0.a) activesupport (>= 5.0.0.a) builder (~> 3.1) - activerecord (7.2.3) - activemodel (= 7.2.3) - activesupport (= 7.2.3) + activerecord (8.1.3) + activemodel (= 8.1.3) + activesupport (= 8.1.3) timeout (>= 0.4.0) - activerecord-session_store (2.0.0) - actionpack (>= 5.2.4.1) - activerecord (>= 5.2.4.1) - multi_json (~> 1.11, >= 1.11.2) - rack (>= 2.0.8, < 3) - railties (>= 5.2.4.1) - activestorage (7.2.3) - actionpack (= 7.2.3) - activejob (= 7.2.3) - activerecord (= 7.2.3) - activesupport (= 7.2.3) + activerecord-session_store (2.2.0) + actionpack (>= 7.0) + activerecord (>= 7.0) + cgi (>= 0.3.6) + rack (>= 2.0.8, < 4) + railties (>= 7.0) + activestorage (8.1.3) + actionpack (= 8.1.3) + activejob (= 8.1.3) + activerecord (= 8.1.3) + activesupport (= 8.1.3) marcel (~> 1.0) - activesupport (7.2.3) + activesupport (8.1.3) base64 - benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + json logger (>= 1.4.2) minitest (>= 5.1) securerandom (>= 0.3) tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) addressable (2.9.0) public_suffix (>= 2.0.2, < 8.0) aes_key_wrap (1.1.0) @@ -114,7 +125,6 @@ GEM base64 (0.3.0) bcp47_spec (0.2.1) bcrypt (3.1.22) - benchmark (0.5.0) better_errors (2.10.1) erubi (>= 1.0.0) rack (>= 0.9.0) @@ -134,6 +144,11 @@ GEM bootstrap-tab-history-rails (0.1.0) railties (>= 3.1) builder (3.3.0) + bundle-audit (0.2.0) + bundler-audit + bundler-audit (0.9.3) + bundler (>= 1.2.0) + thor (~> 1.0) byebug (11.1.3) capybara (3.40.0) addressable @@ -299,12 +314,12 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) - kt-paperclip (7.2.0) + kt-paperclip (7.3.0) activemodel (>= 4.2.0) activesupport (>= 4.2.0) marcel (~> 1.0.1) mime-types - terrapin (~> 0.6.0) + terrapin (>= 0.6.0, < 2.0) language_server-protocol (3.17.0.3) ld-patch (3.3.0) ebnf (~> 2.4) @@ -407,13 +422,11 @@ GEM faraday (< 3) faraday-follow_redirects (>= 0.3.0, < 2) rexml - omniauth (2.1.1) + omniauth (2.1.4) hashie (>= 3.4.6) + logger rack (>= 2.2.3) rack-protection - omniauth-rails_csrf_protection (1.0.1) - actionpack (>= 4.2) - omniauth (~> 2.0) omniauth_openid_connect (0.7.1) omniauth (>= 1.9, < 3) openid_connect (~> 2.2) @@ -459,12 +472,12 @@ GEM i18n (>= 0.5.0) railties (>= 6.1.0) public_suffix (7.0.5) - puma (6.5.0) + puma (8.0.2) nio4r (~> 2.0) pundit (2.3.1) activesupport (>= 3.0.0) racc (1.8.1) - rack (2.2.23) + rack (3.2.6) rack-cors (2.0.2) rack (>= 2.0.0) rack-oauth2 (2.2.0) @@ -474,29 +487,31 @@ GEM faraday-follow_redirects json-jwt (>= 1.11.0) rack (>= 2.1.0) - rack-protection (3.1.0) - rack (~> 2.2, >= 2.2.4) - rack-session (1.0.2) - rack (< 3) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.2) + base64 (>= 0.1.0) + rack (>= 3.0.0) rack-test (2.2.0) rack (>= 1.3) - rackup (1.0.1) - rack (< 3) - webrick - rails (7.2.3) - actioncable (= 7.2.3) - actionmailbox (= 7.2.3) - actionmailer (= 7.2.3) - actionpack (= 7.2.3) - actiontext (= 7.2.3) - actionview (= 7.2.3) - activejob (= 7.2.3) - activemodel (= 7.2.3) - activerecord (= 7.2.3) - activestorage (= 7.2.3) - activesupport (= 7.2.3) + rackup (2.3.1) + rack (>= 3) + rails (8.1.3) + actioncable (= 8.1.3) + actionmailbox (= 8.1.3) + actionmailer (= 8.1.3) + actionpack (= 8.1.3) + actiontext (= 8.1.3) + actionview (= 8.1.3) + activejob (= 8.1.3) + activemodel (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) bundler (>= 1.15.0) - railties (= 7.2.3) + railties (= 8.1.3) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) actionview (>= 5.0.1.rc1) @@ -508,9 +523,9 @@ GEM rails-html-sanitizer (1.7.0) loofah (~> 2.25) nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) - rails-i18n (7.0.8) + rails-i18n (8.1.0) i18n (>= 0.7, < 2) - railties (>= 6.0.0, < 8) + railties (>= 8.0.0, < 9) rails_admin (3.3.0) activemodel-serializers-xml (>= 1.0) csv @@ -518,10 +533,9 @@ GEM nested_form (~> 0.3) rails (>= 6.0, < 9) turbo-rails (>= 1.0, < 3) - railties (7.2.3) - actionpack (= 7.2.3) - activesupport (= 7.2.3) - cgi + railties (8.1.3) + actionpack (= 8.1.3) + activesupport (= 8.1.3) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) @@ -611,7 +625,7 @@ GEM redcarpet (3.6.0) redis (5.0.7) redis-client (>= 0.9.0) - redis-client (0.16.0) + redis-client (0.29.0) connection_pool regexp_parser (2.8.1) reline (0.6.3) @@ -624,7 +638,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - reverse_markdown (2.1.1) + reverse_markdown (3.0.2) nokogiri rexml (3.4.4) rouge (4.1.3) @@ -646,7 +660,7 @@ GEM unicode-display_width (>= 2.4.0, < 3.0) rubocop-ast (1.29.0) parser (>= 3.2.1.0) - ruby-openai (7.3.1) + ruby-openai (8.3.0) event_stream_parser (>= 0.3.0, < 2.0.0) faraday (>= 1) faraday-multipart (>= 1) @@ -676,14 +690,16 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 4.0) websocket (~> 1.0) - sentry-rails (5.11.0) - railties (>= 5.0) - sentry-ruby (~> 5.11.0) - sentry-ruby (5.11.0) + sentry-rails (6.5.0) + railties (>= 5.2.0) + sentry-ruby (~> 6.5.0) + sentry-ruby (6.5.0) + bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sentry-sidekiq (5.11.0) - sentry-ruby (~> 5.11.0) - sidekiq (>= 3.0) + logger + sentry-sidekiq (6.5.0) + sentry-ruby (~> 6.5.0) + sidekiq (>= 5.0) shacl (0.4.1) json-ld (~> 3.3) rdf (~> 3.3) @@ -698,24 +714,23 @@ GEM rdf-xsd (~> 3.3) sparql (~> 3.3) sxp (~> 1.3) - sidekiq (7.1.6) - concurrent-ruby (< 2) - connection_pool (>= 2.3.0) - rack (>= 2.2.4) - redis-client (>= 0.14.0) - sidekiq-status (3.0.3) + sidekiq (8.1.6) + connection_pool (>= 3.0.0) + json (>= 2.16.0) + logger (>= 1.7.0) + rack (>= 3.2.0) + redis-client (>= 0.29.0) + sidekiq-status (4.0.0) + base64 chronic_duration - sidekiq (>= 6.0, < 8) + logger + sidekiq (>= 7, < 9) simple_calendar (2.4.3) rails (>= 3.0) simple_form (5.2.0) actionpack (>= 5.2) activemodel (>= 5.2) simple_po_parser (1.1.6) - simple_token_authentication (1.18.1) - actionmailer (>= 3.2.6, < 8) - actionpack (>= 3.2.6, < 8) - devise (>= 3.2, < 6) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -726,7 +741,7 @@ GEM sitemap-parser (0.5.6) nokogiri (~> 1.6) typhoeus (>= 0.6, < 2.0) - sitemap_generator (6.3.0) + sitemap_generator (7.0.1) builder (~> 3.0) slim (5.1.1) temple (~> 0.10.0) @@ -767,8 +782,8 @@ GEM matrix (~> 0.4) rdf (~> 3.3) temple (0.10.4) - terrapin (0.6.0) - climate_control (>= 0.0.3, < 1.0) + terrapin (1.1.1) + climate_control terser (1.1.17) execjs (>= 0.3.0, < 3) thor (1.5.0) @@ -813,11 +828,10 @@ GEM activesupport faraday (~> 2.0) faraday-follow_redirects - webmock (3.18.1) + webmock (3.26.2) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.9.2) websocket (1.2.11) websocket-driver (0.8.0) base64 @@ -825,7 +839,7 @@ GEM websocket-extensions (0.1.5) whenever (1.0.0) chronic (>= 0.6.3) - will_paginate (4.0.0) + will_paginate (4.0.1) xpath (3.2.0) nokogiri (~> 1.8) yaml-ld (0.0.3) @@ -850,6 +864,7 @@ DEPENDENCIES bootstrap-datepicker-rails bootstrap-sass bootstrap-tab-history-rails + bundle-audit byebug capybara committee @@ -879,7 +894,6 @@ DEPENDENCIES minitest-reporters money-rails oai - omniauth-rails_csrf_protection omniauth_openid_connect pg private_address_check @@ -888,7 +902,7 @@ DEPENDENCIES puma pundit rack-cors - rails (= 7.2.3) + rails (= 8.1.3) rails-controller-testing rails-i18n rails_admin @@ -910,7 +924,7 @@ DEPENDENCIES sidekiq-status simple_calendar (~> 2.4) simple_form - simple_token_authentication + simple_token_authentication! simplecov simplecov-lcov sitemap-parser @@ -930,4 +944,4 @@ DEPENDENCIES will_paginate BUNDLED WITH - 2.5.22 + 4.0.13 diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 13fc15d1f..505184359 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -37,6 +37,7 @@ def pundit_user def handle_error(status_code = 500, message = nil) status_code = (params[:status_code] || status_code) # params[:status_code] comes from routes for 500, 503, 422 and 404 errors if status_code.is_a?(Symbol) # Convert :forbidden, :not_found, etc. to 403, 404 etc. + status_code = :unprocessable_content if status_code == :unprocessable_entity status_code = Rack::Utils::SYMBOL_TO_STATUS_CODE[status_code] || status_code end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 8c1ef2d02..bd7e0cb38 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -251,8 +251,8 @@ def twitter_link(username) end def info_button(title, opts = {}, &block) - classes = 'btn btn-default has-popover' - classes << " #{opts[:class]}" if opts[:class] + classes = %w(btn btn-default has-popover) + classes << opts[:class] if opts[:class] title_text = opts[:hide_text] ? '' : title content_tag(:a, tabindex: 0, class: classes, data: { toggle: 'popover', placement: 'bottom', @@ -300,7 +300,7 @@ def tab(text, icon, href, disabled: { check: false }, active: false, count: nil, options['data-tab-history-update-url'] = true end - text << " (#{count})" if count + text += " (#{count})" if count link_to("##{href}", options) do content_tag(:i, nil, class: icon, 'aria-hidden': 'true') + ' ' + text diff --git a/app/helpers/materials_helper.rb b/app/helpers/materials_helper.rb index 9ddb7feb1..c8a91f3cb 100644 --- a/app/helpers/materials_helper.rb +++ b/app/helpers/materials_helper.rb @@ -84,24 +84,21 @@ def display_attribute(resource, attribute, show_label: true, title: nil, markdow html_escape(block_given? ? yield(value) : value) end end - string = "

" - unless value.blank? || value.try(:strip) == 'License Not Specified' - string << " #{title || resource.class.human_attribute_name(attribute)}: " if show_label - if list - string << '

' - elsif expandable - height_limit = expandable.is_a?(Numeric) ? expandable : nil - string << "
" + value.to_s + '
' - else - string << value.to_s end end - string << '

' - string.html_safe end def display_people(resource, attribute) diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index 798da2c0b..3ddceac5c 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -1,6 +1,6 @@ -require 'request_forgery_protection_token_verifier' - Rails.application.config.middleware.use OmniAuth::Builder do OmniAuth.config.allowed_request_methods = [:post] - OmniAuth.config.before_request_phase = RequestForgeryProtectionTokenVerifier.new + + OmniAuth.config.request_validation_phase = Rails.env.test? ? nil : OmniAuth::AuthenticityTokenProtection.new(key: :_csrf_token) end + diff --git a/lib/ingestors/bioschemas_ingestor.rb b/lib/ingestors/bioschemas_ingestor.rb index 8be6fb050..6d931fe31 100644 --- a/lib/ingestors/bioschemas_ingestor.rb +++ b/lib/ingestors/bioschemas_ingestor.rb @@ -23,7 +23,7 @@ def read(source_url) provider_events = [] provider_materials = [] totals = Hash.new(0) - no_bioschema_urls = "Bioschemas not found in:\n" + no_bioschema_urls = [] sources.each do |url| source = open_url(url) output = read_content(source, url: url) @@ -33,19 +33,19 @@ def read(source_url) output[:totals].each do |key, value| totals[key] += value end - no_bioschema_urls << "\n - #{url}" if !source.nil? && output[:totals].values.sum.zero? + no_bioschema_urls << url if !source.nil? && output[:totals].values.sum.zero? end end if totals.keys.any? - bioschemas_summary = "Bioschemas summary:\n" + bioschemas_summary = ["Bioschemas summary:\n"] totals.each do |type, count| - bioschemas_summary << "\n - #{type}: #{count}" + bioschemas_summary << " - #{type}: #{count}" end - @messages << bioschemas_summary + @messages << bioschemas_summary.join("\n") end - @messages << no_bioschema_urls + @messages << "Bioschemas not found in:\n\n#{no_bioschema_urls.map { |u| " - #{u}" }.join("\n")}" if no_bioschema_urls.any? deduplicate(provider_events).each do |event_params| add_event(event_params) @@ -113,7 +113,7 @@ def read_content(content, url: nil) error = 'A parsing error' comment = 'Please check your page contains valid JSON-LD or HTML.' end - message = "#{error} occurred while reading" + message = String.new("#{error} occurred while reading") if url.present? && url != 'https://example.com' message << ": #{url} " else diff --git a/lib/ingestors/concerns/sitemap_helpers.rb b/lib/ingestors/concerns/sitemap_helpers.rb index e9c6846c6..960f20e43 100644 --- a/lib/ingestors/concerns/sitemap_helpers.rb +++ b/lib/ingestors/concerns/sitemap_helpers.rb @@ -41,7 +41,7 @@ def parse_txt_sitemap(url) end def log_sitemap(ext, url, count) - @messages << "Parsing .#{ext} sitemap: #{url}\n - #{count} URLs found" + @messages << "Parsing .#{ext} sitemap: #{url}\n\n - #{count} URLs found" end end end diff --git a/lib/ingestors/oai_pmh_ingestor.rb b/lib/ingestors/oai_pmh_ingestor.rb index 9ce325e67..fff2426e1 100644 --- a/lib/ingestors/oai_pmh_ingestor.rb +++ b/lib/ingestors/oai_pmh_ingestor.rb @@ -94,11 +94,11 @@ def read_oai_rdf(client) end if totals.keys.any? - bioschemas_summary = "Bioschemas summary:\n" + bioschemas_summary = ["Bioschemas summary:\n"] totals.each do |type, count| - bioschemas_summary << "\n - #{type}: #{count}" + bioschemas_summary << " - #{type}: #{count}" end - @messages << bioschemas_summary + @messages << bioschemas_summary.join("\n") end @bioschemas_manager.deduplicate(provider_events).each do |event_params| diff --git a/lib/request_forgery_protection_token_verifier.rb b/lib/request_forgery_protection_token_verifier.rb deleted file mode 100644 index ed85c1ebe..000000000 --- a/lib/request_forgery_protection_token_verifier.rb +++ /dev/null @@ -1,35 +0,0 @@ -# Provides a callable method that verifies Cross-Site Request Forgery protection -# token. This class includes `ActionController::RequestForgeryProtection` -# directly and utilizes `verified_request?` method to match the way Rails -# performs token verification in Rails controllers. -# -# If you like to learn more about how Rails generate and verify authenticity -# token, you can find the source code at -# https://github.com/rails/rails/blob/v5.2.2/actionpack/lib/action_controller/metal/request_forgery_protection.rb#L217-L240. -class RequestForgeryProtectionTokenVerifier - include ActiveSupport::Configurable - include ActionController::RequestForgeryProtection - - # `ActionController::RequestForgeryProtection` contains a few configurable - # options. As we want to make sure that our configuration is the same as what - # being set in `ActionController::Base`, we should make all out configuration - # methods to delegate to `ActionController::Base`. - config.each_key do |configuration_name| - define_method configuration_name do - ActionController::Base.config[configuration_name] - end - end - - def call(env) - @request = ActionDispatch::Request.new(env) - - unless verified_request? - raise ActionController::InvalidAuthenticityToken - end - end - - private - - attr_reader :request - delegate :params, :session, to: :request -end diff --git a/lib/scraper.rb b/lib/scraper.rb index 3cd864884..b72594334 100644 --- a/lib/scraper.rb +++ b/lib/scraper.rb @@ -87,7 +87,7 @@ def run # --- finished log t('scraper.messages.status', status: 'finish'), 0 rescue StandardError => e - log " Run Scraper failed with: #{e.message}", 0 + log " Run Scraper failed with: #{e.message} (#{e.class.name})", 0 e.backtrace.each do |line| log " #{line}", 0 end @@ -104,7 +104,7 @@ def run def scrape(source, user = get_user, index: 0) source_start = Time.now - output = '' + output = [] log '', 1 if source.enabled log t('scraper.messages.processing', source: source.content_provider&.title, num: index.to_s), 1 @@ -114,9 +114,9 @@ def scrape(source, user = get_user, index: 0) end if validate_source(source) log t('scraper.messages.valid_source'), 2 - output.concat "**Provider:** #{source.content_provider.title}\n\n" - output.concat "**URL:** #{source.url}\n\n" - output.concat "**Method:** #{source.ingestor_title}\n\n" + output << "**Provider:** #{source.content_provider.title}" + output << "**URL:** #{source.url}" + output << "**Method:** #{source.ingestor_title}" # get ingestor ingestor = Ingestors::IngestorFactory.get_ingestor(source.method) @@ -127,16 +127,16 @@ def scrape(source, user = get_user, index: 0) # read records ingestor.read(source.url) unless ingestor.messages.blank? - output.concat "\n## Reading\n\n" - ingestor.messages.each { |m| output.concat "#{m}\n" } + output << "\n## Reading" + output += ingestor.messages ingestor.messages.clear end # write resources ingestor.write(user, source.content_provider, source: source) unless ingestor.messages.blank? - output.concat "\n## Writing\n\n" - ingestor.messages.each { |m| output.concat "#{m}\n" } + output << "\n## Writing" + output += ingestor.messages ingestor.messages.clear end @@ -152,14 +152,14 @@ def scrape(source, user = get_user, index: 0) ", rejected[#{source.resources_rejected}]", 2 end rescue StandardError => e - output.concat "\n**Failed:** #{e.message}\n\n" + output << "\n**Failed:** #{e.message}" log "Ingestor: #{ingestor.class} failed with: #{e.message}\t#{e.backtrace[0]}", 2 ensure source.finished_at = Time.now run_time = source.finished_at - source_start - output.concat "\n**Finished at:** #{source.finished_at.strftime '%H:%M on %A, %d %B %Y (UTC)'}\n" - output.concat "\n**Run time:** #{run_time.round(2)}s\n" - source.log = output + output << "\n**Finished at:** #{source.finished_at.strftime '%H:%M on %A, %d %B %Y (UTC)'}" + output << "**Run time:** #{run_time.round(2)}s" + source.log = output.join("\n\n") begin # only update enabled sources source.save! if source.enabled && !source.is_a?(Scraper::ConfigSource) diff --git a/test/controllers/workflows_controller_test.rb b/test/controllers/workflows_controller_test.rb index b9ef36777..297ccbf33 100644 --- a/test/controllers/workflows_controller_test.rb +++ b/test/controllers/workflows_controller_test.rb @@ -81,7 +81,7 @@ class WorkflowsControllerTest < ActionController::TestCase test "should show workflow" do get :show, params: { id: @workflow } assert_response :success - assert_includes response.headers.keys, 'X-Frame-Options', 'X-Frame-Options header should be present in all actions except `embed`' + assert_includes response.headers.keys.map(&:downcase), 'x-frame-options', 'X-Frame-Options header should be present in all actions except `embed`' assert assigns(:workflow) end @@ -98,7 +98,7 @@ class WorkflowsControllerTest < ActionController::TestCase get :embed, params: { id: @workflow } assert_response :success assert_select '.embedded-container', count: 1 - assert_not_includes response.headers.keys, 'X-Frame-Options', 'X-Frame-Options header should be removed to allow embedding in iframes' + assert_not_includes response.headers.keys.map(&:downcase), 'x-frame-options', 'X-Frame-Options header should be removed to allow embedding in iframes' end test "should not show embedded private workflow" do diff --git a/test/models/event_test.rb b/test/models/event_test.rb index 283874e02..fd25b901a 100644 --- a/test/models/event_test.rb +++ b/test/models/event_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'sidekiq/testing' class EventTest < ActiveSupport::TestCase setup do diff --git a/test/models/profile_test.rb b/test/models/profile_test.rb index c834fee04..901f07b4b 100644 --- a/test/models/profile_test.rb +++ b/test/models/profile_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'sidekiq/testing' class ProfileTest < ActiveSupport::TestCase setup do diff --git a/test/test_helper.rb b/test/test_helper.rb index 8d6583b3f..ed091a0a8 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -39,6 +39,8 @@ # config.allow_http_connections_when_no_cassette = true end +Sidekiq.testing!(:fake) + class ActiveSupport::TestCase include SchemaHelper diff --git a/test/unit/ingestors/scraper_test.rb b/test/unit/ingestors/scraper_test.rb index ad37c3fb1..5a45e23e0 100644 --- a/test/unit/ingestors/scraper_test.rb +++ b/test/unit/ingestors/scraper_test.rb @@ -225,10 +225,10 @@ def run scraper.run end - assert logfile_contains(logfile, 'Run Scraper failed with') - assert logfile_contains(logfile, 'in `block in run') - assert logfile_contains(logfile, 'in `map') - assert logfile_contains(logfile, 'in `run') + assert logfile_contains(logfile, 'Run Scraper failed with: unknown attribute') + assert logfile_contains(logfile, "in 'block in Scraper#run'") + assert logfile_contains(logfile, "in 'Array#map'") + assert logfile_contains(logfile, "in 'Scraper#run'") end test 'does not scrape disabled or unapproved sources' do diff --git a/test/workers/edit_suggestion_worker_test.rb b/test/workers/edit_suggestion_worker_test.rb index 8f4f8eaa5..bb238503a 100644 --- a/test/workers/edit_suggestion_worker_test.rb +++ b/test/workers/edit_suggestion_worker_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'sidekiq/testing' class EditSuggestionWorkerTest < ActiveSupport::TestCase diff --git a/test/workers/geocoding_worker_test.rb b/test/workers/geocoding_worker_test.rb index c21d4e789..c11948254 100644 --- a/test/workers/geocoding_worker_test.rb +++ b/test/workers/geocoding_worker_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'sidekiq/testing' class GeocodingWorkerTest < ActiveSupport::TestCase test 'get coordinates for an event' do diff --git a/test/workers/person_link_worker_test.rb b/test/workers/person_link_worker_test.rb index bc1ca0464..b0bac2419 100644 --- a/test/workers/person_link_worker_test.rb +++ b/test/workers/person_link_worker_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'sidekiq/testing' class PersonLinkWorkerTest < ActiveSupport::TestCase setup do diff --git a/test/workers/source_test_worker_test.rb b/test/workers/source_test_worker_test.rb index bba337594..078c3c3f4 100644 --- a/test/workers/source_test_worker_test.rb +++ b/test/workers/source_test_worker_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'sidekiq/testing' class SourceTestWorkerTest < ActiveSupport::TestCase test 'test source' do