diff --git a/Dockerfile b/Dockerfile index 4b01b084b..456256ca0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # !! This is a builder image. Not for general use !! # Use this as the base image for the Rails / Sidekiq services. ## -FROM ruby:2.7.8-slim-bullseye as spot-base +FROM ruby:2.7.8-slim-bullseye AS spot-base RUN apt-get clean && \ apt-get update && \ @@ -14,6 +14,7 @@ RUN apt-get clean && \ apt-get update -y && apt-get install -y --no-install-recommends \ awscli \ build-essential \ + clamav \ coreutils \ git \ libpq-dev \ @@ -59,7 +60,7 @@ HEALTHCHECK CMD curl -skf https://localhost/healthcheck/default || exit 1 # Target: spot-asset-builder # !! This is a builder image, do not use !! ## -FROM spot-base as spot-asset-builder +FROM spot-base AS spot-asset-builder ENV RAILS_ENV=production COPY . /spot @@ -69,7 +70,7 @@ RUN SECRET_KEY_BASE="$(bin/rake secret)" FEDORA_URL="http://fakehost:8080/rest" # TARGET: pdfjs-installer # !! This is a builder image. Not for general use !! ## -FROM alpine:3 as pdfjs-installer +FROM alpine:3 AS pdfjs-installer ARG PDFJS_VERSION="4.0.379" ENV PDFJS_VERSION="${PDFJS_VERSION}" ADD https://github.com/mozilla/pdf.js/releases/download/v${PDFJS_VERSION}/pdfjs-${PDFJS_VERSION}-legacy-dist.zip /tmp/pdfjs.zip @@ -83,7 +84,7 @@ COPY config/pdfjs/web/lafayette-viewer-enhancements.js /tmp/pdfjs/web # Used for the development version of the user-facing application. # Installs Ruby development dependencies ## -FROM spot-base as spot-web-development +FROM spot-base AS spot-web-development RUN bundle config unset with &&\ bundle config unset without && \ bundle config set with "development:test" && \ @@ -101,7 +102,7 @@ CMD ["bundle", "exec", "rails", "server", "-b", "ssl://0.0.0.0:443?key=/spot/tmp # TARGET: spot-web-production # Precompiles assets for production ## -FROM spot-base as spot-web-production +FROM spot-base AS spot-web-production ENV RAILS_ENV=production COPY . /spot COPY --from=spot-asset-builder /spot/public/assets /spot/public/assets @@ -116,7 +117,7 @@ COPY --from=pdfjs-installer /tmp/pdfjs /spot/public/pdf # @see https://github.com/harvard-lts/fits # @see https://github.com/samvera/hyrax/blob/3.x-stable/Dockerfile#L59-L65 ## -FROM alpine:3 as fits-installer +FROM alpine:3 AS fits-installer ARG FITS_VERSION="1.6.0" ENV FITS_VERSION="${FITS_VERSION}" ADD https://github.com/harvard-lts/fits/releases/download/${FITS_VERSION}/fits-${FITS_VERSION}.zip /tmp/fits.zip @@ -129,7 +130,7 @@ RUN unzip -d /tmp/fits /tmp/fits.zip && \ # TARGET: spot-worker # Installs dependencies for running background jobs ## -FROM spot-base as spot-worker-base +FROM spot-base AS spot-worker-base # @see https://github.com/mperham/sidekiq/wiki/Memory#bloat ENV MALLOC_ARENA_MAX=2 # We don't need the entrypoint script to generate an SSL cert @@ -159,7 +160,7 @@ ENV PATH="${PATH}:/usr/local/fits" CMD ["bundle", "exec", "sidekiq"] EXPOSE 3000 -FROM spot-worker-base as spot-worker-development +FROM spot-worker-base AS spot-worker-development ENV RAILS_ENV=development RUN bundle config unset with &&\ @@ -173,7 +174,7 @@ COPY . /spot/ # Target: spot-worker-production # Copies compiled assets for use in production. ## -FROM spot-worker-base as spot-worker-production +FROM spot-worker-base AS spot-worker-production ENV RAILS_ENV=production COPY . /spot/ diff --git a/Gemfile b/Gemfile index dff3ef7d3..d384f2546 100644 --- a/Gemfile +++ b/Gemfile @@ -55,6 +55,8 @@ gem 'bootsnap', '~> 1.17', require: false gem 'browse-everything', '~> 1.1.2' gem 'bulkrax', '~> 5.5.1' +gem 'clamby', '1.6.8' + # This needs to be here if we want to compile our own JS # (there's like a single coffee-script file still remaining in hyrax) gem 'coffee-rails', '~> 5.0.0' diff --git a/Gemfile.lock b/Gemfile.lock index d0bd9ea55..b86af0300 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -209,6 +209,7 @@ GEM activesupport (>= 4.0.0) mime-types (>= 1.16) ssrf_filter (~> 1.0, < 1.1.0) + clamby (1.6.8) clipboard-rails (1.7.1) coderay (1.1.3) coffee-rails (5.0.0) @@ -1112,6 +1113,7 @@ DEPENDENCIES byebug (~> 11.1.3) capybara (~> 3.38) capybara-screenshot (~> 1.0.26) + clamby (= 1.6.8) coffee-rails (~> 5.0.0) database_cleaner (~> 2.0.1) devise (~> 4.9.0) diff --git a/app/models/spot/virus_scanner.rb b/app/models/spot/virus_scanner.rb new file mode 100644 index 000000000..57fa7cb67 --- /dev/null +++ b/app/models/spot/virus_scanner.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +require 'clamby' + +module Spot + # The ClamAV gem, which Hyrax uses for virus scanning, has been dormant + # for ~13 years, so we're choosing to use the Clamby gem. + # + # @see https://github.com/samvera/hyrax/blob/hyrax-v3.6.0/app/models/hyrax/virus_scanner.rb + class VirusScanner < ::Hyrax::VirusScanner + def infected? + Clamby.virus?(file) + end + end +end diff --git a/app/views/records/edit_fields/_subject.html.erb b/app/views/records/edit_fields/_subject.html.erb index 2a481ba9c..8e6ecdc0a 100644 --- a/app/views/records/edit_fields/_subject.html.erb +++ b/app/views/records/edit_fields/_subject.html.erb @@ -5,14 +5,14 @@ input_html: { class: 'form-control', data: { - 'autocomplete-url' => '/authorities/search/linked_data/oclc_fast', + 'autocomplete-url' => '/authorities/search/assign_fast/all', 'autocomplete' => 'subject' } }, ### Required for the ControlledVocabulary javascript: wrapper_html: { data: { - 'autocomplete-url' => '/authorities/search/linked_data/oclc_fast', + 'autocomplete-url' => '/authorities/search/assign_fast/all', 'field-name' => 'subject' } }, diff --git a/config/authorities/remote_authorities.yml b/config/authorities/remote_authorities.yml index 0212528db..e226358ae 100644 --- a/config/authorities/remote_authorities.yml +++ b/config/authorities/remote_authorities.yml @@ -1,7 +1,7 @@ terms: - id: fast label: FAST - search: /authorities/search/linked_data/oclc_fast + search: /authorities/search/assign_fast/all - id: geonames label: GeoNames search: /authorities/search/geonames diff --git a/config/environments/test.rb b/config/environments/test.rb index 0cff60ba4..7c4718833 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -39,5 +39,4 @@ config.active_support.deprecation = :stderr config.active_job.queue_adapter = :test - config.exceptions_app = ->(env) { ErrorController.action(:show).call(env) } end diff --git a/config/initializers/hyrax.rb b/config/initializers/hyrax.rb index e0f6c6be0..4c1379044 100644 --- a/config/initializers/hyrax.rb +++ b/config/initializers/hyrax.rb @@ -307,6 +307,7 @@ ] config.branding_path = ENV.fetch('HYRAX_COLLECTION_BRANDING_PATH', Rails.root.join('public', 'branding')) + config.virus_scanner = Rails.env.test? ? nil : Spot::VirusScanner end Date::DATE_FORMATS[:standard] = "%m/%d/%Y" diff --git a/config/initializers/spot_overrides.rb b/config/initializers/spot_overrides.rb index 1dc4a6487..cafede025 100644 --- a/config/initializers/spot_overrides.rb +++ b/config/initializers/spot_overrides.rb @@ -231,4 +231,27 @@ def retrieve_cloud_files(files) nil end end + + # Modifying how Questiong Authority returns AssignFAST results by + # converting fst ids into URLs + require 'qa/authorities/assign_fast' + Qa::Authorities::AssignFast::GenericAuthority.class_eval do + private + + def parse_authority_response(raw_response) + raw_response['response']['docs'].map do |doc| + index = Qa::Authorities::AssignFast.index_for_authority(subauthority) + term = doc[index].first + term += " (USE #{doc['auth']})" if doc['type'] == 'alt' + fast_id = Array.wrap(doc['idroot']).first + + { + fast_id: fast_id, + id: "http://id.worldcat.org/fast/#{fast_id.gsub(/^fst/, '')}", + label: term, + value: doc['auth'] + } + end + end + end end diff --git a/docker-compose.yml b/docker-compose.yml index 80b34d9cc..3ca3c2217 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.9' services: app: image: lafayette/spot-web @@ -78,6 +77,13 @@ services: depends_on: - db + fits-servlet: + build: + context: ./docker/fits_servlet + ports: + - 8000:8080 + restart: unless-stopped + minio: image: minio/minio:latest environment: @@ -105,6 +111,8 @@ services: target: spot-worker-development env_file: - .env.local + environment: + - FITS_SERVLET_URL=http://fits-servlet:8080/fits ports: - "3003:3000" command: ["bundle", "exec", "sidekiq"] @@ -115,6 +123,7 @@ services: depends_on: - db - fedora + - fits-servlet - redis - solr diff --git a/docker/fits_servlet/Dockerfile b/docker/fits_servlet/Dockerfile new file mode 100644 index 000000000..340fd6d59 --- /dev/null +++ b/docker/fits_servlet/Dockerfile @@ -0,0 +1,79 @@ +## +# Target: fits-installer +# !! Not for general consumption !! +## +FROM alpine:3 AS fits-installer +ARG FITS_VERSION="1.6.0" +ENV FITS_VERSION="${FITS_VERSION}" +WORKDIR /tmp + +ADD https://github.com/harvard-lts/fits/releases/download/${FITS_VERSION}/fits-${FITS_VERSION}.zip /tmp + +RUN unzip -d /tmp/fits /tmp/fits-${FITS_VERSION}.zip && \ + chmod a+x /tmp/fits/fits.sh && \ + rm /tmp/fits-${FITS_VERSION}.zip + +# Remove bundled version of mediainfo in favor of system +# @see https://github.com/harvard-lts/fits?tab=readme-ov-file#system-requirements +RUN rm -r /tmp/fits/tools/mediainfo + + +## +# Target: servlet-installer +# !! Not for general consumption !! +## +FROM alpine:3 AS servlet-installer +ARG FITS_SERVLET_VERSION="2.1.0" +ENV FITS_SERVLET_VERSION="${FITS_SERVLET_VERSION}" +ADD https://github.com/harvard-lts/FITSservlet/releases/download/${FITS_SERVLET_VERSION}/fits-service-${FITS_SERVLET_VERSION}.war /tmp/fits-service.war + +## +# Default image +## +FROM docker.io/tomcat:9-jre17-temurin-jammy + +# see https://github.com/harvard-lts/FITSservlet/blob/2.1.0/docker/Dockerfile#L18-L42 +RUN apt-get update && \ + apt-get install -yqq \ + # jpylyzer dependencies + python3 \ + python-is-python3 \ + # exiftool dependencies https://github.com/exiftool/exiftool + libarchive-zip-perl \ + libio-compress-perl \ + libcompress-raw-zlib-perl \ + libcompress-bzip2-perl \ + libcompress-raw-bzip2-perl \ + libio-digest-perl \ + libdigest-md5-file-perl \ + libdigest-perl-md5-perl \ + libdigest-sha-perl \ + libposix-strptime-perl \ + libunicode-linebreak-perl\ + # file dependencies + file \ + make \ + gcc \ + # mediainfo dependencies + mediainfo \ + libmms0 \ + libcurl3-gnutls \ + && rm -rf /var/lib/apt/lists/* + +# Install FITS + servlet +COPY --from=fits-installer /tmp/fits /opt/fits +COPY --from=servlet-installer /tmp/fits-service.war $CATALINA_HOME/webapps/fits.war + +# Copy config, scripts, etc +COPY assets/catalina.properties $CATALINA_HOME/conf/catalina.properties +COPY assets/log4j2.xml $CATALINA_HOME/conf/log4j2.xml +COPY scripts/fits-servlet-entrypoint.sh /opt/fits-servlet-entrypoint.sh + +# Bind the current working directory here so that FITS can access files on the host +WORKDIR /work + +ENTRYPOINT ["/opt/fits-servlet-entrypoint.sh"] + +# need to explictly provide a CMD when adding a custom ENTRYPOINT +# @see https://github.com/docker-library/tomcat/blob/master/9.0/jre21/temurin-jammy/Dockerfile#L47 +CMD ["catalina.sh", "run"] \ No newline at end of file diff --git a/docker/fits_servlet/assets/catalina.properties b/docker/fits_servlet/assets/catalina.properties new file mode 100644 index 000000000..fa990c3a0 --- /dev/null +++ b/docker/fits_servlet/assets/catalina.properties @@ -0,0 +1,211 @@ + Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# List of comma-separated packages that start with or equal this string +# will cause a security exception to be thrown when +# passed to checkPackageAccess unless the +# corresponding RuntimePermission ("accessClassInPackage."+package) has +# been granted. +package.access=sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,org.apache.tomcat. +# +# List of comma-separated packages that start with or equal this string +# will cause a security exception to be thrown when +# passed to checkPackageDefinition unless the +# corresponding RuntimePermission ("defineClassInPackage."+package) has +# been granted. +# +# by default, no packages are restricted for definition, and none of +# the class loaders supplied with the JDK call checkPackageDefinition. +# +package.definition=sun.,java.,org.apache.catalina.,org.apache.coyote.,\ +org.apache.jasper.,org.apache.naming.,org.apache.tomcat. + +# +# +# List of comma-separated paths defining the contents of the "common" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute. +# If left as blank,the JVM system loader will be used as Catalina's "common" +# loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +# +# Note: Values are enclosed in double quotes ("...") in case either the +# ${catalina.base} path or the ${catalina.home} path contains a comma. +# Because double quotes are used for quoting, the double quote character +# may not appear in a path. +common.loader="${catalina.base}/lib","${catalina.base}/lib/*.jar","${catalina.home}/lib","${catalina.home}/lib/*.jar" + +# +# List of comma-separated paths defining the contents of the "server" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute. +# If left as blank, the "common" loader will be used as Catalina's "server" +# loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +# +# Note: Values may be enclosed in double quotes ("...") in case either the +# ${catalina.base} path or the ${catalina.home} path contains a comma. +# Because double quotes are used for quoting, the double quote character +# may not appear in a path. +server.loader= + +fits.home=/opt/fits +# +# List of comma-separated paths defining the contents of the "shared" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_BASE path or absolute. If left as blank, +# the "common" loader will be used as Catalina's "shared" loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +# Please note that for single jars, e.g. bar.jar, you need the URL form +# starting with file:. +# +# Note: Values may be enclosed in double quotes ("...") in case either the +# ${catalina.base} path or the ${catalina.home} path contains a comma. +# Because double quotes are used for quoting, the double quote character +# may not appear in a path. +shared.loader=${fits.home}/lib/*.jar + +# Default list of JAR files that should not be scanned using the JarScanner +# functionality. This is typically used to scan JARs for configuration +# information. JARs that do not contain such information may be excluded from +# the scan to speed up the scanning process. This is the default list. JARs on +# this list are excluded from all scans. The list must be a comma separated list +# of JAR file names. +# The list of JARs to skip may be over-ridden at a Context level for individual +# scan types by configuring a JarScanner with a nested JarScanFilter. +# The JARs listed below include: +# - Tomcat Bootstrap JARs +# - Tomcat API JARs +# - Catalina JARs +# - Jasper JARs +# - Tomcat JARs +# - Common non-Tomcat JARs +# - Test JARs (JUnit, Cobertura and dependencies) +tomcat.util.scan.StandardJarScanFilter.jarsToSkip=\ +annotations-api.jar,\ +ant-junit*.jar,\ +ant-launcher.jar,\ +ant.jar,\ +asm-*.jar,\ +aspectj*.jar,\ +bootstrap.jar,\ +catalina-ant.jar,\ +catalina-ha.jar,\ +catalina-ssi.jar,\ +catalina-storeconfig.jar,\ +catalina-tribes.jar,\ +catalina.jar,\ +cglib-*.jar,\ +cobertura-*.jar,\ +commons-beanutils*.jar,\ +commons-codec*.jar,\ +commons-collections*.jar,\ +commons-daemon.jar,\ +commons-dbcp*.jar,\ +commons-digester*.jar,\ +commons-fileupload*.jar,\ +commons-httpclient*.jar,\ +commons-io*.jar,\ +commons-lang*.jar,\ +commons-logging*.jar,\ +commons-math*.jar,\ +commons-pool*.jar,\ +derby-*.jar,\ +dom4j-*.jar,\ +easymock-*.jar,\ +ecj-*.jar,\ +el-api.jar,\ +geronimo-spec-jaxrpc*.jar,\ +h2*.jar,\ +ha-api-*.jar,\ +hamcrest-*.jar,\ +hibernate*.jar,\ +httpclient*.jar,\ +icu4j-*.jar,\ +jasper-el.jar,\ +jasper.jar,\ +jaspic-api.jar,\ +jaxb-*.jar,\ +jaxen-*.jar,\ +jaxws-rt-*.jar,\ +jdom-*.jar,\ +jetty-*.jar,\ +jmx-tools.jar,\ +jmx.jar,\ +jsp-api.jar,\ +jstl.jar,\ +jta*.jar,\ +junit-*.jar,\ +junit.jar,\ +log4j*.jar,\ +mail*.jar,\ +objenesis-*.jar,\ +oraclepki.jar,\ +oro-*.jar,\ +servlet-api-*.jar,\ +servlet-api.jar,\ +slf4j*.jar,\ +taglibs-standard-spec-*.jar,\ +tagsoup-*.jar,\ +tomcat-api.jar,\ +tomcat-coyote.jar,\ +tomcat-dbcp.jar,\ +tomcat-i18n-*.jar,\ +tomcat-jdbc.jar,\ +tomcat-jni.jar,\ +tomcat-juli-adapters.jar,\ +tomcat-juli.jar,\ +tomcat-util-scan.jar,\ +tomcat-util.jar,\ +tomcat-websocket.jar,\ +tools.jar,\ +websocket-api.jar,\ +wsdl4j*.jar,\ +xercesImpl.jar,\ +xml-apis.jar,\ +xmlParserAPIs-*.jar,\ +xmlParserAPIs.jar,\ +xom-*.jar + +# Default list of JAR files that should be scanned that overrides the default +# jarsToSkip list above. This is typically used to include a specific JAR that +# has been excluded by a broad file name pattern in the jarsToSkip list. +# The list of JARs to scan may be over-ridden at a Context level for individual +# scan types by configuring a JarScanner with a nested JarScanFilter. +tomcat.util.scan.StandardJarScanFilter.jarsToScan=\ +log4j-taglib*.jar,\ +log4j-web*.jar,\ +log4javascript*.jar,\ +slf4j-taglib*.jar + +# String cache configuration. +tomcat.util.buf.StringCache.byte.enabled=true +#tomcat.util.buf.StringCache.char.enabled=true +#tomcat.util.buf.StringCache.trainThreshold=500000 +#tomcat.util.buf.StringCache.cacheSize=5000 +FITS_SERVICE_PROPS=${catalina.home}/conf/fits-service.properties \ No newline at end of file diff --git a/docker/fits_servlet/assets/log4j2.xml b/docker/fits_servlet/assets/log4j2.xml new file mode 100644 index 000000000..48fb9668b --- /dev/null +++ b/docker/fits_servlet/assets/log4j2.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker/fits_servlet/scripts/fits-servlet-entrypoint.sh b/docker/fits_servlet/scripts/fits-servlet-entrypoint.sh new file mode 100755 index 000000000..845a4406a --- /dev/null +++ b/docker/fits_servlet/scripts/fits-servlet-entrypoint.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +# generate fits-service properties file from environment variables +cat <<-EOPROPS > $CATALINA_HOME/conf/fits-service.properties +max.objects.in.pool=${MAX_OBJECTS_IN_POOL:-5} +max.upload.file.size.MB=${MAX_UPLOAD_FILE_SIZE_MB:-2000} +max.request.size.MB=${MAX_REQUEST_SIZE_MB:-2000} +max.in.memory.file.size.MB=${MAX_IN_MEMORY_FILE_SIZE_MB:-4} +EOPROPS + +export CATALINA_OPTS="${CATALINA_OPTS} -Dlog4j2.configurationFile=${CATALINA_HOME}/conf/log4j2.xml" +exec "$@" \ No newline at end of file diff --git a/spec/features/create_image_spec.rb b/spec/features/create_image_spec.rb index b500437e8..20294bafd 100644 --- a/spec/features/create_image_spec.rb +++ b/spec/features/create_image_spec.rb @@ -85,7 +85,7 @@ fill_in_autocomplete '.image_subject', with: attrs[:subject].first expect(page).to have_css('.image_subject.form-control[data-autocomplete="subject"]', visible: false) - expect(page).to have_css('.image_subject.form-control[data-autocomplete-url="/authorities/search/linked_data/oclc_fast"]', visible: false) + expect(page).to have_css('.image_subject.form-control[data-autocomplete-url="/authorities/search/assign_fast/all"]', visible: false) expect(page).to have_css('.image_subject .controls-add-text') # multi-authority for location diff --git a/spec/models/spot/virus_scanner_spec.rb b/spec/models/spot/virus_scanner_spec.rb new file mode 100644 index 000000000..ca2f6181e --- /dev/null +++ b/spec/models/spot/virus_scanner_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +RSpec.describe Spot::VirusScanner do + describe '.infected?' do + subject { described_class.infected?(file_path) } + + let(:file_path) { '/path/to/a/file' } + + before do + allow(Clamby).to receive(:virus?).with(file_path).and_return(verdict) + end + + context 'when file_path has a virus' do + let(:verdict) { true } + + it { is_expected.to be true } + end + + context 'when file_path is clean' do + let(:verdict) { false } + + it { is_expected.to be false } + end + end +end