diff --git a/.github/workflows/ruby-unit-test.yml b/.github/workflows/ruby-unit-test.yml index 7c5c6153..85233f9c 100644 --- a/.github/workflows/ruby-unit-test.yml +++ b/.github/workflows/ruby-unit-test.yml @@ -1,6 +1,12 @@ name: Ruby Unit Test -on: [push, pull_request] +on: + push: + branches: + - '**' + tags-ignore: + - '**' # ignore all tag pushes + pull_request: permissions: contents: read @@ -13,11 +19,11 @@ jobs: fail-fast: false matrix: goo-slice: [ '100'] - ruby-version: [ '3.2.0' ] - triplestore: [ 'fs', 'ag', 'vo', 'gb' ] + ruby-version: [ '3.2' ] + triplestore: [ 'fs', 'ag', 'vo', 'gd' ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Dependencies run: sudo apt-get update && sudo apt-get -y install raptor2-utils - name: Set up Ruby @@ -29,8 +35,8 @@ jobs: # tempoaray workaround for the config.rb file requirement run: echo 'Goo.config do |config| end' > config/config.rb - name: List directory contents - run: ls -R ./test/data + run: ls -R ./test/fixtures - name: Run tests run: GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v" - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1dd2150b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +ARG RUBY_VERSION=3.2 +ARG DISTRO=bullseye + +FROM ruby:$RUBY_VERSION-$DISTRO + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + git \ + libxml2 \ + libxslt-dev \ + libxslt1-dev zlib1g-dev \ + # openjdk-11-jre-headless \ + raptor2-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY Gemfile* *.gemspec ./ + +#Install the exact Bundler version from Gemfile.lock (if it exists) +RUN gem update --system && \ + if [ -f Gemfile.lock ]; then \ + BUNDLER_VERSION=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1 | tr -d ' '); \ + gem install bundler -v "$BUNDLER_VERSION"; \ + else \ + gem install bundler; \ + fi + +RUN bundle config set --global no-document 'true' +RUN bundle install --jobs 4 --retry 3 + +COPY . ./ + +CMD ["bundle", "exec", "rake"] diff --git a/README.md b/README.md index b1553b79..3ea16b6e 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,42 @@ To see Goo in action browse to the following links: - [BioPortal New API Documentation](http://stagedata.bioontology.org/documentation) - [BioPortal New API Access](http://stagedata.bioontology.org/) +## Tests + +Run the unit tests: + +``` +bundle exec rake test +``` + +Run docker-backed tests (host Ruby): + +``` +bundle exec rake test:docker:fs +bundle exec rake test:docker:ag +bundle exec rake test:docker:vo +bundle exec rake test:docker:gd +``` + +Run docker-backed tests inside the Linux container: + +``` +bundle exec rake test:docker:fs:linux +bundle exec rake test:docker:ag:linux +bundle exec rake test:docker:vo:linux +bundle exec rake test:docker:gd:linux +``` + +Start a shell in the Linux test container (default backend `fs`): + +``` +bundle exec rake test:docker:shell +bundle exec rake test:docker:shell[ag] +``` + +Set `OP_KEEP_CONTAINERS=1` to keep services up after tests or shell exit. +Set `OP_TEST_DOCKER_BACKEND=ag` (or `fs`, `vo`, `gd`) to change the default backend for `test:docker:shell`, `test:docker:up`, and `test:docker:down`. + ## Schema Definitions (DSL) diff --git a/config/config.rb.sample b/config/config.rb.sample index 0429e9cd..edac79df 100644 --- a/config/config.rb.sample +++ b/config/config.rb.sample @@ -1,14 +1,14 @@ Goo.config do |config| # 4store config.goo_backend_name = '4store' - config.goo_port = 8080 + config.goo_port = 9000 config.goo_host = 'localhost' config.goo_path_query = '/sparql/' config.goo_path_data = '/data/' config.goo_path_update = '/update/' # AllegroGraph - # config.goo_backend_name = 'AG' + # config.goo_backend_name = 'allegrograph' # config.goo_port = 10035 # config.goo_host = 'localhost' # config.goo_path_query = "/repositories/ontoportal" @@ -16,8 +16,8 @@ Goo.config do |config| # config.goo_path_update = "/repositories/ontoportal/statements/" config.search_server_url = 'http://localhost:8983/solr/' - config.redis_host = 'localhost' - config.redis_port = 6379 + config.goo_redis_host = 'localhost' + config.goo_redis_port = 6379 config.bioportal_namespace = 'http://data.bioontology.org/' config.queries_debug = false end diff --git a/dev/compose/linux/ag.yml b/dev/compose/linux/ag.yml new file mode 100644 index 00000000..b56d9c8e --- /dev/null +++ b/dev/compose/linux/ag.yml @@ -0,0 +1,16 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: allegrograph + GOO_PORT: 10035 + GOO_HOST: agraph-ut + GOO_PATH_QUERY: /repositories/ontoportal_test + GOO_PATH_DATA: /repositories/ontoportal_test/statements + GOO_PATH_UPDATE: /repositories/ontoportal_test/statements + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + agraph-ut: + condition: service_healthy diff --git a/dev/compose/linux/fs.yml b/dev/compose/linux/fs.yml new file mode 100644 index 00000000..27acf4b3 --- /dev/null +++ b/dev/compose/linux/fs.yml @@ -0,0 +1,13 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: '4store' + GOO_HOST: 4store-ut + GOO_PORT: 9000 + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + 4store-ut: + condition: service_healthy diff --git a/dev/compose/linux/gd.yml b/dev/compose/linux/gd.yml new file mode 100644 index 00000000..f83a32f6 --- /dev/null +++ b/dev/compose/linux/gd.yml @@ -0,0 +1,19 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: graphdb + GOO_PORT: 7200 + GOO_HOST: graphdb-ut + GOO_PATH_QUERY: /repositories/ontoportal_test + GOO_PATH_DATA: /repositories/ontoportal_test/statements + GOO_PATH_UPDATE: /repositories/ontoportal_test/statements + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + graphdb-ut: + condition: service_healthy + + graphdb-ut: + ports: diff --git a/dev/compose/linux/no-ports.yml b/dev/compose/linux/no-ports.yml new file mode 100644 index 00000000..f42191b9 --- /dev/null +++ b/dev/compose/linux/no-ports.yml @@ -0,0 +1,13 @@ +services: + redis-ut: + ports: [] + solr-ut: + ports: [] + agraph-ut: + ports: [] + 4store-ut: + ports: [] + virtuoso-ut: + ports: [] + graphdb-ut: + ports: [] diff --git a/dev/compose/linux/vo.yml b/dev/compose/linux/vo.yml new file mode 100644 index 00000000..c47dd654 --- /dev/null +++ b/dev/compose/linux/vo.yml @@ -0,0 +1,16 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: 'virtuoso' + GOO_HOST: virtuoso-ut + GOO_PORT: 8890 + GOO_PATH_QUERY: /sparql + GOO_PATH_DATA: /sparql + GOO_PATH_UPDATE: /sparql + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + virtuoso-ut: + condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 4ddd6807..2b1ae3b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,75 +1,110 @@ +# unit tests in containerased env services: + test-linux: + build: + context: . + args: + RUBY_VERSION: '3.2' + command: ["bash", "-lc", "bundle exec rake test"] + environment: + COVERAGE: 'true' # enable simplecov code coverage + REDIS_HOST: redis-ut + SEARCH_SERVER_URL: http://solr-ut:8983/solr + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + profiles: + - linux + redis-ut: image: redis ports: - 6379:6379 + command: ["redis-server", "--save", "", "--appendonly", "no"] healthcheck: test: redis-cli ping - interval: 1s + interval: 10s timeout: 3s - retries: 30 + retries: 10 solr-ut: - image: solr:8.11.2 + image: solr:9 + command: bin/solr start -cloud -f ports: - 8983:8983 - command: bin/solr start -cloud -f + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8983/solr/admin/info/system?wt=json"] + start_period: 5s + interval: 10s + timeout: 5s + retries: 5 - agraph-ut: - image: franzinc/agraph:v8.3.1 + 4store-ut: + image: bde2020/4store platform: linux/amd64 + ports: + - 9000:9000 + command: > + bash -c "4s-backend-setup --segments 4 ontoportal_test + && 4s-backend ontoportal_test + && 4s-httpd -D -s-1 -p 9000 ontoportal_test" + healthcheck: + test: ["CMD", "4s-backend-info", "ontoportal_test"] + start_period: 5s + interval: 10s + timeout: 10s + retries: 5 + profiles: + - fs + + agraph-ut: + image: franzinc/agraph:v8.4.3 + platform: linux/amd64 #agraph doesn't provide arm platform environment: - AGRAPH_SUPER_USER=test - AGRAPH_SUPER_PASSWORD=xyzzy shm_size: 1g ports: - # - 10035:10035 - - 10000-10035:10000-10035 - volumes: - - agdata:/agraph/data - # - ./agraph/etc:/agraph/etc + - 10035:10035 command: > - bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start - ; agtool repos create ontoportal_test --supersede - ; agtool users add anonymous - ; agtool users grant anonymous root:ontoportal_test:rw - ; tail -f /agraph/data/agraph.log" - # healthcheck: - # test: ["CMD-SHELL", "curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"] - # start_period: 10s - # interval: 10s - # timeout: 5s - # retries: 5 + bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start + ; agtool repos create --supersede ontoportal_test + ; agtool users add anonymous + ; agtool users grant anonymous root:ontoportal_test:rw + ; tail -f /agraph/data/agraph.log" + healthcheck: + test: ["CMD", "agtool", "storage-report", "ontoportal_test"] + start_period: 30s #AllegroGraph can take a loooooong time to start + interval: 20s + timeout: 10s + retries: 20 profiles: - ag - 4store-ut: - image: bde2020/4store - platform: linux/amd64 - ports: - - 9000:9000 - command: > - bash -c "4s-backend-setup --segments 4 ontoportal_kb - && 4s-backend ontoportal_kb - && 4s-httpd -D -s-1 -p 9000 ontoportal_kb" - profiles: - - fs virtuoso-ut: - image: tenforce/virtuoso:virtuoso7.2.5 - platform: linux/amd64 + image: openlink/virtuoso-opensource-7:7.2.16 environment: - SPARQL_UPDATE=true + - DBA_PASSWORD=dba + - DAV_PASSWORD=dba ports: - 1111:1111 - 8890:8890 - + volumes: + - ./test/fixtures/backends/virtuoso_initdb_d:/initdb.d + healthcheck: + test: [ "CMD-SHELL", "echo 'status();' | isql localhost:1111 dba dba || exit 1" ] + start_period: 10s + interval: 10s + timeout: 5s + retries: 3 profiles: - vo - graphdb: - image: ontotext/graphdb:10.3.3 - platform: linux/amd64 - privileged: true + graphdb-ut: + image: ontotext/graphdb:10.8.12 environment: GDB_HEAP_SIZE: 5G GDB_JAVA_OPTS: >- @@ -77,18 +112,14 @@ services: ports: - 7200:7200 - 7300:7300 + healthcheck: + test: [ "CMD", "curl", "-sf", "http://localhost:7200/repositories/ontoportal_test/health" ] + start_period: 10s + interval: 10s volumes: - - ./test/data/graphdb-repo-config.ttl:/opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl - - ./test/data/graphdb-test-load.nt:/opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt - + - ./test/fixtures/backends/graphdb:/opt/graphdb/dist/configs/templates/data entrypoint: > - bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt ; graphdb -Ddefault.min.distinct.threshold=3000 " + bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt + ; graphdb -Ddefault.min.distinct.threshold=3000 " profiles: - - gb - -volumes: - agdata: - - - - + - gd diff --git a/lib/goo/config/config.rb b/lib/goo/config/config.rb index d842b152..37b59e01 100644 --- a/lib/goo/config/config.rb +++ b/lib/goo/config/config.rb @@ -21,8 +21,8 @@ def config(&block) @settings.goo_path_data ||= ENV['GOO_PATH_DATA'] || '/data/' @settings.goo_path_update ||= ENV['GOO_PATH_UPDATE'] || '/update/' @settings.search_server_url ||= ENV['SEARCH_SERVER_URL'] || 'http://localhost:8983/solr' - @settings.redis_host ||= ENV['REDIS_HOST'] || 'localhost' - @settings.redis_port ||= ENV['REDIS_PORT'] || 6379 + @settings.goo_redis_host ||= ENV['REDIS_HOST'] || 'localhost' + @settings.goo_redis_port ||= ENV['REDIS_PORT'] || 6379 @settings.bioportal_namespace ||= ENV['BIOPORTAL_NAMESPACE'] || 'http://data.bioontology.org/' @settings.query_logging ||= ENV['QUERIES_LOGGING'] || false @settings.query_logging_file ||= ENV['QUERIES_LOGGING_FILE'] || './sparql.log' @@ -30,7 +30,7 @@ def config(&block) @settings.slice_loading_size ||= ENV['GOO_SLICES']&.to_i || 500 puts "(GOO) >> Using RDF store (#{@settings.goo_backend_name}) #{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}" puts "(GOO) >> Using term search server at #{@settings.search_server_url}" - puts "(GOO) >> Using Redis instance at #{@settings.redis_host}:#{@settings.redis_port}" + puts "(GOO) >> Using Redis instance at #{@settings.goo_redis_host}:#{@settings.goo_redis_port}" puts "(GOO) >> Using Query logging: #{@settings.query_logging_file}" if @settings.query_logging connect_goo @@ -69,17 +69,4 @@ def connect_goo end end - def self.test_reset - if @@sparql_backends[:main][:query].url.to_s["localhost"].nil? - raise Exception, "only for testing" - end - @@sparql_backends = {} - Goo.add_sparql_backend(:main, - backend_name: @settings.goo_backend_name, - query: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}", - data: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_data}", - update: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_update}", - options: { rules: :NONE }) - end - end diff --git a/rakelib/docker_based_test.rake b/rakelib/docker_based_test.rake index c84879a9..c3cd32aa 100644 --- a/rakelib/docker_based_test.rake +++ b/rakelib/docker_based_test.rake @@ -1,121 +1,240 @@ -# Rake tasks for running unit tests with backend services running as docker containers - -desc 'Run unit tests with docker based backend' +# Docker compose driven unit test orchestration +# +# Notes: +# - Backend names match compose profile names (ag, fs, vo, gd). +# - Hostnames are NOT set here. The app defaults them (localhost for host runs). +# - Linux container env is provided via compose override files: +# dev/compose/linux/ag.yml +# dev/compose/linux/fs.yml +# dev/compose/linux/vo.yml +# dev/compose/linux/gd.yml namespace :test do namespace :docker do - task :up do - system("docker compose up -d") || abort("Unable to start docker containers") - unless system("curl -sf http://localhost:8983/solr || exit 1") - printf("waiting for Solr container to initialize") - sec = 0 - until system("curl -sf http://localhost:8983/solr || exit 1") do - sleep(1) - printf(".") - sec += 1 - if sec > 30 - abort(" Solr container hasn't initialized properly") - end - end - printf("\n") - end + BASE_COMPOSE = 'docker-compose.yml' + LINUX_OVERRIDE_DIR = 'dev/compose/linux' + LINUX_NO_PORTS_OVERRIDE = "#{LINUX_OVERRIDE_DIR}/no-ports.yml" + TIMEOUT = (ENV['OP_TEST_DOCKER_TIMEOUT'] || '600').to_i + DEFAULT_BACKEND = (ENV['OP_TEST_DOCKER_BACKEND'] || 'fs').to_sym + + # Minimal per-backend config for host runs only. + # Do not set hostnames here. The app defaults them. + BACKENDS = { + ag: { + host_env: { + 'GOO_BACKEND_NAME' => 'allegrograph', + 'GOO_PORT' => '10035', + 'GOO_PATH_QUERY' => '/repositories/ontoportal_test', + 'GOO_PATH_DATA' => '/repositories/ontoportal_test/statements', + 'GOO_PATH_UPDATE' => '/repositories/ontoportal_test/statements' + } + }, + fs: { + host_env: { + 'GOO_BACKEND_NAME' => '4store', + 'GOO_PORT' => '9000', + 'GOO_PATH_QUERY' => '/sparql/', + 'GOO_PATH_DATA' => '/data/', + 'GOO_PATH_UPDATE' => '/update/' + } + }, + vo: { + host_env: { + 'GOO_BACKEND_NAME' => 'virtuoso', + 'GOO_PORT' => '8890', + 'GOO_PATH_QUERY' => '/sparql', + 'GOO_PATH_DATA' => '/sparql', + 'GOO_PATH_UPDATE' => '/sparql' + } + }, + gd: { + host_env: { + 'GOO_BACKEND_NAME' => 'graphdb', + 'GOO_PORT' => '7200', + 'GOO_PATH_QUERY' => '/repositories/ontoportal_test', + 'GOO_PATH_DATA' => '/repositories/ontoportal_test/statements', + 'GOO_PATH_UPDATE' => '/repositories/ontoportal_test/statements' + } + } + }.freeze + + def abort_with(msg) + warn(msg) + exit(1) + end + + def shell!(cmd) + system(cmd) || abort_with("Command failed: #{cmd}") + end + + def cfg!(key) + cfg = BACKENDS[key] + abort_with("Unknown backend key: #{key}. Supported: #{BACKENDS.keys.join(', ')}") unless cfg + cfg + end + + def compose_files(*files) + files.flatten.map { |f| "-f #{f}" }.join(' ') + end + + def linux_override_for(key) + "#{LINUX_OVERRIDE_DIR}/#{key}.yml" + end + + def compose_up(key, files:) + # Host tests use only the backend profile. Linux tests add the linux profile. + # `docker compose up --wait` only applies to services started by `up`, + # so linux runs still call `run` separately after this wait completes. + shell!("docker compose #{compose_files(files)} --profile #{key} up -d --wait --wait-timeout #{TIMEOUT}") + end + + def compose_down(files:) + return puts('OP_KEEP_CONTAINERS=1 set, skipping docker compose down') if ENV['OP_KEEP_CONTAINERS'] == '1' + + shell!( + "docker compose #{compose_files(files)} " \ + '--profile ag --profile fs --profile vo --profile gd --profile linux down' + ) + end + def apply_host_env(key) + cfg!(key)[:host_env].each { |k, v| ENV[k] = v } end - task :down do - #system("docker compose --profile fs --profile ag stop") - #system("docker compose --profile fs --profile ag kill") + + def run_host_tests(key) + apply_host_env(key) + files = [BASE_COMPOSE] + + compose_up(key, files: files) + Rake::Task['test'].invoke + end + + def run_linux_tests(key) + override = linux_override_for(key) + abort_with("Missing compose override file: #{override}") unless File.exist?(override) + abort_with("Missing compose override file: #{LINUX_NO_PORTS_OVERRIDE}") unless File.exist?(LINUX_NO_PORTS_OVERRIDE) + + files = [BASE_COMPOSE, override, LINUX_NO_PORTS_OVERRIDE] + # docker compose is handleling wait_for_healthy + compose_up(key, files: files) + + shell!( + "docker compose #{compose_files(files)} --profile linux --profile #{key} " \ + 'run --rm --build test-linux bundle exec rake test TESTOPTS="-v"' + ) end - desc "run tests with docker AG backend" + + def run_linux_shell(key) + override = linux_override_for(key) + abort_with("Missing compose override file: #{override}") unless File.exist?(override) + abort_with("Missing compose override file: #{LINUX_NO_PORTS_OVERRIDE}") unless File.exist?(LINUX_NO_PORTS_OVERRIDE) + + files = [BASE_COMPOSE, override, LINUX_NO_PORTS_OVERRIDE] + compose_up(key, files: files) + + shell!( + "docker compose #{compose_files(files)} --profile linux --profile #{key} " \ + 'run --rm --build test-linux bash' + ) + end + + # + # Public tasks + # + + desc 'Run unit tests with AllegroGraph backend (docker deps, host Ruby)' task :ag do - ENV["GOO_BACKEND_NAME"]="allegrograph" - ENV["GOO_PORT"]="10035" - ENV["GOO_PATH_QUERY"]="/repositories/ontoportal_test" - ENV["GOO_PATH_DATA"]="/repositories/ontoportal_test/statements" - ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal_test/statements" - ENV["COMPOSE_PROFILES"]="ag" - Rake::Task["test:docker:up"].invoke - # AG takes some time to start and create databases/accounts - # TODO: replace system curl command with native ruby code - unless system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") - printf("waiting for AllegroGraph container to initialize") - sec = 0 - until system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") do - sleep(1) - printf(".") - sec += 1 - end + run_host_tests(:ag) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with AllegroGraph backend (docker deps, Linux container)' + task 'ag:linux' do + files = [BASE_COMPOSE, linux_override_for(:ag)] + begin + run_linux_tests(:ag) + ensure + compose_down(files: files) end - puts - system("docker compose ps") # TODO: remove after GH actions troubleshooting is complete - Rake::Task["test"].invoke - Rake::Task["test:docker:down"].invoke end - desc "run tests with docker 4store backend" + desc 'Run unit tests with 4store backend (docker deps, host Ruby)' task :fs do - ENV["GOO_PORT"]="9000" - ENV["COMPOSE_PROFILES"]='fs' - Rake::Task["test:docker:up"].invoke - Rake::Task["test"].invoke - Rake::Task["test:docker:down"].invoke + run_host_tests(:fs) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with 4store backend (docker deps, Linux container)' + task 'fs:linux' do + files = [BASE_COMPOSE, linux_override_for(:fs)] + begin + run_linux_tests(:fs) + ensure + compose_down(files: files) + end end - desc "run tests with docker Virtuoso backend" + desc 'Run unit tests with Virtuoso backend (docker deps, host Ruby)' task :vo do - ENV["GOO_BACKEND_NAME"]="virtuoso" - ENV["GOO_PORT"]="8890" - ENV["GOO_PATH_QUERY"]="/sparql" - ENV["GOO_PATH_DATA"]="/sparql" - ENV["GOO_PATH_UPDATE"]="/sparql" - ENV["COMPOSE_PROFILES"]="vo" - Rake::Task["test:docker:up"].invoke - # - unless system("curl -sf http://localhost:8890/sparql || exit 1") - printf("waiting for Virtuoso container to initialize") - sec = 0 - until system("curl -sf http://localhost:8890/sparql || exit 1") do - sleep(1) - printf(".") - sec += 1 - if sec > 30 - system("docker compose logs virtuoso-ut") - abort(" Virtuoso container hasn't initialized properly") - end - end + run_host_tests(:vo) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with Virtuoso backend (docker deps, Linux container)' + task 'vo:linux' do + files = [BASE_COMPOSE, linux_override_for(:vo)] + begin + run_linux_tests(:vo) + ensure + compose_down(files: files) end - Rake::Task["test"].invoke - Rake::Task["test:docker:down"].invoke - end - - - desc "run tests with docker GraphDb backend" - task :gb do - ENV["GOO_BACKEND_NAME"]="graphdb" - ENV["GOO_PORT"]="7200" - ENV["GOO_PATH_QUERY"]="/repositories/ontoportal" - ENV["GOO_PATH_DATA"]="/repositories/ontoportal/statements" - ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal/statements" - ENV["COMPOSE_PROFILES"]="gb" - Rake::Task["test:docker:up"].invoke - - #system("docker compose cp ./test/data/graphdb-repo-config.ttl graphdb:/opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl") - #system("docker compose cp ./test/data/graphdb-test-load.nt graphdb:/opt/graphdb/dist/configs/templates/graphdb-test-load.nt") - #system('docker compose exec graphdb sh -c "importrdf load -f -c /opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/graphdb-test-load.nt ;"') - unless system("curl -sf http://localhost:7200/repositories || exit 1") - printf("waiting for Graphdb container to initialize") - sec = 0 - until system("curl -sf http://localhost:7200/repositories || exit 1") do - sleep(1) - printf(".") - sec += 1 - if sec > 30 - system("docker compose logs graphdb") - abort(" Graphdb container hasn't initialized properly") - end - end + end + + desc 'Run unit tests with GraphDB backend (docker deps, host Ruby)' + task :gd do + run_host_tests(:gd) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with GraphDB backend (docker deps, Linux container)' + task 'gd:linux' do + files = [BASE_COMPOSE, linux_override_for(:gd)] + begin + run_linux_tests(:gd) + ensure + compose_down(files: files) end - Rake::Task["test"].invoke - Rake::Task["test:docker:down"].invoke end + desc 'Start a shell in the Linux test container (default backend: fs)' + task :shell, [:backend] do |_t, args| + key = (args[:backend] || DEFAULT_BACKEND).to_sym + cfg!(key) + files = [BASE_COMPOSE, linux_override_for(key), LINUX_NO_PORTS_OVERRIDE] + begin + run_linux_shell(key) + ensure + compose_down(files: files) + end + end + + desc 'Start backend services for development (default backend: fs)' + task :up, [:backend] do |_t, args| + key = (args[:backend] || DEFAULT_BACKEND).to_sym + cfg!(key) + compose_up(key, files: [BASE_COMPOSE]) + end + + desc 'Stop backend services for development (default backend: fs)' + task :down, [:backend] do |_t, args| + compose_down(files: [BASE_COMPOSE]) + end end end diff --git a/test/data/graphdb-repo-config.ttl b/test/fixtures/backends/graphdb/graphdb-repo-config.ttl similarity index 95% rename from test/data/graphdb-repo-config.ttl rename to test/fixtures/backends/graphdb/graphdb-repo-config.ttl index 9200da9a..84032a0b 100644 --- a/test/data/graphdb-repo-config.ttl +++ b/test/fixtures/backends/graphdb/graphdb-repo-config.ttl @@ -3,8 +3,8 @@ @prefix sail: . @prefix xsd: . -<#ontoportal> a rep:Repository; - rep:repositoryID "ontoportal"; +<#ontoportal_test> a rep:Repository; + rep:repositoryID "ontoportal_test"; rep:repositoryImpl [ rep:repositoryType "graphdb:SailRepository"; [ @@ -30,4 +30,4 @@ sail:sailType "owlim:Sail" ] ]; - rdfs:label "" . \ No newline at end of file + rdfs:label "" . diff --git a/test/data/graphdb-test-load.nt b/test/fixtures/backends/graphdb/graphdb-test-load.nt similarity index 100% rename from test/data/graphdb-test-load.nt rename to test/fixtures/backends/graphdb/graphdb-test-load.nt diff --git a/test/fixtures/backends/virtuoso_initdb_d/virtuoso-grant-write-sparql-access.sql b/test/fixtures/backends/virtuoso_initdb_d/virtuoso-grant-write-sparql-access.sql new file mode 100644 index 00000000..d509c6fb --- /dev/null +++ b/test/fixtures/backends/virtuoso_initdb_d/virtuoso-grant-write-sparql-access.sql @@ -0,0 +1,3 @@ +GRANT EXECUTE ON DB.DBA.SPARQL_INSERT_DICT_CONTENT TO "SPARQL"; +GRANT SPARQL_UPDATE TO "SPARQL"; +DB.DBA.RDF_DEFAULT_USER_PERMS_SET ('nobody', 7); diff --git a/test/test_cache.rb b/test/test_cache.rb index 11dccf41..aff0c426 100644 --- a/test/test_cache.rb +++ b/test/test_cache.rb @@ -154,7 +154,7 @@ def x.response *args #different query programs = Program.where(name: "BioInformatics X", university: [ name: "Stanford" ]).all end - Goo.test_reset + TestHelpers.test_reset Goo.use_cache=false end diff --git a/test/test_case.rb b/test/test_case.rb index af7f2a84..22fd77bc 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -22,6 +22,81 @@ require_relative "../lib/goo.rb" require_relative '../config/config' +# Safety guard for destructive tests: ensure test targets are safe (localhost or -ut suffix) +module TestSafety + SAFE_HOSTS = Regexp.new(/localhost|-ut/) + MAX_REDIS_KEYS = 10 + + def self.safe_host?(value) + value = value.to_s + return false if value.empty? + !!(value =~ SAFE_HOSTS) + end + + def self.targets + { + triplestore: Goo.settings.goo_host.to_s, + search: Goo.settings.search_server_url.to_s, + redis: Goo.settings.goo_redis_host.to_s + } + end + + def self.unsafe_targets? + t = targets + unsafe = !safe_host?(t[:triplestore]) || !safe_host?(t[:search]) || !safe_host?(t[:redis]) + [unsafe, t] + end + + def self.ensure_safe_test_targets! + return if @safety_checked + unsafe, t = unsafe_targets? + return if !unsafe || ENV['CI'] == 'true' + + if $stdin.tty? + puts "\n\n================================== WARNING ==================================\n" + puts "** TESTS CAN BE DESTRUCTIVE -- YOU ARE POINTING TO A POTENTIAL PRODUCTION/STAGE SERVER **" + puts "Servers:" + puts "triplestore -- #{t[:triplestore]}" + puts "search -- #{t[:search]}" + puts "redis -- #{t[:redis]}" + print "Type 'y' to continue: " + $stdout.flush + confirm = $stdin.gets + abort('Canceling tests...') unless confirm && confirm.strip == 'y' + puts 'Running tests...' + $stdout.flush + else + abort('Aborting tests: non-whitelisted targets and non-interactive session.') + end + ensure + @safety_checked = true + end + + def self.ensure_safe_redis_size! + redis = Goo.redis_client + return unless redis + count = redis.dbsize + return if count <= MAX_REDIS_KEYS + abort("Aborting tests: redis has #{count} keys, expected <= #{MAX_REDIS_KEYS} for a test instance.") + end +end + +TestSafety.ensure_safe_test_targets! + +module TestHelpers + def self.test_reset + TestSafety.ensure_safe_test_targets! + TestSafety.ensure_safe_redis_size! + Goo.class_variable_set(:@@sparql_backends, {}) + Goo.add_sparql_backend(:main, + backend_name: Goo.settings.goo_backend_name, + query: "http://#{Goo.settings.goo_host}:#{Goo.settings.goo_port}#{Goo.settings.goo_path_query}", + data: "http://#{Goo.settings.goo_host}:#{Goo.settings.goo_port}#{Goo.settings.goo_path_data}", + update: "http://#{Goo.settings.goo_host}:#{Goo.settings.goo_port}#{Goo.settings.goo_path_update}", + options: { rules: :NONE }) + end +end + class GooTest class Unit < MiniTest::Unit @@ -79,4 +154,3 @@ def self.count_pattern(pattern) end end -