Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
.github
build*
18 changes: 10 additions & 8 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,23 +25,25 @@ jobs:
docker exec -u sftnight -t cvmfs-dev bash -c \
"sudo cvmfs_config setup"

- name: Run Tests
- name: Run client/server tests
run: |
docker exec -u sftnight -t cvmfs-dev bash -c \
"cd /home/sftnight/cvmfs/test/common/container && CVMFS_TEST_PROXY=DIRECT bash test.sh"

- name: Archive logs
if: ${{ always () }}
if: ${{ always() }}
run: |
docker cp cvmfs-dev:/tmp/cvmfs-client-test.log /tmp/cvmfs-client-test.log
docker cp cvmfs-dev:/tmp/cvmfs-server-test.log /tmp/cvmfs-server-test.log
for logname in client server; do
log=/tmp/cvmfs-$logname-test.log
docker cp cvmfs-dev:$log $log || true
done

- name: Upload logs as artifact
if: ${{ always () }}
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: CVMFS test logs
with:
name: CVMFS client-server test logs
if-no-files-found: warn
path: |
/tmp/cvmfs-client-test.log
/tmp/cvmfs-server-test.log

1 change: 0 additions & 1 deletion .github/workflows/ci_container_unpacker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ name: container_unpacker
on:
pull_request:
branches: [devel, cvmfs*]
push:
workflow_dispatch:

jobs:
Expand Down
69 changes: 69 additions & 0 deletions .github/workflows/ci_migration.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
name: Hotpatch and migration tests

on:
workflow_dispatch:
#TODO: the containerized workflow runs into an unsolved issue in migration test 500
# The remount fails after the snapshot.
# push:
# branches: ['devel', 'cvmfs*', 'migration-ci']
# pull_request:
# branches: ['devel', 'cvmfs*']

jobs:
docker:
name: "Hotpatch/migration tests (dockerized, alma9)"
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Build and cache
uses: docker/build-push-action@v5
with:
context: .
file: test/common/container/Dockerfile-dev
push: false
tags: cvmfs-dev:latest
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Start container
run: |
cd test/common/container
docker compose up -d cvmfs-dev

- name: Setup CVMFS
run: |
docker exec -u sftnight -t cvmfs-dev bash -c \
"sudo yum -y install git"
docker exec -u sftnight -t cvmfs-dev bash -c \
"sudo cvmfs_config setup"

- name: Run hotpatch/migration tests
run: |
docker exec -u sftnight -t cvmfs-dev bash -c \
"cd /home/sftnight/cvmfs && CVMFS_TEST_PROXY=DIRECT bash test/common/container/test_migration.sh"

- name: Archive logs
if: ${{ always() }}
run: |
mkdir -p /tmp/logs
docker cp cvmfs-dev:/tmp/cvmfs-migration-test.log /tmp/logs/ || true

- name: Dump logs
if: ${{ always() }}
run: |
cat /tmp/logs/cvmfs-migration-test.log || true

- name: Upload logs as artifact
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: CVMFS hotpatch-migration logs
if-no-files-found: warn
path: |
/tmp/logs/cvmfs-migration-test.log
128 changes: 128 additions & 0 deletions .github/workflows/ci_migration_ubuntu.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
name: Hotpatch and migration tests (Ubuntu)

on:
push:
branches: ['devel', 'cvmfs*', 'migration-ci']
pull_request:
branches: ['devel', 'cvmfs*']
workflow_dispatch:

jobs:
ubuntu:
name: "Hotpatch/migration tests (Ubuntu, no Docker)"
runs-on: ubuntu-latest
env:
CVMFS_PACKAGE_DIR: /tmp/build

steps:
- id: lsb-release
run: |
if [ "$RUNNER_OS" == "Linux" ]; then
source /etc/lsb-release
echo "id=${DISTRIB_ID}" >> $GITHUB_OUTPUT
echo "release=${DISTRIB_RELEASE}" >> $GITHUB_OUTPUT
echo "codename=${DISTRIB_CODENAME}" >> $GITHUB_OUTPUT
echo "description=${DISTRIB_DESCRIPTION}" >> $GITHUB_OUTPUT
echo "id-release=${DISTRIB_ID}-${DISTRIB_RELEASE}" >> $GITHUB_OUTPUT
echo "arch=$(uname -m)" >> $GITHUB_OUTPUT
fi

- name: Checkout repository
uses: actions/checkout@v4

- name: Create cache directories
run: |
mkdir -p /home/runner/.cache

- name: Restore CI cache
uses: actions/cache/restore@v4
id: cvmfs-ci-cache-restore
with:
path: /home/runner/.cache/
key: v3-cvmfs-ci-cache-${{ steps.lsb-release.outputs.id-release }}-${{ steps.lsb-release.outputs.arch }}

- name: Check for builddep package
id: cvmfs-builddep-check
run: |
if [ -f "/home/runner/.cache/cvmfs-build-deps_*_all.deb" ]; then
echo "package_exists=true" >> $GITHUB_OUTPUT
else
echo "package_exists=false" >> $GITHUB_OUTPUT
fi

- name: Generate build-dep metapackage
if: steps.cvmfs-builddep-check.outputs.package_exists != 'true'
run: |
sudo apt-get -y update
sudo apt-get install -y devscripts equivs
mk-build-deps ./packaging/debian/cvmfs/control
mv cvmfs-build-deps_*_all.deb /home/runner/.cache

- name: Install build dependencies
run: |
sudo apt-get install -y /home/runner/.cache/cvmfs-build-deps_*_all.deb

- name: Build and install CVMFS config packages
run: |
mkdir -p /tmp/cvmfs-config
./ci/build_package.sh $PWD /tmp/cvmfs-config cvmfs-config
sudo apt-get install -y /tmp/cvmfs-config/*.deb

- name: Build CVMFS debian packages
run: |
sudo apt-get -y install ccache
export CVMFS_EXTERNALS_PREFIX=/home/runner/.cache/cvmfs_externals_install
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
mkdir -p $CVMFS_PACKAGE_DIR
./ci/build_package.sh $PWD $CVMFS_PACKAGE_DIR cvmfs

- name: Check CCache Hits
run: |
ccache -s

- name: Install runtime dependencies
run: |
sudo apt-get -y install \
apache2 \
attr \
autofs \
fuse3 \
jq \
libapache2-mod-wsgi-py3 \
lsof \
netcat-openbsd \
python3 \
sqlite3 \
wget

- name: Install CVMFS packages
run: |
sudo apt-get install -y $CVMFS_PACKAGE_DIR/*.deb

- name: Setup CVMFS
run: |
sudo cvmfs_config setup
sudo systemctl start apache2

- name: Run hotpatch/migration tests
run: |
cd test
CVMFS_TEST_PROXY=DIRECT CVMFS_TEST_CLASS_NAME=MigrationTests \
./run.sh /tmp/cvmfs-migration-test.log \
migration_tests/001-hotpatch \
migration_tests/500-server_repo

- name: Dump logs
if: ${{ failure() }}
run: |
cat /tmp/cvmfs-migration-test.log || true

- name: Upload logs as artifact
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: CVMFS hotpatch-migration Ubuntu logs
if-no-files-found: warn
path: |
/tmp/cvmfs-migration-test.log

4 changes: 2 additions & 2 deletions .github/workflows/ci_ubuntu.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ name: cloudtests_ubuntu
on:
pull_request:
branches: [devel, cvmfs*]
push:
workflow_dispatch:
inputs:
inputTestCases:
Expand Down Expand Up @@ -210,7 +209,8 @@ jobs:
export CVMFS_TESTCASES=${{ github.event.inputs.testCases || '"src/0* src/1* src/5* src/6* src/7* src/8*"' }}
export CVMFS_TEST_SUITES=${{ github.event.inputs.testSuites || 'quick' }}
CVMFS_TEST_PROXY=http://localhost:3128 \
CVMFS_TEST_USER=$(whoami) \
CVMFS_TEST_USER=$(id -un) \
CVMFS_TEST_GROUP=$(id -gn) \
./run.sh /tmp/test.log \
-s "${CVMFS_TEST_SUITES}" \
-x src/095-fuser \
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/ci_ubuntu_unittest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ name: unittest_ubuntu
on:
pull_request:
branches: [devel, cvmfs*]
push:
workflow_dispatch:

jobs:
Expand Down
13 changes: 9 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
#

cmake_minimum_required (VERSION 3.24..3.30)

# Enable creation of compile_commands.json by default. Needed by LSPs, useful for devs
# Must be set before project() so the generator picks it up
set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "Export compile_commands.json for LSPs")

set (PROJECT_NAME "CernVM-FS")
project (${PROJECT_NAME})

Expand All @@ -15,12 +20,12 @@ message ("Running CMake version ${CMAKE_VERSION}")
# The version numbers
#
# DON'T DELETE
## CVMFS_VERSION 2.13.3
## CVMFS_PRERELEASE
## CVMFS_VERSION 2.14.0
## CVMFS_PRERELEASE ~pre1
#---------------------
set (CernVM-FS_VERSION_MAJOR 2)
set (CernVM-FS_VERSION_MINOR 13)
set (CernVM-FS_VERSION_PATCH 3)
set (CernVM-FS_VERSION_MINOR 14)
set (CernVM-FS_VERSION_PATCH 0)
set (CernVM-FS_VERSION_STRING "${CernVM-FS_VERSION_MAJOR}.${CernVM-FS_VERSION_MINOR}.${CernVM-FS_VERSION_PATCH}")
if (CMAKE_VERSION VERSION_LESS "3.12.0")
add_definitions(-DCVMFS_VERSION="${CernVM-FS_VERSION_STRING}")
Expand Down
2 changes: 2 additions & 0 deletions ci/build_incremental_multi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ if can_build_gateway; then
build_gateway="ON"
fi

host cvm-gomod-proxy1.cern.ch > /dev/null 2>&1 && export GOPROXY=http://cvm-gomod-proxy1.cern.ch:3000

echo "configuring using CMake..."
cmake -DBUILD_SERVER=$build_server \
-DBUILD_SERVER_DEBUG=$build_server \
Expand Down
32 changes: 5 additions & 27 deletions ci/build_install_builddeps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,16 +79,14 @@ detect_platform() {
PKG_MGR="apt-get"; OS_FAMILY="deb"; return 0
fi
if echo "$like_all" | grep -Eq '(rhel|centos|fedora|rocky|almalinux|ol)'; then
if check_available dnf; then PKG_MGR="dnf"; else PKG_MGR="yum"; fi
OS_FAMILY="rhel"; return 0
PKG_MGR="dnf"; OS_FAMILY="rhel"; return 0
fi
if echo "$like_all" | grep -Eq '(suse|sles|opensuse)'; then
PKG_MGR="zypper"; OS_FAMILY="suse"; return 0
fi
# Fallbacks by package manager availability
if check_available apt-get; then PKG_MGR="apt-get"; OS_FAMILY="deb"; return 0; fi
if check_available dnf; then PKG_MGR="dnf"; OS_FAMILY="rhel"; return 0; fi
if check_available yum; then PKG_MGR="yum"; OS_FAMILY="rhel"; return 0; fi
if check_available zypper; then PKG_MGR="zypper"; OS_FAMILY="suse"; return 0; fi
return 1
}
Expand Down Expand Up @@ -172,35 +170,15 @@ install_deps_deb() {
install_deps_rhel() {
[ -f "$RPM_SPEC" ] || die "RPM spec file not found at $RPM_SPEC"
if ! check_available rpmbuild; then
if [[ "$PKG_MGR" = "dnf" ]]; then
$SUDO dnf -y install rpm-build
else
$SUDO yum -y install rpm-build
fi
fi
if [[ "$PKG_MGR" = "dnf" ]]; then
$SUDO dnf -y install dnf-plugins-core || true
if check_available dnf; then
$SUDO dnf builddep -y "$RPM_SPEC" && return 0
fi
fi
if [[ "$PKG_MGR" = "yum" ]]; then
$SUDO yum -y install yum-utils || true
if check_available yum-builddep; then
$SUDO yum-builddep -y "$RPM_SPEC" && return 0
else
$SUDO yum builddep -y "$RPM_SPEC" && return 0 || true
fi
$SUDO dnf -y install rpm-build
fi
$SUDO dnf -y install dnf-plugins-core || true
$SUDO dnf builddep -y "$RPM_SPEC" && return 0
# Fallback: parse spec and install packages directly
local pkgs
pkgs=$(list_deps_rpm "$RPM_SPEC" || true)
[ -n "${pkgs:-}" ] || die "Could not determine RPM build dependencies"
if [[ "$PKG_MGR" = "dnf" ]]; then
$SUDO dnf -y install $pkgs
else
$SUDO yum -y install $pkgs
fi
$SUDO dnf -y install $pkgs
}

install_deps_suse() {
Expand Down
2 changes: 1 addition & 1 deletion ci/build_package.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/sh
#!/usr/bin/env sh

#
# This script builds a package that fits the current platform using custom build
Expand Down
2 changes: 1 addition & 1 deletion ci/common.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/sh
#!/usr/bin/env sh

die() {
local msg="$1"
Expand Down
7 changes: 4 additions & 3 deletions ci/cvmfs/pkg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ pkg_build_dir=${CVMFS_RESULT_LOCATION}/pkg_result
[ ! -d $pkg_install_dir ] || die "build directory was used before ($pkg_install_dir exists)"
[ ! -d $pkg_build_dir ] || die "build directory was used before ($pkg_build_dir exists)"
[ ! -d $pkg_resource_dir ] || die "build directory was used before ($pkg_resource_dir exists)"
which pkgbuild > /dev/null 2>&1 || die "didn't find 'pkgbuild' utility"
which productbuild > /dev/null 2>&1 || die "didn't find 'productbuild' utility"
which tiffutil > /dev/null 2>&1 || die "didn't find 'tiffutil' utility"
command -v pkgbuild > /dev/null 2>&1 || die "didn't find 'pkgbuild' utility"
command -v productbuild > /dev/null 2>&1 || die "didn't find 'productbuild' utility"
command -v tiffutil > /dev/null 2>&1 || die "didn't find 'tiffutil' utility"

# setup environment
mkdir -p $pkg_install_dir
Expand All @@ -54,6 +54,7 @@ echo "detected upstream version: $cvmfs_version"

echo "building CernVM-FS $cvmfs_version in '$CVMFS_RESULT_LOCATION' from '$CVMFS_SOURCE_LOCATION'"
cd $CVMFS_RESULT_LOCATION

cmake -DCMAKE_INSTALL_PREFIX:PATH=$CVMFS_INSTALL_PREFIX \
-DBUILTIN_EXTERNALS_EXCLUDE=${CVMFS_EXTERNALS_EXCLUDE:-""} \
-DCMAKE_CXX_STANDARD=11 \
Expand Down
Loading