diff --git a/.github/workflows/apt.yml b/.github/workflows/apt.yml index e25e68b8f..b7e57971c 100644 --- a/.github/workflows/apt.yml +++ b/.github/workflows/apt.yml @@ -11,7 +11,7 @@ concurrency: jobs: build: name: Build - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -54,16 +54,34 @@ jobs: fluent-package/apt/repositories fluent-apt-source/apt/repositories fluent-lts-apt-source/apt/repositories + v6-test/fluent-package/apt/repositories key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', 'fluent-package/templates/**', 'fluent-package/debian/**', 'fluent-package/apt/**/Dockerfile') }} - name: Build deb with Docker if: ${{ ! steps.cache-deb.outputs.cache-hit }} run: | rake apt:build APT_TARGETS=${{ matrix.rake-job }} + - uses: actions/checkout@master + if: ${{ ! steps.cache-deb.outputs.cache-hit }} + with: + path: v6-test + - name: Build v6 deb with Docker + if: ${{ ! steps.cache-deb.outputs.cache-hit }} + run: | + cd v6-test + git config user.email "fluentd@googlegroups.com" + git config user.name "Fluentd developers" + git am fluent-package/bump-version-v6.patch + rake apt:build APT_TARGETS=${{ matrix.rake-job }} - name: Upload fluent-package deb uses: actions/upload-artifact@master with: name: packages-${{ matrix.rake-job }} path: fluent-package/apt/repositories + - name: Upload v6 fluent-package deb + uses: actions/upload-artifact@master + with: + name: v6-packages-${{ matrix.rake-job }} + path: v6-test/fluent-package/apt/repositories - name: Upload fluent-apt-source deb uses: actions/upload-artifact@master with: @@ -147,7 +165,9 @@ jobs: - "update-from-v4.sh local" - "update-from-v4.sh v5" - "update-from-v4.sh lts" + - "update-from-v5-lts.sh" - "downgrade-to-v4.sh" + - "downgrade-to-v5-lts.sh" - "install-newly.sh local" - "install-newly.sh v5" - "install-newly.sh lts" @@ -157,6 +177,15 @@ jobs: - "update-to-next-version-service-status.sh enabled inactive" - "update-to-next-version-service-status.sh disabled active" - "update-to-next-version-service-status.sh disabled inactive" + - "update-to-next-version-with-auto-and-manual.sh" + - "update-to-next-major-version.sh auto active" + - "update-to-next-major-version.sh auto inactive" + - "update-to-next-major-version.sh manual active" + - "update-to-next-major-version.sh manual inactive" + - "update-to-next-major-version.sh etc active" + - "update-to-next-major-version.sh etc inactive" + - "update-without-data-lost.sh v5 v6" + - "update-without-data-lost.sh v6 v5" include: - label: Debian bullseye amd64 rake-job: debian-bullseye @@ -199,9 +228,31 @@ jobs: - uses: actions/download-artifact@v4 with: name: packages-${{ matrix.rake-job }} + - uses: actions/download-artifact@v4 + with: + name: v6-packages-${{ matrix.rake-job }} + path: v6-test - uses: actions/download-artifact@v4 with: name: packages-apt-source-${{ matrix.rake-job }} - uses: canonical/setup-lxd@v0.1.2 + - name: Run diagnostic + run: | + uname -a + echo "::group::snap info lxd" + snap info lxd + echo "::endgroup::" + echo "::group::snap services lxd" + snap services lxd + echo "::endgroup::" + echo "::group::snap logs lxd" + sudo snap logs lxd + echo "::endgroup::" + echo "::group::lxc remote list" + lxc remote list + echo "::endgroup::" + echo "::group::lxc list images:" + lxc image list images: + echo "::endgroup::" - name: Run Test ${{ matrix.test }} on ${{ matrix.lxc-image }} run: fluent-package/apt/systemd-test/test.sh ${{ matrix.lxc-image }} ${{ matrix.test }} diff --git a/.github/workflows/yum.yml b/.github/workflows/yum.yml index 301bca49e..c7daeb765 100644 --- a/.github/workflows/yum.yml +++ b/.github/workflows/yum.yml @@ -11,7 +11,7 @@ concurrency: jobs: build: name: Build - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -50,17 +50,36 @@ jobs: uses: actions/cache@v4 id: cache-rpm with: - path: fluent-package/yum/repositories + path: | + fluent-package/yum/repositories + v6-test/fluent-package/yum/repositories key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', '**/*.spec.in', 'fluent-package/templates/**', 'fluent-package/yum/**/Dockerfile') }} - name: Build rpm with Docker if: ${{ ! steps.cache-rpm.outputs.cache-hit }} run: | rake yum:build YUM_TARGETS=${{ matrix.rake-job }} + - uses: actions/checkout@master + if: ${{ ! steps.cache-rpm.outputs.cache-hit }} + with: + path: v6-test + - name: Build v6 rpm with Docker + if: ${{ ! steps.cache-rpm.outputs.cache-hit }} + run: | + cd v6-test + git config user.email "fluentd@googlegroups.com" + git config user.name "Fluentd developers" + git am fluent-package/bump-version-v6.patch + rake yum:build YUM_TARGETS=${{ matrix.rake-job }} - name: Upload fluent-package rpm uses: actions/upload-artifact@v4 with: name: packages-${{ matrix.rake-job }} path: fluent-package/yum/repositories + - name: Upload v6 fluent-package rpm + uses: actions/upload-artifact@v4 + with: + name: v6-packages-${{ matrix.rake-job }} + path: v6-test/fluent-package/yum/repositories # TODO move the following steps to "Test" job - name: Check Package Size run: | @@ -121,7 +140,9 @@ jobs: - AmazonLinux 2023 x86_64 test: - "update-from-v4.sh" + - "update-from-v5-lts.sh" - "downgrade-to-v4.sh" + - "downgrade-to-v5-lts.sh" - "install-newly.sh local" - "install-newly.sh v5" - "install-newly.sh lts" @@ -131,6 +152,15 @@ jobs: - "update-to-next-version-service-status.sh enabled inactive" - "update-to-next-version-service-status.sh disabled active" - "update-to-next-version-service-status.sh disabled inactive" + - "update-to-next-version-with-auto-and-manual.sh" + - "update-to-next-major-version.sh auto active" + - "update-to-next-major-version.sh auto inactive" + - "update-to-next-major-version.sh manual active" + - "update-to-next-major-version.sh manual inactive" + - "update-to-next-major-version.sh etc active" + - "update-to-next-major-version.sh etc inactive" + - "update-without-data-lost.sh v5 v6" + - "update-without-data-lost.sh v6 v5" include: - label: AmazonLinux 2 x86_64 rake-job: amazonlinux-2 @@ -150,6 +180,10 @@ jobs: - uses: actions/download-artifact@v4 with: name: packages-${{ matrix.rake-job }} + - uses: actions/download-artifact@v4 + with: + name: v6-packages-${{ matrix.rake-job }} + path: v6-test - uses: canonical/setup-lxd@v0.1.2 - name: Run diagnostic run: | @@ -185,7 +219,9 @@ jobs: - AlmaLinux 9 x86_64 test: - "update-from-v4.sh" + - "update-from-v5-lts.sh" - "downgrade-to-v4.sh" + - "downgrade-to-v5-lts.sh" - "install-newly.sh local" - "install-newly.sh v5" - "install-newly.sh lts" @@ -195,6 +231,15 @@ jobs: - "update-to-next-version-service-status.sh enabled inactive" - "update-to-next-version-service-status.sh disabled active" - "update-to-next-version-service-status.sh disabled inactive" + - "update-to-next-version-with-auto-and-manual.sh" + - "update-to-next-major-version.sh auto active" + - "update-to-next-major-version.sh auto inactive" + - "update-to-next-major-version.sh manual active" + - "update-to-next-major-version.sh manual inactive" + - "update-to-next-major-version.sh etc active" + - "update-to-next-major-version.sh etc inactive" + - "update-without-data-lost.sh v5 v6" + - "update-without-data-lost.sh v6 v5" include: - label: RockyLinux 8 x86_64 rake-job: rockylinux-8 @@ -207,6 +252,10 @@ jobs: - uses: actions/download-artifact@v4 with: name: packages-${{ matrix.rake-job }} + - uses: actions/download-artifact@v4 + with: + name: v6-packages-${{ matrix.rake-job }} + path: v6-test - uses: canonical/setup-lxd@v0.1.2 - name: Run Test ${{ matrix.test }} on ${{ matrix.lxc-image }} run: fluent-package/yum/systemd-test/test.sh ${{ matrix.lxc-image }} ${{ matrix.test }} diff --git a/fluent-package/Rakefile b/fluent-package/Rakefile index 3d1f8c579..84c7a2caa 100755 --- a/fluent-package/Rakefile +++ b/fluent-package/Rakefile @@ -494,7 +494,7 @@ class BuildTask remove_needless_files end - debian_pkg_scripts = ["preinst", "postinst", "postrm"] + debian_pkg_scripts = ["preinst", "postinst", "prerm", "postrm"] debian_pkg_scripts.each do |script| CLEAN.include(File.join("..", "debian", script)) end diff --git a/fluent-package/apt/commonvar.sh b/fluent-package/apt/commonvar.sh index 11147319b..597bb131f 100644 --- a/fluent-package/apt/commonvar.sh +++ b/fluent-package/apt/commonvar.sh @@ -3,6 +3,8 @@ architecture=$(dpkg --print-architecture) repositories_dir=/fluentd/fluent-package/apt/repositories java_jdk=openjdk-11-jre td_agent_version=4.5.2 +fluent_package_lts_version=5.0.5 + case ${code_name} in xenial) distribution=ubuntu diff --git a/fluent-package/apt/systemd-test/downgrade-to-v5-lts.sh b/fluent-package/apt/systemd-test/downgrade-to-v5-lts.sh new file mode 100755 index 000000000..2858daa2e --- /dev/null +++ b/fluent-package/apt/systemd-test/downgrade-to-v5-lts.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/../commonvar.sh + +# Install v5 LTS to register the repository +curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5-lts.sh | sh + +sudo apt purge -y fluent-package + +# Install the current +sudo apt install -V -y \ + /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb + +# Test: service status +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent +main_pid=$(eval $(systemctl show td-agent --property=MainPID) && echo $MainPID) + +# Downgrade to v5 LTS +apt install -V -y fluent-package=${fluent_package_lts_version}-1 --allow-downgrades + +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent + +# Fluentd should be restarted. +# NOTE: Unlike RPM, the restart behavior depends on TO-side. So, it restarts. +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) diff --git a/fluent-package/apt/systemd-test/test.sh b/fluent-package/apt/systemd-test/test.sh index 45469953b..b91e456af 100755 --- a/fluent-package/apt/systemd-test/test.sh +++ b/fluent-package/apt/systemd-test/test.sh @@ -15,7 +15,7 @@ dir="/host/fluent-package/apt/systemd-test" set -eux echo "::group::Run test: launch $image" -lxc launch $image target +lxc launch $image target --debug sleep 5 echo "::endgroup::" echo "::group::Run test: configure $image" diff --git a/fluent-package/apt/systemd-test/update-from-v5-lts.sh b/fluent-package/apt/systemd-test/update-from-v5-lts.sh new file mode 100755 index 000000000..9baa72fba --- /dev/null +++ b/fluent-package/apt/systemd-test/update-from-v5-lts.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/../commonvar.sh + + +# If it update from v5 LTS without stopping Fluentd, Fluentd will not be restarted. +# Install v5 LTS +curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5-lts.sh | sh + +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent +main_pid=$(eval $(systemctl show td-agent --property=MainPID) && echo $MainPID) + +# Install the current +sudo apt install -V -y \ + /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb + +# Test: service status +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent + +# Fluentd should NOT be restarted. +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +apt purge -y fluent-package + +# If it update from v5 LTS with stopping Fluentd, Fluentd will be started when service is enabled. +# Install v5 LTS +curl --fail --silent --show-error --location https://toolbelt.treasuredata.com/sh/install-${distribution}-${code_name}-fluent-package5-lts.sh | sh + +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent +main_pid=$(eval $(systemctl show td-agent --property=MainPID) && echo $MainPID) + +systemctl stop fluentd + +# Install the current +sudo apt install -V -y \ + /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb + +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent + +# Fluentd should be started if service was stopped before update. +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Test: environmental variables +pid=$(systemctl show fluentd --property=MainPID --value) +env_vars=$(sudo sed -e 's/\x0/\n/g' /proc/$pid/environ) +test $(eval $env_vars && echo $HOME) = "/var/lib/fluent" +test $(eval $env_vars && echo $LOGNAME) = "_fluentd" +test $(eval $env_vars && echo $USER) = "_fluentd" +test $(eval $env_vars && echo $FLUENT_CONF) = "/etc/fluent/fluentd.conf" +test $(eval $env_vars && echo $FLUENT_PACKAGE_LOG_FILE) = "/var/log/fluent/fluentd.log" +test $(eval $env_vars && echo $FLUENT_PLUGIN) = "/etc/fluent/plugin" +test $(eval $env_vars && echo $FLUENT_SOCKET) = "/var/run/fluent/fluentd.sock" + +# Test: No error logs +# (v5 default config outputs 'warn' log, so we should check only 'error' and 'fatal' logs) +sleep 3 +test -e /var/log/fluent/fluentd.log +(! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) + +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd) +(! sudo /usr/sbin/fluentd -v) +sudo /usr/sbin/fluentd --dry-run diff --git a/fluent-package/apt/systemd-test/update-to-next-major-version.sh b/fluent-package/apt/systemd-test/update-to-next-major-version.sh new file mode 100755 index 000000000..cb6fb0f9d --- /dev/null +++ b/fluent-package/apt/systemd-test/update-to-next-major-version.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/../commonvar.sh + +service_restart=$1 +status_before_update=$2 # active / inactive + +# Install the current +sudo apt install -V -y \ + /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb + +if [ "$status_before_update" = inactive ]; then + sudo systemctl stop fluentd +fi + +# Set FLUENT_PACKAGE_SERVICE_RESTART +sed -i "s/=auto/=$service_restart/" /etc/default/fluentd + +# Install plugin manually (plugin and gem) +sudo /opt/fluent/bin/fluent-gem install --no-document fluent-plugin-concat +sudo /opt/fluent/bin/fluent-gem install --no-document gqtp + +# Install next major version +sudo apt install -V -y \ + /host/v6-test/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb 2>&1 | tee upgrade.log + +# Test: needrestart was suppressed +if dpkg-query --show --showformat='${Version}' needrestart ; then + case $code_name in + focal) + # dpkg-query succeeds even though needrestart is not installed. + (! grep "No services need to be restarted." upgrade.log) + ;; + *) + grep "No services need to be restarted." upgrade.log + ;; + esac +fi + +# Test: Check whether plugin/gem were installed during upgrading +if [ "$service_restart" != manual ] && [ "$status_before_update" = active ]; then + # plugin gem should be installed automatically + /opt/fluent/bin/fluent-gem list | grep fluent-plugin-concat + # Non fluent-plugin- prefix gem should not be installed automatically + (! /opt/fluent/bin/fluent-gem list | grep gqtp) +else + # plugin gem should not be installed automatically + (! /opt/fluent/bin/fluent-gem list | grep fluent-plugin-concat) +fi + diff --git a/fluent-package/apt/systemd-test/update-to-next-version-service-status.sh b/fluent-package/apt/systemd-test/update-to-next-version-service-status.sh index 502896283..7b2031a56 100755 --- a/fluent-package/apt/systemd-test/update-to-next-version-service-status.sh +++ b/fluent-package/apt/systemd-test/update-to-next-version-service-status.sh @@ -55,10 +55,14 @@ else (! systemctl is-enabled fluentd) fi -if [ "$status_before_update" = active ] || [ "$enabled_before_update" = enabled ] ; then - # The service should restart automatically after update +if [ "$status_before_update" = active ]; then + # The service should NOT restart automatically after update + # (The process before update should continue to run) + systemctl is-active fluentd + test $main_pid -eq $(systemctl show --value --property=MainPID fluentd) +elif [ "$enabled_before_update" = enabled ]; then + # The service should start automatically systemctl is-active fluentd - test $main_pid -ne $(systemctl show --value --property=MainPID fluentd) else # The service should NOT start automatically (! systemctl is-active fluentd) diff --git a/fluent-package/apt/systemd-test/update-to-next-version-with-auto-and-manual.sh b/fluent-package/apt/systemd-test/update-to-next-version-with-auto-and-manual.sh new file mode 100755 index 000000000..63f2646f6 --- /dev/null +++ b/fluent-package/apt/systemd-test/update-to-next-version-with-auto-and-manual.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/../commonvar.sh + +package="/host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb" + +# Make a dummy pacakge for the next version +dpkg-deb -R /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb tmp +last_ver=$(cat tmp/DEBIAN/control | grep "Version: " | sed -E "s/Version: ([0-9.]+)-([0-9]+)/\2/g") +sed -i -E "s/Version: ([0-9.]+)-([0-9]+)/Version: \1-$(($last_ver+1))/g" tmp/DEBIAN/control +dpkg-deb --build tmp next_version.deb + +# Upgrade package with auto feature +sudo apt install -V -y $package +main_pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +sudo apt install -V -y ./next_version.deb +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Main process should be replaced by USR2 signal in auto mode +sleep 15 +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +sudo apt purge -y fluent-package + +# Upgrade package with manual feature +sudo apt install -V -y $package +sudo systemctl enable --now fluentd +sed -i 's/=auto/=manual/' /etc/default/fluentd +main_pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +sudo apt install -V -y ./next_version.deb +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Main process should NOT be replaced until USR2 signal fired +sleep 15 +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +kill -USR2 $main_pid + +# Main process should be replaced by USR2 signal +sleep 15 +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) diff --git a/fluent-package/apt/systemd-test/update-without-data-lost.sh b/fluent-package/apt/systemd-test/update-without-data-lost.sh new file mode 100755 index 000000000..851edab51 --- /dev/null +++ b/fluent-package/apt/systemd-test/update-without-data-lost.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/../commonvar.sh + +v5_package="/host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb" +v6_package="/host/v6-test/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb" + +case "$1" in + v5) + package=$v5_package + ;; + v6) + package=$v6_package + ;; + *) + echo "Invalid argument: $1" + exit 1 + ;; +esac + +case "$2" in + v5) + next_package=$v5_package + ;; + v6) + next_package=$v6_package + ;; + *) + echo "Invalid argument: $2" + exit 1 + ;; +esac + +sudo apt install -V -y rsyslog + +# Install the current +sudo apt install -V -y $package + +# Set up configuration +cat < $(dirname $0)/../../test-tools/rsyslog.conf >> /etc/rsyslog.conf +cp $(dirname $0)/../../test-tools/fluentd.conf /etc/fluent/fluentd.conf + +# Launch rsyslog +sudo systemctl restart rsyslog + +# Launch fluentd +sudo systemctl restart fluentd +main_pid=$(systemctl show --value --property=MainPID fluentd) + +# Ensure to wait for fluentd launching +sleep 1 + +# Send logs in background for 4 seconds +/opt/fluent/bin/ruby $(dirname $0)/../../test-tools/logdata-sender.rb \ + --udp-data-count 50 --tcp-data-count 60 --syslog-data-count 70 --syslog-identifer "test-syslog" --duration 16 & + +sleep 1 + +# Update to the next version +sudo apt install -V -y --allow-downgrades $next_package +test $main_pid -eq $(systemctl show --value --property=MainPID fluentd) + +# Main process should be replaced by USR2 signal +sleep 20 +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Stop fluentd to flush the logs and check +systemctl stop fluentd +test $(wc -l /var/log/fluent/test_udp*.log | tail -n 1 | awk '{print $1}') = "50" +test $(wc -l /var/log/fluent/test_tcp*.log | tail -n 1 | awk '{print $1}') = "60" +test $(grep "test-syslog" /var/log/fluent/test_syslog*.log | wc -l) = "70" diff --git a/fluent-package/bump-version-v6.patch b/fluent-package/bump-version-v6.patch new file mode 100644 index 000000000..dfa68a27b --- /dev/null +++ b/fluent-package/bump-version-v6.patch @@ -0,0 +1,65 @@ +From 887c36b60822995aa889fea6a181f00975cdfff7 Mon Sep 17 00:00:00 2001 +From: Kentaro Hayashi +Date: Thu, 17 Oct 2024 19:45:02 +0900 +Subject: [PATCH 1/2] bump version + +Signed-off-by: Kentaro Hayashi +--- + fluent-package/config.rb | 6 +++--- + fluent-package/debian/changelog | 6 ++++++ + fluent-package/yum/fluent-package.spec.in | 3 +++ + 3 files changed, 12 insertions(+), 3 deletions(-) + +diff --git a/fluent-package/config.rb b/fluent-package/config.rb +index a9e13bc..e61bca4 100644 +--- a/fluent-package/config.rb ++++ b/fluent-package/config.rb +@@ -1,5 +1,5 @@ + PACKAGE_NAME = "fluent-package" +-PACKAGE_VERSION = "5.1.0" ++PACKAGE_VERSION = "6.0.0" + + # Keep internal path (/opt/td-agent) for package name migration + SERVICE_NAME = "fluentd" +@@ -23,8 +23,8 @@ OPENSSL_FOR_MACOS_SHA256SUM = "6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d + BUNDLER_VERSION= "2.3.27" + + # https://www.ruby-lang.org/en/downloads/ (tar.gz) +-BUNDLED_RUBY_VERSION = "3.2.5" +-BUNDLED_RUBY_SOURCE_SHA256SUM = "ef0610b498f60fb5cfd77b51adb3c10f4ca8ed9a17cb87c61e5bea314ac34a16" ++BUNDLED_RUBY_VERSION = "3.3.5" ++BUNDLED_RUBY_SOURCE_SHA256SUM = "3781a3504222c2f26cb4b9eb9c1a12dbf4944d366ce24a9ff8cf99ecbce75196" + + BUNDLED_RUBY_PATCHES = [ + # An example entry: +diff --git a/fluent-package/debian/changelog b/fluent-package/debian/changelog +index f24ec3e..0089f56 100644 +--- a/fluent-package/debian/changelog ++++ b/fluent-package/debian/changelog +@@ -1,3 +1,9 @@ ++fluent-package (6.0.0-1) unstable; urgency=low ++ ++ * New upstream release. ++ ++ -- Kentaro Hayashi Tue, 8 Oct 2024 07:47:29 -0000 ++ + fluent-package (5.1.0-1) unstable; urgency=low + + * New upstream release. +diff --git a/fluent-package/yum/fluent-package.spec.in b/fluent-package/yum/fluent-package.spec.in +index 4833f8e..da9da8c 100644 +--- a/fluent-package/yum/fluent-package.spec.in ++++ b/fluent-package/yum/fluent-package.spec.in +@@ -478,6 +478,9 @@ fi + # NOTE: %{_tmpfilesdir} is available since CentOS 7 + %attr(0755,fluentd,fluentd) %dir /tmp/@PACKAGE_DIR@ + %changelog ++* Mon Oct 7 2024 Kentaro Hayashi - 6.0.0-1 ++- New upstream release. (Dummy) ++ + * Mon Jul 29 2024 Kentaro Hayashi - 5.1.0-1 + - New upstream release. + +-- +2.45.2 + diff --git a/fluent-package/debian/rules b/fluent-package/debian/rules index 0b7a2352d..7f918cd33 100755 --- a/fluent-package/debian/rules +++ b/fluent-package/debian/rules @@ -16,6 +16,12 @@ override_dh_auto_install: CI=$CI rake build:all FLUENT_PACKAGE_STAGING_PATH="$(CURDIR)/debian/tmp" PATH="$(HOME)/.cargo/bin:$(PATH)" dh_installman +# Suppress forcing service start/restart in postinst by omitting +# related start/restart hook script embedding. It does not omit +# deb-systemd-helper enable/update-state fluentd.service +override_dh_installsystemd: + dh_installsystemd --no-restart-after-upgrade --no-stop-on-upgrade + override_dh_auto_clean: rake clean dh_auto_clean diff --git a/fluent-package/templates/etc/systemd/fluentd.erb b/fluent-package/templates/etc/systemd/fluentd.erb index 1a0436121..01eff104b 100644 --- a/fluent-package/templates/etc/systemd/fluentd.erb +++ b/fluent-package/templates/etc/systemd/fluentd.erb @@ -1 +1,3 @@ FLUENT_PACKAGE_OPTIONS="" +# Control method to upgrade service (auto/manual) restart +FLUENT_PACKAGE_SERVICE_RESTART=auto diff --git a/fluent-package/templates/package-scripts/fluent-package/deb/postinst b/fluent-package/templates/package-scripts/fluent-package/deb/postinst index dee4c4a67..8042204f7 100755 --- a/fluent-package/templates/package-scripts/fluent-package/deb/postinst +++ b/fluent-package/templates/package-scripts/fluent-package/deb/postinst @@ -145,6 +145,42 @@ migration_from_v4_post_process() { fi } +local_base_plugins="/tmp/<%= package_dir %>/.previous_plugin_list" +install_missing_plugins() { + # Install missing gems (even though systemd is not available, it works) + if [ -f $local_base_plugins ]; then + local_current_plugins=$(/usr/sbin/fluent-gem list '^fluent-plugin-' --no-versions --no-verbose) + if ! grep --fixed-strings --line-regexp --invert-match "$local_current_plugins" $local_base_plugins; then + echo "No missing plugins to install" + else + if ! curl --fail --silent -O https://rubygems.org/specs.4.8.gz; then + echo "No network connectivity..." + else + grep --fixed-strings --line-regexp --invert-match "$local_current_plugins" $local_base_plugins | while read missing_gem + do + if ! /usr/sbin/fluent-gem install --no-document $missing_gem; then + echo "Can't install missing plugin automatically: please install $missing_gem manually." + fi + done + fi + fi + rm -f $local_base_plugins + fi +} + +pid_for_auto_restart="/tmp/<%= package_dir %>/.pid_for_auto_restart" +fluentd_auto_restart() { + if [ ! -f "$pid_for_auto_restart" ]; then + return + fi + + pid=$(cat "$pid_for_auto_restart") + echo "Kick auto restart to MainPID:$pid" + kill -USR2 $pid + + rm -f "$pid_for_auto_restart" +} + case "$1" in configure) add_system_user @@ -152,6 +188,8 @@ case "$1" in migration_from_v4_main_process fixperms migration_from_v4_post_process + install_missing_plugins + fluentd_auto_restart ;; abort-upgrade|abort-deconfigure|abort-remove) : diff --git a/fluent-package/templates/package-scripts/fluent-package/deb/postrm b/fluent-package/templates/package-scripts/fluent-package/deb/postrm index 24e237e04..6e98aa3ab 100755 --- a/fluent-package/templates/package-scripts/fluent-package/deb/postrm +++ b/fluent-package/templates/package-scripts/fluent-package/deb/postrm @@ -60,6 +60,11 @@ purge_bin_symlinks() { fi } +purge_tmp_files_for_upgrade() { + rm -f "/tmp/<%= package_dir %>/.plugin_list" + rm -f "/tmp/<%= package_dir %>/.main_pid" +} + case $1 in remove) purge_var_run @@ -72,9 +77,13 @@ case $1 in purge_users purge_bin_symlinks ;; + upgrade) + purge_tmp_files_for_upgrade + ;; *) # nothing to do for upgrade, failed-upgrade, abort-install, abort-upgrade ;; esac #DEBHELPER# + diff --git a/fluent-package/templates/package-scripts/fluent-package/deb/preinst b/fluent-package/templates/package-scripts/fluent-package/deb/preinst new file mode 100755 index 000000000..96e478104 --- /dev/null +++ b/fluent-package/templates/package-scripts/fluent-package/deb/preinst @@ -0,0 +1,48 @@ +#!/bin/sh + +set -e + +# Summary of how this script can be called: +# * 'install' +# * 'install' +# * 'upgrade' +# * 'abort-upgrade' +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package. + +mark_auto_restart_ready() { + # Copy tmp files made by FROM-side because they should be cleaned postrm of FROM-side. + # (To ensure cleanup, tmp files should be cleaned by its own-side). + # The sequence of these tmp files is as follows: + # 1. FROM-prerm(upgrade): Leave tmp files if need. + # 2. TO-preinst(upgrade): Copy tmp files for TO-side. + # 3. FROM-postrm(upgrade): Clean tmp files of FROM-side(1.). + # 4. TO-postinst(configure): Use and clean tmp files of TO-side(2.). + if [ -f "/tmp/<%= package_dir %>/.plugin_list" ]; then + cp "/tmp/<%= package_dir %>/.plugin_list" "/tmp/<%= package_dir %>/.previous_plugin_list" + fi + if [ -f "/tmp/<%= package_dir %>/.main_pid" ]; then + cp "/tmp/<%= package_dir %>/.main_pid" "/tmp/<%= package_dir %>/.pid_for_auto_restart" + fi +} + +case "$1" in + upgrade) + mark_auto_restart_ready + ;; + abort-upgrade) + ;; + install) + ;; + *) + echo "preinst called with unknown argument '$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/fluent-package/templates/package-scripts/fluent-package/deb/prerm b/fluent-package/templates/package-scripts/fluent-package/deb/prerm new file mode 100755 index 000000000..bd1c662bf --- /dev/null +++ b/fluent-package/templates/package-scripts/fluent-package/deb/prerm @@ -0,0 +1,72 @@ +#!/bin/sh +# prerm script for <%= service_name %>. +# +# See: dh_installdeb(1). + +set -e + +# Summary of how this script can be called: +# * 'remove' +# * 'upgrade' +# * 'failed-upgrade' +# * 'remove' 'in-favour' +# * 'deconfigure' 'in-favour' +# 'removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package. + +make_sure_working_dir_exists() { + # This func is for CI ONLY. + # Some CI cases don't provide env with systemd. + # tmpfiles.d owns /tmp/<%= package_dir %>, but not created without systemd + if [ -d /run/systemd/system ]; then + return + fi + mkdir -p "/tmp/<%= package_dir %>" +} + +leave_info_for_auto_restart_if_need() { + # Some CI cases that don't provide env with systemd, so disable auto restart feature in those cases. + if [ ! -d /run/systemd/system ]; then + return + fi + + pid="$(systemctl show "<%= service_name %>" --property=MainPID --value)" + if [ $pid -eq 0 ]; then + echo "Do not use auto restart because the service is not active" + return + fi + + . "/etc/default/<%= service_name %>" + echo "FLUENT_PACKAGE_SERVICE_RESTART: $FLUENT_PACKAGE_SERVICE_RESTART" + if [ "$FLUENT_PACKAGE_SERVICE_RESTART" = manual ]; then + return + fi + + /usr/sbin/fluent-gem list '^fluent-plugin-' --no-version --no-verbose > "/tmp/<%= package_dir %>/.plugin_list" + + echo "$pid" > "/tmp/<%= package_dir %>/.main_pid" +} + +case "$1" in + upgrade) + make_sure_working_dir_exists + leave_info_for_auto_restart_if_need + ;; + remove|deconfigure) + ;; + failed-upgrade) + ;; + *) + echo "prerm called with unknown argument '$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/fluent-package/test-tools/fluentd.conf b/fluent-package/test-tools/fluentd.conf new file mode 100644 index 000000000..cdfc38423 --- /dev/null +++ b/fluent-package/test-tools/fluentd.conf @@ -0,0 +1,72 @@ + + @type udp + tag test_udp + + port 5170 + bind 0.0.0.0 + + + @type none + + + + + @type tcp + tag test_tcp + + port 5170 + bind 0.0.0.0 + + + @type none + + + + + @type syslog + port 5140 + bind 0.0.0.0 + tag test_syslog + + + + @type file + path /var/log/fluent/test_udp + + + @type file + path /var/log/fluent/test_udp-buffer + flush_method interval + flush_mode interval + flush_interval 10 + flush_at_shutdown true + + + + + @type file + path /var/log/fluent/test_tcp + + + @type file + path /var/log/fluent/test_tcp-buffer + flush_method interval + flush_mode interval + flush_interval 10 + flush_at_shutdown true + + + + + @type file + path /var/log/fluent/test_syslog + + + @type file + path /var/log/fluent/test_syslog-buffer + flush_method interval + flush_mode interval + flush_interval 10 + flush_at_shutdown true + + diff --git a/fluent-package/test-tools/logdata-sender.rb b/fluent-package/test-tools/logdata-sender.rb new file mode 100644 index 000000000..cd79c0ad3 --- /dev/null +++ b/fluent-package/test-tools/logdata-sender.rb @@ -0,0 +1,74 @@ +require "socket" +require "syslog" +require "optparse" + +udp_data_count = 50 +tcp_data_count = 60 +syslog_data_count = 70 +syslog_identifer = "test-syslog" +output_duration_sec = 4.0 + +opt = OptionParser.new +opt.on("--udp-data-count num") { |v| udp_data_count = v.to_i } +opt.on("--tcp-data-count num") { |v| tcp_data_count = v.to_i } +opt.on("--syslog-data-count num") { |v| syslog_data_count = v.to_i } +opt.on("--syslog-identifer name") { |v| syslog_identifer = v } +opt.on("--duration num") { |v| output_duration_sec = v.to_f } +opt.parse!(ARGV) + +threads = [] + +## UDP +threads << Thread.new do + i = 0 + begin + s = UDPSocket.open + s.connect("localhost", 5170) + + loop do + break if i == udp_data_count + s.puts "[udp][#{i}] hello" + i += 1 + sleep (output_duration_sec / udp_data_count) + end + rescue Errno::ECONNRESET, Errno::ECONNREFUSED + retry + end +ensure + s.close +end + +## TCP +threads << Thread.new do + i = 0 + begin + s = TCPSocket.open("localhost", 5170) + + loop do + break if i == tcp_data_count + s.puts "[tcp][#{i}] hello" + i += 1 + sleep (output_duration_sec / tcp_data_count) + end + rescue Errno::ECONNRESET, Errno::ECONNREFUSED + retry + end +ensure + s.close +end + +## Syslog +threads << Thread.new do + Syslog.open(syslog_identifer) + i = 0 + loop do + break if i == syslog_data_count + Syslog.info("[syslog][#{i}] hello") + i += 1 + sleep (output_duration_sec / syslog_data_count) + end +ensure + Syslog.close +end + +threads.each(&:join) diff --git a/fluent-package/test-tools/rsyslog.conf b/fluent-package/test-tools/rsyslog.conf new file mode 100644 index 000000000..e74a06ae6 --- /dev/null +++ b/fluent-package/test-tools/rsyslog.conf @@ -0,0 +1 @@ +user.* @127.0.0.1:5140 diff --git a/fluent-package/yum/fluent-package.spec.in b/fluent-package/yum/fluent-package.spec.in index 0549d7bc7..a204f9cdf 100644 --- a/fluent-package/yum/fluent-package.spec.in +++ b/fluent-package/yum/fluent-package.spec.in @@ -32,6 +32,9 @@ %define v4migration_with_restart /tmp/@PACKAGE_DIR@/.v4migration_with_restart %define v4migration_old_rotate_config_saved /tmp/@PACKAGE_DIR@/.old_rotate_config %define v4migration_enabled_service /tmp/@PACKAGE_DIR@/.v4migration_enabled_service +%define local_base_plugins /tmp/@PACKAGE_DIR@/.local_base_plugins +%define install_plugins /tmp/@PACKAGE_DIR@/.install_plugins +%define pid_for_auto_restart /tmp/@PACKAGE_DIR@/.pid_for_auto_restart # Omit the brp-python-bytecompile automagic because post hook for ffi fails on AmazonLinux 2. %if %{_amazon_ver} == 2 @@ -159,6 +162,14 @@ mkdir -p %{buildroot}%{_sysconfdir}/@PACKAGE_DIR@/plugin mkdir -p %{buildroot}/tmp/@PACKAGE_DIR@ %pre +# Make sure the previous tmp files for auto restart does not remain. +# Note: +# %preun (FROM-side) can create these files, but they will be removed in %posttrans (TO-side). +# This means that these files may not be removed depending on the version of TO-side. +# In the future, want to figure out a more secure way to manage tmp files... +rm -f %{install_plugins} +rm -f %{pid_for_auto_restart} + if ! getent group @COMPAT_SERVICE_NAME@ >/dev/null; then if ! getent group @SERVICE_NAME@ >/dev/null; then /usr/sbin/groupadd --system @SERVICE_NAME@ @@ -186,9 +197,36 @@ else /usr/sbin/useradd --system --uid $TD_UID --gid $TD_GID --home-dir %{_localstatedir}/lib/@PACKAGE_DIR@ --shell /sbin/nologin --non-unique @SERVICE_NAME@ fi fi +if [ $1 -eq 2 ]; then + # Collect plugin-list. + # Note: + # This should be done in %preun(1) of FROM-side, but we have no choice but to do this here. + # %preun(1) of FROM-side is executed after TO-side installs the files and replaces embedded Ruby. + # So, this needs to be done before it. + if [ -e /usr/sbin/fluent-gem ]; then + /usr/sbin/fluent-gem list '^fluent-plugin-' --no-versions --no-verbose > %{local_base_plugins} + fi +fi %preun %systemd_preun @SERVICE_NAME@.service +if [ $1 -eq 1 ]; then + # systemctl ... --property=MainPID --value is available since systemd 230 or later. + # thus for amazonlinux:2, it can not be used. + pid="$(systemctl show "@SERVICE_NAME@" --property=MainPID | cut -d'=' -f2)" + if [ "$pid" -eq 0 ]; then + echo "Do not use auto restart because the service is not active" + else + . %{_sysconfdir}/sysconfig/@SERVICE_NAME@ + echo "FLUENT_PACKAGE_SERVICE_RESTART: $FLUENT_PACKAGE_SERVICE_RESTART" + if [ "$FLUENT_PACKAGE_SERVICE_RESTART" != manual ]; then + # Present that FROM-side wants auto installing plugins and restarting. + # Note: Wants to collect plugin-list here, but we need to do it in %pre (see comments in %pre). + touch %{install_plugins} + echo "$pid" > %{pid_for_auto_restart} + fi + fi +fi %post %systemd_post @SERVICE_NAME@.service @@ -284,7 +322,7 @@ if [ -f "%{_sysconfdir}/prelink.conf" ]; then fi %postun -%systemd_postun_with_restart @SERVICE_NAME@.service +# Disable systemd_postun_with_restart to manage restart on the package side. if [ $1 -eq 0 ]; then # Uninstall # Without this uninstall conditional guard block ($1 -eq 0), symlink @@ -349,6 +387,31 @@ if [ -f %{v4migration} ]; then rm -f %{v4migration_with_restart} fi fi +if [ -f %{install_plugins} ] && [ -f %{local_base_plugins} ]; then + local_current_plugins=$(/usr/sbin/fluent-gem list '^fluent-plugin-' --no-versions --no-verbose) + if ! grep --fixed-strings --line-regexp --invert-match "$local_current_plugins" %{local_base_plugins}; then + echo "No missing plugins to install" + else + if ! curl --fail --silent -O https://rubygems.org/specs.4.8.gz; then + echo "No network connectivity..." + else + grep --fixed-strings --line-regexp --invert-match "$local_current_plugins" %{local_base_plugins} | while read missing_gem + do + if ! /usr/sbin/fluent-gem install --no-document $missing_gem; then + echo "Can't install missing plugin automatically: please install $missing_gem manually." + fi + done + fi + fi +fi +rm -f %{install_plugins} +rm -f %{local_base_plugins} +if [ -f %{pid_for_auto_restart} ]; then + pid=$(cat %{pid_for_auto_restart}) + echo "Kick auto restart to MainPID:$pid" + kill -USR2 $pid + rm -f %{pid_for_auto_restart} +fi %files %doc README.md diff --git a/fluent-package/yum/systemd-test/commonvar.sh b/fluent-package/yum/systemd-test/commonvar.sh index 9a847fd8a..234de734f 100755 --- a/fluent-package/yum/systemd-test/commonvar.sh +++ b/fluent-package/yum/systemd-test/commonvar.sh @@ -3,6 +3,7 @@ distribution=$(cat /etc/system-release-cpe | awk '{print substr($1, index($1, "o"))}' | cut -d: -f2) version=$(cat /etc/system-release-cpe | awk '{print substr($1, index($1, "o"))}' | cut -d: -f4) td_agent_version=4.5.2 +fluent_package_lts_version=5.0.5 case $distribution in amazon) diff --git a/fluent-package/yum/systemd-test/downgrade-to-v5-lts.sh b/fluent-package/yum/systemd-test/downgrade-to-v5-lts.sh new file mode 100755 index 000000000..8c1fa7a50 --- /dev/null +++ b/fluent-package/yum/systemd-test/downgrade-to-v5-lts.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/commonvar.sh + +# Install v5 LTS to register the repository +case $distribution in + amazon) + case $version in + 2023) + curl -fsSL https://toolbelt.treasuredata.com/sh/install-amazon2023-fluent-package5-lts.sh | sh + ;; + 2) + curl -fsSL https://toolbelt.treasuredata.com/sh/install-amazon2-fluent-package5-lts.sh | sh + ;; + esac + ;; + *) + curl -fsSL https://toolbelt.treasuredata.com/sh/install-redhat-fluent-package5-lts.sh | sh + ;; +esac + +sudo $DNF remove -y fluent-package + +# Install the current +sudo $DNF install -y \ + /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm + +# Customize the env file to prevent replacing by downgrade. +# Need this to test the case where some tmp files is left. +echo "FOO=foo" >> /etc/sysconfig/fluentd + +sudo systemctl enable --now fluentd +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent +main_pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Downgrade to v5 LTS +sudo $DNF downgrade -y fluent-package-${fluent_package_lts_version} + +# Test: take over enabled state +systemctl is-enabled fluentd + +# Test: service status +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent + +# Fluentd should NOT be restarted. +# NOTE: Unlike DEB, the restart behavior depends on FROM-side. So, it does not restart. +# (it should restarts only when triggering zerodowntime-restart). +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# === Test: Remained tmp files should not affect to next upgrade === +# (This happens when env file was customized but the FLUENT_PACKAGE_SERVICE_RESTART was still `auto`) + +# Some tmp files remains, though it is not happy. +test -e /tmp/fluent/.install_plugins +test -e /tmp/fluent/.pid_for_auto_restart + +sudo $DNF install -y \ + /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm | tee upgrade.log + +# zerodowntime-restart should NOT be triggered. +(! grep "Kick auto restart" upgrade.log) + +# ====== diff --git a/fluent-package/yum/systemd-test/test.sh b/fluent-package/yum/systemd-test/test.sh index 4a4d58120..4c73e2c30 100755 --- a/fluent-package/yum/systemd-test/test.sh +++ b/fluent-package/yum/systemd-test/test.sh @@ -15,7 +15,7 @@ dir="/host/fluent-package/yum/systemd-test" set -eux echo "::group::Run test: launch $image" -lxc launch $image target +lxc launch $image target --debug sleep 5 echo "::endgroup::" echo "::group::Run test: configure $image" diff --git a/fluent-package/yum/systemd-test/update-from-v5-lts.sh b/fluent-package/yum/systemd-test/update-from-v5-lts.sh new file mode 100755 index 000000000..b51b0959c --- /dev/null +++ b/fluent-package/yum/systemd-test/update-from-v5-lts.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/commonvar.sh + +# If it update from v5 LTS without stopping Fluentd, Fluentd will be restarted. +# Install v5 LTS +case $distribution in + amazon) + case $version in + 2023) + curl -fsSL https://toolbelt.treasuredata.com/sh/install-amazon2023-fluent-package5-lts.sh | sh + ;; + 2) + curl -fsSL https://toolbelt.treasuredata.com/sh/install-amazon2-fluent-package5-lts.sh | sh + ;; + esac + ;; + *) + curl -fsSL https://toolbelt.treasuredata.com/sh/install-redhat-fluent-package5-lts.sh | sh + ;; +esac + +sudo systemctl enable --now fluentd +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent +main_pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Install the current +sudo $DNF install -y \ + /host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm + +# Test: take over enabled state +systemctl is-enabled fluentd + +# Test: service status +systemctl status --no-pager fluentd +systemctl status --no-pager td-agent + +# Fluentd should be restarted when update from v5 LTS. +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Test: environmental variables +pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) +env_vars=$(sudo sed -e 's/\x0/\n/g' /proc/$pid/environ) +test $(eval $env_vars && echo $HOME) = "/var/lib/fluent" +test $(eval $env_vars && echo $LOGNAME) = "fluentd" +test $(eval $env_vars && echo $USER) = "fluentd" +test $(eval $env_vars && echo $FLUENT_CONF) = "/etc/fluent/fluentd.conf" +test $(eval $env_vars && echo $FLUENT_PACKAGE_LOG_FILE) = "/var/log/fluent/fluentd.log" +test $(eval $env_vars && echo $FLUENT_PLUGIN) = "/etc/fluent/plugin" +test $(eval $env_vars && echo $FLUENT_SOCKET) = "/var/run/fluent/fluentd.sock" + +# Test: No error logs +# (v5 default config outputs 'warn' log, so we should check only 'error' and 'fatal' logs) +sleep 3 +test -e /var/log/fluent/fluentd.log +(! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) + +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd) +(! sudo /usr/sbin/fluentd -v) +sudo /usr/sbin/fluentd --dry-run + +# Uninstall +sudo $DNF remove -y fluent-package +sudo systemctl daemon-reload + +getent passwd fluentd >/dev/null +getent group fluentd >/dev/null + +# `sudo systemctl daemon-reload` clears the service completely. +# (The result of `systemctl status` will be `unfound`) +# Note: RPM does not leave links like `@/etc/systemd/system/fluentd.service`. +# (Different from deb) + +(! systemctl status --no-pager td-agent) +(! systemctl status --no-pager fluentd) diff --git a/fluent-package/yum/systemd-test/update-to-next-major-version.sh b/fluent-package/yum/systemd-test/update-to-next-major-version.sh new file mode 100755 index 000000000..f2a888de1 --- /dev/null +++ b/fluent-package/yum/systemd-test/update-to-next-major-version.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/commonvar.sh + +service_restart=$1 +status_before_update=$2 # active / inactive + +# Install the current +package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-*.rpm" +sudo $DNF install -y $package + +if [ "$status_before_update" = active ]; then + sudo systemctl start fluentd +fi + +# Set FLUENT_PACKAGE_SERVICE_RESTART +sed -i "s/=auto/=$service_restart/" /etc/sysconfig/fluentd + +# Install plugin manually (plugin and gem) +sudo /opt/fluent/bin/fluent-gem install --no-document fluent-plugin-concat +sudo /opt/fluent/bin/fluent-gem install --no-document gqtp + +# Install next major version +package="/host/v6-test/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-*.rpm" +sudo $DNF install -y $package + +# Test: Check whether plugin/gem were installed during upgrading +if [ "$service_restart" != manual ] && [ "$status_before_update" = active ]; then + # plugin gem should be installed automatically + /opt/fluent/bin/fluent-gem list | grep fluent-plugin-concat + # Non fluent-plugin- prefix gem should not be installed automatically + (! /opt/fluent/bin/fluent-gem list | grep gqtp) +else + # plugin gem should not be installed automatically + (! /opt/fluent/bin/fluent-gem list | grep fluent-plugin-concat) +fi diff --git a/fluent-package/yum/systemd-test/update-to-next-version-service-status.sh b/fluent-package/yum/systemd-test/update-to-next-version-service-status.sh index 845560e2c..48bd7b7c8 100755 --- a/fluent-package/yum/systemd-test/update-to-next-version-service-status.sh +++ b/fluent-package/yum/systemd-test/update-to-next-version-service-status.sh @@ -65,9 +65,10 @@ else fi if [ "$status_before_update" = active ]; then - # The service should restart automatically after update + # The service should NOT restart automatically after update + # (The process before update should continue to run) systemctl is-active fluentd - test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) else # The service should NOT start automatically (! systemctl is-active fluentd) diff --git a/fluent-package/yum/systemd-test/update-to-next-version-with-auto-and-manual.sh b/fluent-package/yum/systemd-test/update-to-next-version-with-auto-and-manual.sh new file mode 100755 index 000000000..03db0da23 --- /dev/null +++ b/fluent-package/yum/systemd-test/update-to-next-version-with-auto-and-manual.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/commonvar.sh + +package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-[0-9]*.rpm" + +# Make a dummy pacakge for the next version +case $distribution in + amazon) + case $version in + 2023) + curl -L -o rpmrebuild.noarch.rpm https://sourceforge.net/projects/rpmrebuild/files/latest/download + sudo $DNF install -y ./rpmrebuild.noarch.rpm + ;; + 2) + sudo amazon-linux-extras install -y epel + sudo $DNF install -y rpmrebuild + ;; + esac + ;; + *) + sudo $DNF install -y epel-release + sudo $DNF install -y rpmrebuild + ;; +esac + + +# Example: "1.el9" +release=$(rpmquery --queryformat="%{Release}" -p $package) +# Example: "1" +release_ver=$(echo $release | cut -d . -f1) +# Example: "2.el9" +next_release=$(($release_ver+1)).$(echo $release | cut -d. -f2) +rpmrebuild --release=$next_release --modify="find $HOME -name fluentd.service | xargs sed -i -E 's/FLUENT_PACKAGE_VERSION=([0-9.]+)/FLUENT_PACKAGE_VERSION=\1.1/g'" --package $package +next_package=$(find rpmbuild -name "*.rpm") +rpm2cpio $next_package | cpio -id ./usr/lib/systemd/system/fluentd.service +next_package_ver=$(cat ./usr/lib/systemd/system/fluentd.service | grep "FLUENT_PACKAGE_VERSION" | sed -E "s/Environment=FLUENT_PACKAGE_VERSION=(.+)/\1/") +echo "repacked next fluent-package version: $next_package_ver" + +# Upgrade package with auto feature +sudo $DNF install -y $package +sudo systemctl enable --now fluentd +main_pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +sudo $DNF install -y ./$next_package +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Main process should be replaced by USR2 signal in auto mode +sleep 15 +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +sudo $DNF remove -y fluent-package + +# Upgrade package with manual feature +sudo $DNF install -y $package +sudo systemctl enable --now fluentd +sed -i 's/=auto/=manual/' /etc/sysconfig/fluentd +main_pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +sudo $DNF install -y ./$next_package +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Main process should NOT be replaced until USR2 signal fired +sleep 15 +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +kill -USR2 $main_pid + +# Main process should be replaced by USR2 signal +sleep 15 +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) diff --git a/fluent-package/yum/systemd-test/update-without-data-lost.sh b/fluent-package/yum/systemd-test/update-without-data-lost.sh new file mode 100755 index 000000000..f63a86cd8 --- /dev/null +++ b/fluent-package/yum/systemd-test/update-without-data-lost.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/commonvar.sh + +v5_package="/host/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-*.rpm" +v6_package="/host/v6-test/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/fluent-package-*.rpm" + +case "$1" in + v5) + package=$v5_package + ;; + v6) + package=$v6_package + ;; + *) + echo "Invalid argument: $1" + exit 1 + ;; +esac + +command="install" +case "$2" in + v5) + next_package=$v5_package + command="downgrade" # Avoid error in AmazonLinux2 + ;; + v6) + next_package=$v6_package + ;; + *) + echo "Invalid argument: $2" + exit 1 + ;; +esac + +sudo $DNF install -y rsyslog + +# Install the current +sudo $DNF install -y $package + +# Set up configuration +cat < $(dirname $0)/../../test-tools/rsyslog.conf >> /etc/rsyslog.conf +cp $(dirname $0)/../../test-tools/fluentd.conf /etc/fluent/fluentd.conf + +# Launch rsyslog +sudo systemctl restart rsyslog + +# Launch fluentd +sudo systemctl enable --now fluentd +main_pid=$(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Ensure to wait for fluentd launching +sleep 1 + +# Send logs in background for 4 seconds +/opt/fluent/bin/ruby $(dirname $0)/../../test-tools/logdata-sender.rb \ + --udp-data-count 50 --tcp-data-count 60 --syslog-data-count 70 --syslog-identifer "test-syslog" --duration 16 & + +sleep 1 + +# Update to the next version +sudo $DNF $command -y $next_package +test $main_pid -eq $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Main process should be replaced by USR2 signal +sleep 20 +test $main_pid -ne $(eval $(systemctl show fluentd --property=MainPID) && echo $MainPID) + +# Stop fluentd to flush the logs and check +systemctl stop fluentd +test $(wc -l /var/log/fluent/test_udp*.log | tail -n 1 | awk '{print $1}') = "50" +test $(wc -l /var/log/fluent/test_tcp*.log | tail -n 1 | awk '{print $1}') = "60" +test $(grep "test-syslog" /var/log/fluent/test_syslog*.log | wc -l) = "70"