diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da6c43e66..05ec99de0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -614,6 +614,10 @@ jobs: run: just -v tag=$env:VERSION object=local worker=htcmock ingress=false prometheus=false grafana=false seq=false queue=rabbitmq091 deploy shell: powershell + - name: Show health status + if: always() + run: docker inspect armonik.compute.pollingagent0 | jq .[].State.Health + - name: Run HtcMock test 100 tasks 1 level timeout-minutes: 3 shell: powershell diff --git a/ArmoniK.Core.sln b/ArmoniK.Core.sln index 05b8c8138..e00ce4450 100644 --- a/ArmoniK.Core.sln +++ b/ArmoniK.Core.sln @@ -101,6 +101,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Tests.Connecti EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Adapters.SQS", "Adaptors\SQS\src\ArmoniK.Core.Adapters.SQS.csproj", "{399C779C-CE8D-4757-8098-7F055467D96C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArmoniK.Core.HealthChecker", "HealthChecker\src\ArmoniK.Core.HealthChecker.csproj", "{3F1B3424-D1FC-4E60-9C87-81A7E17E2008}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -413,6 +415,14 @@ Global {399C779C-CE8D-4757-8098-7F055467D96C}.Release|Any CPU.Build.0 = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.ActiveCfg = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.Build.0 = Release|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Debug|x64.ActiveCfg = Debug|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Debug|x64.Build.0 = Debug|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Release|Any CPU.Build.0 = Release|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Release|x64.ActiveCfg = Release|Any CPU + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -465,6 +475,7 @@ Global {9F19E6DD-D9CD-43EC-B5CC-0081997ED8B8} = {6EE499A5-760F-4823-8AB6-DBDE8C5B5F45} {6ABF29BB-2F42-4088-AE93-E41416CB3271} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} {399C779C-CE8D-4757-8098-7F055467D96C} = {1A9BCE53-79D0-4761-B3A2-6967B610FA94} + {3F1B3424-D1FC-4E60-9C87-81A7E17E2008} = {79DE0448-1BF2-48D2-8A33-243E402CFD27} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1285A466-2AF6-43E6-8DCC-2F93A5D5F02E} diff --git a/Dockerfile b/Dockerfile index f8918d636..96c6163cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ COPY ["Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj", "Comp COPY ["Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj", "Control/Metrics/src/"] COPY ["Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj", "Control/PartitionMetrics/src/"] COPY ["Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj", "Control/Submitter/src/"] +COPY ["HealthChecker/src/ArmoniK.Core.HealthChecker.csproj", "HealthChecker/src/"] COPY ["Utils/src/ArmoniK.Core.Utils.csproj", "Utils/src/"] RUN dotnet restore -a "${TARGETARCH}" "Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj" @@ -40,6 +41,7 @@ RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Amqp/src/ArmoniK.Core.Adapters.A RUN dotnet restore -a "${TARGETARCH}" "Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj" +RUN dotnet restore -a "${TARGETARCH}" "HealthChecker/src/ArmoniK.Core.HealthChecker.csproj" # git ls-tree -r HEAD --name-only --full-tree | grep "csproj$" | xargs -I % sh -c "export D=\$(dirname %) ; echo COPY [\\\"\$D\\\", \\\"\$D\\\"]" COPY ["Adaptors/Amqp/src", "Adaptors/Amqp/src"] @@ -58,6 +60,7 @@ COPY ["Compute/PollingAgent/src", "Compute/PollingAgent/src"] COPY ["Control/Metrics/src", "Control/Metrics/src"] COPY ["Control/PartitionMetrics/src", "Control/PartitionMetrics/src"] COPY ["Control/Submitter/src", "Control/Submitter/src"] +COPY ["HealthChecker/src", "HealthChecker/src"] COPY ["Utils/src", "Utils/src"] WORKDIR /src/Adaptors/SQS/src @@ -84,6 +87,9 @@ RUN dotnet publish "ArmoniK.Core.Control.PartitionMetrics.csproj" -a "${TARGETAR WORKDIR /src/Control/Submitter/src RUN dotnet publish "ArmoniK.Core.Control.Submitter.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/submitter /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION +WORKDIR /src/HealthChecker/src +RUN dotnet publish "ArmoniK.Core.HealthChecker.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/healthchecker -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION +RUN [ -e /app/publish/healthchecker/ArmoniK.Core.HealthChecker.exe ] || mv /app/publish/healthchecker/ArmoniK.Core.HealthChecker /app/publish/healthchecker/ArmoniK.Core.HealthChecker.exe FROM base-${TARGETOS} as polling_agent WORKDIR /adapters/queue/sqs @@ -94,6 +100,8 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /healthchecker +COPY --from=build /app/publish/healthchecker . WORKDIR /app COPY --from=build /app/publish/polling_agent . ENV ASPNETCORE_URLS http://+:1080 @@ -102,6 +110,8 @@ CMD ["ArmoniK.Core.Compute.PollingAgent.dll"] FROM base-${TARGETOS} as metrics +WORKDIR /healthchecker +COPY --from=build /app/publish/healthchecker . WORKDIR /app COPY --from=build /app/publish/metrics . ENV ASPNETCORE_URLS http://+:1080 @@ -110,6 +120,8 @@ CMD ["ArmoniK.Core.Control.Metrics.dll"] FROM base-${TARGETOS} as partition_metrics +WORKDIR /healthchecker +COPY --from=build /app/publish/healthchecker . WORKDIR /app COPY --from=build /app/publish/partition_metrics . ENV ASPNETCORE_URLS http://+:1080 @@ -126,6 +138,8 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /healthchecker +COPY --from=build /app/publish/healthchecker . WORKDIR /app COPY --from=build /app/publish/submitter . ENV ASPNETCORE_URLS http://+:1080, http://+:1081 diff --git a/HealthChecker/src/ArmoniK.Core.HealthChecker.csproj b/HealthChecker/src/ArmoniK.Core.HealthChecker.csproj new file mode 100644 index 000000000..0c7bd255b --- /dev/null +++ b/HealthChecker/src/ArmoniK.Core.HealthChecker.csproj @@ -0,0 +1,26 @@ + + + + net8.0 + win-x64;linux-x64;linux-arm64 + ANEO + Copyright (C) ANEO, 2021-2021 + AGPL-3.0-or-later + True + true + enable + Exe + + + + Embedded + true + DEBUG;TRACE + + + + true + true + snupkg + + diff --git a/HealthChecker/src/ArmoniK.Core.HealthChecker.csproj.DotSettings b/HealthChecker/src/ArmoniK.Core.HealthChecker.csproj.DotSettings new file mode 100644 index 000000000..89316e414 --- /dev/null +++ b/HealthChecker/src/ArmoniK.Core.HealthChecker.csproj.DotSettings @@ -0,0 +1,2 @@ + + Library \ No newline at end of file diff --git a/HealthChecker/src/Program.cs b/HealthChecker/src/Program.cs new file mode 100644 index 000000000..bf9d45556 --- /dev/null +++ b/HealthChecker/src/Program.cs @@ -0,0 +1,63 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace ArmoniK.Core.HealthChecker; + +public static class Program +{ + public static async Task Main(string[] args) + { + try + { + string uri; + switch (args.Length) + { + case 0: + uri = "http://localhost:1080/liveness"; + break; + case 1: + uri = args[0]; + break; + default: + Console.WriteLine("Too many arguments"); + return 1; + } + + var client = new HttpClient(); + var response = await client.GetStringAsync(uri) + .ConfigureAwait(false); + + switch (response) + { + case "Healthy": + return 0; + default: + Console.WriteLine("Received:" + response); + return 1; + } + } + catch (Exception e) + { + Console.WriteLine(e); + return 1; + } + } +} diff --git a/terraform/modules/compute_plane/main.tf b/terraform/modules/compute_plane/main.tf index 399f23190..65872c2b1 100644 --- a/terraform/modules/compute_plane/main.tf +++ b/terraform/modules/compute_plane/main.tf @@ -64,6 +64,7 @@ resource "docker_container" "polling_agent" { } restart = "unless-stopped" + wait = true dynamic "mounts" { for_each = var.volumes @@ -74,5 +75,13 @@ resource "docker_container" "polling_agent" { } } + healthcheck { + test = ["CMD", "/healthchecker/ArmoniK.Core.HealthChecker.exe", "http://127.0.0.1:1080/liveness"] + interval = "5s" + timeout = "3s" + start_period = "20s" + retries = 5 + } + depends_on = [docker_container.worker] } diff --git a/terraform/modules/monitoring/metrics/main.tf b/terraform/modules/monitoring/metrics/main.tf index 531674fbb..fb40aa43c 100644 --- a/terraform/modules/monitoring/metrics/main.tf +++ b/terraform/modules/monitoring/metrics/main.tf @@ -20,4 +20,13 @@ resource "docker_container" "metrics" { internal = 1080 external = var.exposed_port } + + wait = true + healthcheck { + test = ["CMD", "/healthchecker/ArmoniK.Core.HealthChecker.exe"] + interval = "5s" + timeout = "3s" + start_period = "20s" + retries = 5 + } } diff --git a/terraform/modules/submitter/main.tf b/terraform/modules/submitter/main.tf index 5f6f1c2e4..3e07f08f6 100644 --- a/terraform/modules/submitter/main.tf +++ b/terraform/modules/submitter/main.tf @@ -34,4 +34,13 @@ resource "docker_container" "submitter" { source = mounts.key } } + + wait = true + healthcheck { + test = ["CMD", "/healthchecker/ArmoniK.Core.HealthChecker.exe", "http://127.0.0.1:1081/liveness"] + interval = "5s" + timeout = "3s" + start_period = "20s" + retries = 5 + } }