diff --git a/charts/nudgebee-agent/values.yaml b/charts/nudgebee-agent/values.yaml index 196e02d9..e3612023 100644 --- a/charts/nudgebee-agent/values.yaml +++ b/charts/nudgebee-agent/values.yaml @@ -65,7 +65,7 @@ runnerServiceAccount: runner: image: repository: ghcr.io/nudgebee/nudgebee-agent - tag: 2026-08-20T09-53-59_1e7426c7f4b8555ec2c620ac834c256ea1aceac7 + tag: 2026-08-20T12-43-13_7a2f3c93f2575994b793bb376c5b7e1f5c60d7ed # Image template the pod_profiler action launches debugger pods from. # The agent substitutes `{}` for the variant (bpf, jvm, python, perf, ruby). # Surfaces as PROFILER_IMAGE; leave empty to fall back to the binary default. diff --git a/runner/cmd/agent/main.go b/runner/cmd/agent/main.go index bafb2ec8..603927e6 100644 --- a/runner/cmd/agent/main.go +++ b/runner/cmd/agent/main.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "maps" "net/http" @@ -991,9 +992,10 @@ func run(ctx context.Context, logger *slog.Logger, cfg *config.Config) error { probeCtx, probeCancel := context.WithTimeout(gctx, 30*time.Second) defer probeCancel() probeClient := &http.Client{Timeout: 5 * time.Second} - logsProvider, logsURL, logsOK, logCfg := probeLogsProvider(probeCtx, cfg) + logsProvider, logsURL, logsOK, logsErr, logCfg := probeLogsProvider(probeCtx, cfg) as := telemetry.DetectAutoScaler(probeCtx, typedKube, providerInfo.Provider, logger) clickhouseStatus, clickhouseErr := probeClickhouse(probeCtx, probeClient, clickhouseHost, clickhousePort) + promConnected, promErr := prometheusConnected(probeCtx, promClient, logger) return telemetry.Datasources{ PrometheusURL: cfg.PrometheusURL, AlertManagerURL: cfg.AlertManagerURL, @@ -1002,8 +1004,10 @@ func run(ctx context.Context, logger *slog.Logger, cfg *config.Config) error { LogsProvider: logsProvider, LogsProviderURL: logsURL, LogsProviderStatus: logsOK, + LogsProviderError: logsErr, LogProviderConfig: logCfg, - PrometheusConnected: prometheusConnected(probeCtx, promClient, logger), + PrometheusConnected: promConnected, + PrometheusConnectedError: promErr, NodeAgentCount: queryNodeAgentCount(probeCtx, promClient, logger), PrometheusRetentionTime: telemetry.PrometheusRetention(probeCtx, promClient, logger), PrometheusAdditionalLabels: promExtraLabels, @@ -1169,24 +1173,27 @@ func (a *grafanaAdapter) HandlePrometheus(ctx context.Context, r *dispatch.Grafa // auth — Chronosphere, Thanos Query, Grafana Mimir, Amazon Managed Prometheus — // are reported Connected when metric queries work. Returns false on any error // so a broken backend shows Disconnected rather than panicking the tick. -func prometheusConnected(ctx context.Context, c *prometheus.Client, logger *slog.Logger) bool { +func prometheusConnected(ctx context.Context, c *prometheus.Client, logger *slog.Logger) (ok bool, reason string) { if c == nil || c.BaseURL == "" { - return false + return false, "" } cctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() raw, err := c.Query(cctx, "vector(1)", "", "") if err != nil { logger.Debug("prometheus health query failed", "err", err) - return false + return false, err.Error() } var resp struct { Status string `json:"status"` } if err := json.Unmarshal(raw, &resp); err != nil { - return false + return false, err.Error() + } + if resp.Status == "success" { + return true, "" } - return resp.Status == "success" + return false, fmt.Sprintf("prometheus query returned status %q", resp.Status) } // queryNodeAgentCount @@ -1249,46 +1256,55 @@ func selectedLogsProvider(cfg *config.Config) (provider, url string) { // configured provider's own client at action-handler time. Fail-closed: any // non-2xx → status=false, URL stays in payload so the UI can show "URL // configured but unhealthy". -func probeLogsProvider(ctx context.Context, cfg *config.Config) (provider, url string, ok bool, providerCfg map[string]any) { +func probeLogsProvider(ctx context.Context, cfg *config.Config) (provider, url string, ok bool, reason string, providerCfg map[string]any) { httpClient := &http.Client{Timeout: 5 * time.Second} switch { case cfg.PinotURL != "": - ok = httpProbe(ctx, httpClient, cfg.PinotURL+"/health") - return "pinot", cfg.PinotURL, ok, map[string]any{} + err := httpProbeErr(ctx, httpClient, cfg.PinotURL+"/health") + return "pinot", cfg.PinotURL, err == nil, errString(err), map[string]any{} case cfg.ElasticsearchEnabled && cfg.ElasticsearchURL != "": // ES exposes a `_cluster/health` endpoint; we treat 200 as healthy. // Probe with the configured credentials so the badge reflects whether // queries will actually succeed — a secured OpenSearch/ES otherwise 401s // on an unauthenticated probe even when the configured creds work fine. - ok = httpProbe(ctx, httpClient, cfg.ElasticsearchURL+"/_cluster/health", esAuthHeader(cfg)) + err := httpProbeErr(ctx, httpClient, cfg.ElasticsearchURL+"/_cluster/health", esAuthHeader(cfg)) providerCfg = map[string]any{} if v := os.Getenv("ELASTICSEARCH_LOG_INDEX"); v != "" { providerCfg["default_index"] = v } - return "ES", cfg.ElasticsearchURL, ok, providerCfg + return "ES", cfg.ElasticsearchURL, err == nil, errString(err), providerCfg case cfg.SignozURL != "": // Signoz health endpoint: /api/v1/health. - ok = httpProbe(ctx, httpClient, cfg.SignozURL+"/api/v1/health") + err := httpProbeErr(ctx, httpClient, cfg.SignozURL+"/api/v1/health") providerCfg = map[string]any{} // Report the Signoz server version so the backend/UI can surface it // and gate version-specific behaviour. /api/v1/version is unauthed. if v := fetchSignozVersion(ctx, httpClient, cfg.SignozURL); v != "" { providerCfg["version"] = v } - return "signoz", cfg.SignozURL, ok, providerCfg + return "signoz", cfg.SignozURL, err == nil, errString(err), providerCfg case cfg.LokiURL != "": // LOKI_URL points at the loki gateway, whose nginx only proxies the // `/loki/...` API paths — the backend `/ready` is not exposed there and // 404s. Probe a gateway-served API endpoint instead so the badge // reflects query reachability. - ok = httpProbe(ctx, httpClient, cfg.LokiURL+"/loki/api/v1/status/buildinfo") + err := httpProbeErr(ctx, httpClient, cfg.LokiURL+"/loki/api/v1/status/buildinfo") providerCfg = map[string]any{"url": cfg.LokiURL} - return "loki", cfg.LokiURL, ok, providerCfg + return "loki", cfg.LokiURL, err == nil, errString(err), providerCfg default: - return "", "", false, map[string]any{} + return "", "", false, "", map[string]any{} } } +// errString renders a probe failure for the health UI: empty when healthy so +// the wire field clears, the error text otherwise. +func errString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + // probeClickhouse mirrors the legacy _check_clickhouse → db.health() probe. // Returns false (without probing) when CLICKHOUSE_HOST is unset — the Helm // chart only wires the host when clickhouse/otel-collector is enabled, so @@ -1394,8 +1410,21 @@ func httpProbeErr(ctx context.Context, c *http.Client, url string, headers ...ma } defer func() { _ = resp.Body.Close() }() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return fmt.Errorf("HTTP %d", resp.StatusCode) + // Include a compact body snippet (whitespace collapsed, truncated) — + // backends put the useful detail ("token is expired", CORS/auth pages) + // in the body, and the UI renders this string verbatim. + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + msg := strings.Join(strings.Fields(string(body)), " ") + if len(msg) > 200 { + msg = msg[:200] + "…" + } + if msg == "" { + return fmt.Errorf("HTTP %d", resp.StatusCode) + } + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, msg) } + // Drain the body so the transport can reuse the keep-alive connection. + _, _ = io.Copy(io.Discard, resp.Body) return nil } diff --git a/runner/cmd/agent/probe_test.go b/runner/cmd/agent/probe_test.go index ca24fe9c..c2c65f5a 100644 --- a/runner/cmd/agent/probe_test.go +++ b/runner/cmd/agent/probe_test.go @@ -34,7 +34,7 @@ func TestProbeLogsProvider_ESDisabledFallsThroughToLoki(t *testing.T) { ElasticsearchEnabled: false, LokiURL: loki.URL, } - provider, url, ok, _ := probeLogsProvider(context.Background(), cfg) + provider, url, ok, _, _ := probeLogsProvider(context.Background(), cfg) if provider != "loki" { t.Fatalf("provider = %q, want loki", provider) } @@ -67,7 +67,7 @@ func TestProbeLogsProvider_StrayESURLDoesNotMaskSignoz(t *testing.T) { ElasticsearchEnabled: false, SignozURL: signoz.URL, } - provider, url, ok, _ := probeLogsProvider(context.Background(), cfg) + provider, url, ok, _, _ := probeLogsProvider(context.Background(), cfg) if provider != "signoz" { t.Fatalf("provider = %q, want signoz (stray ES URL must not mask SigNoz)", provider) } @@ -97,7 +97,7 @@ func TestProbeLogsProvider_ESProbeSendsAuth(t *testing.T) { ElasticsearchUser: "admin", ElasticsearchPassword: "pw", } - provider, _, ok, _ := probeLogsProvider(context.Background(), cfg) + provider, _, ok, _, _ := probeLogsProvider(context.Background(), cfg) if provider != "ES" { t.Fatalf("provider = %q, want ES", provider) } @@ -134,7 +134,7 @@ func TestPrometheusConnected_ChronosphereStyleBackend(t *testing.T) { c := prometheus.New(srv.URL, nil) c.ExtraHeaders = config.ParseHeaders("Authorization: Bearer tok") - if !prometheusConnected(context.Background(), c, slog.Default()) { + if ok, _ := prometheusConnected(context.Background(), c, slog.Default()); !ok { t.Error("expected connected=true for query-only backend serving /api/v1/query") } if !queried { @@ -151,14 +151,16 @@ func TestPrometheusConnected_FailuresReportDisconnected(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) defer down.Close() - if prometheusConnected(context.Background(), prometheus.New(down.URL, nil), slog.Default()) { + if ok, reason := prometheusConnected(context.Background(), prometheus.New(down.URL, nil), slog.Default()); ok { t.Error("expected connected=false when backend returns 500") + } else if reason == "" { + t.Error("expected a non-empty failure reason when backend returns 500") } // Nil / unconfigured client → not connected, no panic. - if prometheusConnected(context.Background(), nil, slog.Default()) { + if ok, _ := prometheusConnected(context.Background(), nil, slog.Default()); ok { t.Error("expected connected=false for nil client") } - if prometheusConnected(context.Background(), prometheus.New("", nil), slog.Default()) { + if ok, _ := prometheusConnected(context.Background(), prometheus.New("", nil), slog.Default()); ok { t.Error("expected connected=false for empty base URL") } } diff --git a/runner/pkg/telemetry/service.go b/runner/pkg/telemetry/service.go index ea0f8c90..67774cd6 100644 --- a/runner/pkg/telemetry/service.go +++ b/runner/pkg/telemetry/service.go @@ -38,34 +38,47 @@ import ( // We keep `omitempty` on the strings/maps and explicit emit on // numerics/bools so a `false`/`0` is always present. type ActivityStats struct { - AlertManagerConnection bool `json:"alertManagerConnection"` - PrometheusConnection bool `json:"prometheusConnection"` - PrometheusRetentionTime string `json:"prometheusRetentionTime,omitempty"` - TracesEnabled bool `json:"tracesEnabled"` - LogsConnectionProvider string `json:"logsConnectionProvider,omitempty"` - LogsConnection bool `json:"logsConnection"` - NodeAgentConnection bool `json:"nodeAgentConnection"` - NodeAgentCount int `json:"nodeAgentCount"` - OpencostConnection bool `json:"opencostConnection"` - GrafanaEnabled bool `json:"grafanaEnabled"` - Errors []string `json:"errors"` - InstallationNamespace string `json:"installationNamespace,omitempty"` - LogProviderConfig map[string]any `json:"log_provider_config,omitempty"` - LogProviderURL string `json:"logProviderUrl,omitempty"` - PrometheusURL string `json:"prometheusUrl,omitempty"` - PrometheusAdditionalLabels map[string]string `json:"prometheusAdditionalLabels,omitempty"` - AlertManagerURL string `json:"alertmanagerUrl,omitempty"` - OpencostURL string `json:"opencostUrl,omitempty"` - TracesURL string `json:"tracesUrl,omitempty"` - AutoScalerVersion string `json:"autoScalerVersion,omitempty"` - AutoScalerEnabled bool `json:"autoScalerEnabled"` - AutoScalerNamespace string `json:"autoScalerNamespace,omitempty"` - AutoScalerType string `json:"autoScalerType,omitempty"` - AgentURL string `json:"agentUrl,omitempty"` - AgentWSEnabled bool `json:"agentWSEnabled"` - HealthCheckDuration float64 `json:"healthCheckDuration,omitempty"` - TraceProvider string `json:"traceProvider,omitempty"` - TraceProviderConfig map[string]any `json:"traceProviderConfig,omitempty"` + AlertManagerConnection bool `json:"alertManagerConnection"` + PrometheusConnection bool `json:"prometheusConnection"` + PrometheusRetentionTime string `json:"prometheusRetentionTime,omitempty"` + TracesEnabled bool `json:"tracesEnabled"` + LogsConnectionProvider string `json:"logsConnectionProvider,omitempty"` + LogsConnection bool `json:"logsConnection"` + NodeAgentConnection bool `json:"nodeAgentConnection"` + NodeAgentCount int `json:"nodeAgentCount"` + OpencostConnection bool `json:"opencostConnection"` + GrafanaEnabled bool `json:"grafanaEnabled"` + Errors []string `json:"errors"` + InstallationNamespace string `json:"installationNamespace,omitempty"` + // Per-feature health-check failure reasons, populated only when the + // corresponding *Connection probe failed (e.g. "HTTP 401: token is + // expired"). The UI renders these next to the "Disconnected" status. + // + // Like TracesConnectionError below, deliberately no `omitempty`: the + // collector merges activity_stats into `agent.connection_status` with the + // jsonb `||` operator, so an omitted key would strand the last failure + // reason in the DB after the datasource recovers — an explicit "" is + // required to clear it. + PrometheusConnectionError string `json:"prometheusConnectionError"` + AlertManagerConnectionError string `json:"alertManagerConnectionError"` + LogsConnectionError string `json:"logsConnectionError"` + OpencostConnectionError string `json:"opencostConnectionError"` + LogProviderConfig map[string]any `json:"log_provider_config,omitempty"` + LogProviderURL string `json:"logProviderUrl,omitempty"` + PrometheusURL string `json:"prometheusUrl,omitempty"` + PrometheusAdditionalLabels map[string]string `json:"prometheusAdditionalLabels,omitempty"` + AlertManagerURL string `json:"alertmanagerUrl,omitempty"` + OpencostURL string `json:"opencostUrl,omitempty"` + TracesURL string `json:"tracesUrl,omitempty"` + AutoScalerVersion string `json:"autoScalerVersion,omitempty"` + AutoScalerEnabled bool `json:"autoScalerEnabled"` + AutoScalerNamespace string `json:"autoScalerNamespace,omitempty"` + AutoScalerType string `json:"autoScalerType,omitempty"` + AgentURL string `json:"agentUrl,omitempty"` + AgentWSEnabled bool `json:"agentWSEnabled"` + HealthCheckDuration float64 `json:"healthCheckDuration,omitempty"` + TraceProvider string `json:"traceProvider,omitempty"` + TraceProviderConfig map[string]any `json:"traceProviderConfig,omitempty"` // TracesConnectionError is the reason traces are disconnected, rendered by // the UI's Agent Health card under the Traces pill ("Reason - ..."). // @@ -125,7 +138,11 @@ type Datasources struct { LogsProvider string // "ES" | "signoz" | "loki" | "" LogsProviderURL string LogsProviderStatus bool - LogProviderConfig map[string]any + // LogsProviderError is the health-probe failure reason for the logs + // provider, set by the caller when LogsProviderStatus is false (e.g. + // "HTTP 401: token is expired"). Empty when healthy or unconfigured. + LogsProviderError string + LogProviderConfig map[string]any // PrometheusConnected is the caller-computed connectivity result: an // authenticated `vector(1)` query via the prometheus client (which carries @@ -136,6 +153,10 @@ type Datasources struct { // admin endpoint and require auth, so the old unauthenticated /-/healthy // probe reported them Disconnected even when queries worked. PrometheusConnected bool + // PrometheusConnectedError is the failure reason set by the caller when + // PrometheusConnected is false (query error / non-success status). Empty + // when connected or unconfigured. Surfaced next to "Disconnected" in the UI. + PrometheusConnectedError string // Prometheus retention from `flags.retentionTime` (utils.get_prometheus_flags). PrometheusRetentionTime string @@ -347,6 +368,7 @@ func (s *Service) probe(ctx context.Context, ds Datasources) ActivityStats { LogProviderConfig: ds.LogProviderConfig, LogsConnectionProvider: ds.LogsProvider, LogsConnection: ds.LogsProviderStatus, + LogsConnectionError: ds.LogsProviderError, NodeAgentCount: ds.NodeAgentCount, NodeAgentConnection: ds.NodeAgentCount > 0, // line 254: count > 0 OpencostURL: ds.OpencostURL, @@ -362,15 +384,16 @@ func (s *Service) probe(ctx context.Context, ds Datasources) ActivityStats { // query (see Datasources.PrometheusConnected) so query-only backends that // don't serve /-/healthy (Chronosphere/Thanos/Mimir/AMP) report correctly. out.PrometheusConnection = ds.PrometheusConnected + out.PrometheusConnectionError = ds.PrometheusConnectedError // AlertManager: HTTP /-/healthy — the endpoint kube-prometheus-stack ships, // no auth needed for in-cluster AlertManager. if ds.AlertManagerURL != "" { - out.AlertManagerConnection = httpHealth(ctx, s.HTTP, ds.AlertManagerURL+"/-/healthy") + out.AlertManagerConnection, out.AlertManagerConnectionError = httpHealth(ctx, s.HTTP, ds.AlertManagerURL+"/-/healthy") } // OpenCost: only probe if a URL is set. // where missing URL → opencost=False, no request. if ds.OpencostURL != "" { - out.OpencostConnection = httpHealth(ctx, s.HTTP, ds.OpencostURL+"/healthz") + out.OpencostConnection, out.OpencostConnectionError = httpHealth(ctx, s.HTTP, ds.OpencostURL+"/healthz") } // Trace status / provider / URL — verbatim port of the backend. @@ -479,19 +502,42 @@ func isClickHouseEnabled(ds Datasources) bool { return ds.ClickHouseStatus && ds.ClickHouseURL != "" } -// httpHealth returns true iff GET returns 2xx within 5s. -func httpHealth(ctx context.Context, c *http.Client, url string) bool { +// httpHealth probes GET with a 5s budget. It returns ok=true iff the +// response is 2xx; on failure it also returns a short one-line reason +// (transport error or "HTTP : ") that the UI surfaces +// next to the "Disconnected" status. reason is empty when ok is true. +func httpHealth(ctx context.Context, c *http.Client, url string) (ok bool, reason string) { cctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(cctx, http.MethodGet, url, nil) if err != nil { - return false + return false, err.Error() } resp, err := c.Do(req) if err != nil { - return false + return false, err.Error() } defer func() { _ = resp.Body.Close() }() - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1024)) - return resp.StatusCode >= 200 && resp.StatusCode < 300 + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + // Drain the body so the transport can reuse the keep-alive connection; + // this probe runs every telemetry cycle against every datasource. + _, _ = io.Copy(io.Discard, resp.Body) + return true, "" + } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return false, healthErr(resp.StatusCode, body) +} + +// healthErr formats a non-2xx probe response into a compact single-line reason +// suitable for the health UI: whitespace collapsed and truncated so a verbose +// JSON error body doesn't blow up the payload. +func healthErr(status int, body []byte) string { + msg := strings.Join(strings.Fields(string(body)), " ") + if len(msg) > 200 { + msg = msg[:200] + "…" + } + if msg == "" { + return fmt.Sprintf("HTTP %d", status) + } + return fmt.Sprintf("HTTP %d: %s", status, msg) } diff --git a/runner/pkg/telemetry/service_test.go b/runner/pkg/telemetry/service_test.go index 2ae4bab5..76360ca4 100644 --- a/runner/pkg/telemetry/service_test.go +++ b/runner/pkg/telemetry/service_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "log/slog" + "net" "net/http" "net/http/httptest" "strings" @@ -420,3 +421,69 @@ func TestActivityStats_TracesConnectionErrorAlwaysEmitted(t *testing.T) { t.Errorf("marshalled ActivityStats omits an empty tracesConnectionError, so the\ncollector's jsonb merge would keep a stale reason forever: %s", buf) } } + +// A healthy probe must not consume the response body as data — it drains it so +// the transport can reuse the keep-alive connection. httpHealth runs every +// telemetry cycle against every configured datasource, so a probe that stranded +// an undrained body would open a fresh TCP connection each time. +func TestHTTPHealth_HealthyProbeReusesConnection(t *testing.T) { + var conns int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Larger than the 512-byte snippet limit: an undrained body of this + // size is what breaks connection reuse. + _, _ = w.Write([]byte(strings.Repeat("x", 4096))) + })) + srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + atomic.AddInt32(&conns, 1) + } + } + defer srv.Close() + + c := srv.Client() + for i := 0; i < 3; i++ { + if ok, reason := httpHealth(context.Background(), c, srv.URL); !ok || reason != "" { + t.Fatalf("probe %d: ok=%v reason=%q; want true, empty", i, ok, reason) + } + } + if got := atomic.LoadInt32(&conns); got != 1 { + t.Errorf("opened %d connections across 3 probes; want 1 (body must be drained on success)", got) + } +} + +// A failing probe must explain itself in one compact line — this is the string +// the UI renders next to the integration's "Disconnected" pill. +func TestHTTPHealth_FailureReportsCompactReason(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + // Multi-line body: the reason must collapse to a single line. + _, _ = w.Write([]byte("{\n \"error\": \"Token is expired\"\n}")) + })) + defer srv.Close() + + ok, reason := httpHealth(context.Background(), srv.Client(), srv.URL) + if ok { + t.Fatal("ok = true; want false for a 401") + } + if !strings.Contains(reason, "HTTP 401") || !strings.Contains(reason, "Token is expired") { + t.Errorf("reason = %q; want it to carry the status and the backend's message", reason) + } + if strings.ContainsAny(reason, "\n\r") { + t.Errorf("reason = %q; want a single line", reason) + } +} + +// A verbose error body must not blow up the telemetry payload. +func TestHealthErr_TruncatesLongBodies(t *testing.T) { + got := healthErr(500, []byte(strings.Repeat("a", 1000))) + if len(got) > 240 { + t.Errorf("reason is %d chars; want it truncated", len(got)) + } + if !strings.HasPrefix(got, "HTTP 500: ") { + t.Errorf("reason = %q; want it to lead with the status", got) + } + // An empty body still names the status rather than going silent. + if got := healthErr(503, nil); got != "HTTP 503" { + t.Errorf("healthErr(503, nil) = %q; want %q", got, "HTTP 503") + } +}