Skip to content

Commit 0e87fa9

Browse files
creydrcreydr
authored
Persist status updates intermediately (#41)
1 parent 1188a7b commit 0e87fa9

4 files changed

Lines changed: 300 additions & 12 deletions

File tree

‎api/v1alpha1/function_lifecycle.go‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,25 @@ func (f *Function) CalculateReadyCondition() {
5252
reason := ""
5353
message := ""
5454
for _, condition := range f.Status.Conditions {
55-
if condition.Type != TypeReady && condition.Status == metav1.ConditionFalse {
56-
allReady = false
57-
reason = condition.Reason
58-
message = condition.Message
59-
continue
55+
if condition.Type != TypeReady {
56+
if condition.Status == metav1.ConditionFalse {
57+
allReady = false
58+
reason = condition.Reason
59+
message = condition.Message
60+
continue
61+
} else if condition.Status == metav1.ConditionUnknown {
62+
allReady = false
63+
64+
// override reason & message only if not set already
65+
// (e.g. if set by a ConditionFalse as this takes preference)
66+
if reason == "" {
67+
reason = condition.Reason
68+
}
69+
if message == "" {
70+
message = condition.Message
71+
}
72+
continue
73+
}
6074
}
6175
}
6276

‎api/v1alpha1/function_lifecycle_test.go‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
package v1alpha1
2+
3+
import (
4+
"testing"
5+
6+
"k8s.io/apimachinery/pkg/api/meta"
7+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
8+
)
9+
10+
func TestCalculateReadyCondition(t *testing.T) {
11+
tests := []struct {
12+
name string
13+
conditions []metav1.Condition
14+
expectedStatus metav1.ConditionStatus
15+
expectedReason string
16+
expectedMessage string
17+
}{
18+
{
19+
name: "all conditions true",
20+
conditions: []metav1.Condition{
21+
{Type: TypeSourceReady, Status: metav1.ConditionTrue, Reason: "CloneSucceeded"},
22+
{Type: TypeDeployed, Status: metav1.ConditionTrue, Reason: "DeploySucceeded"},
23+
{Type: TypeMiddlewareUpToDate, Status: metav1.ConditionTrue, Reason: "UpToDate"},
24+
},
25+
expectedStatus: metav1.ConditionTrue,
26+
expectedReason: "ReconcileSucceeded",
27+
},
28+
{
29+
name: "one condition false",
30+
conditions: []metav1.Condition{
31+
{Type: TypeSourceReady, Status: metav1.ConditionTrue, Reason: "CloneSucceeded"},
32+
{Type: TypeDeployed, Status: metav1.ConditionFalse, Reason: "DeployFailed", Message: "deployment failed"},
33+
{Type: TypeMiddlewareUpToDate, Status: metav1.ConditionTrue, Reason: "UpToDate"},
34+
},
35+
expectedStatus: metav1.ConditionFalse,
36+
expectedReason: "DeployFailed",
37+
expectedMessage: "deployment failed",
38+
},
39+
{
40+
name: "one condition unknown",
41+
conditions: []metav1.Condition{
42+
{Type: TypeSourceReady, Status: metav1.ConditionTrue, Reason: "CloneSucceeded"},
43+
{Type: TypeDeployed, Status: metav1.ConditionUnknown, Reason: "NotChecked", Message: "deployment not checked yet"},
44+
{Type: TypeMiddlewareUpToDate, Status: metav1.ConditionTrue, Reason: "UpToDate"},
45+
},
46+
expectedStatus: metav1.ConditionFalse,
47+
expectedReason: "NotChecked",
48+
expectedMessage: "deployment not checked yet",
49+
},
50+
{
51+
name: "multiple conditions unknown",
52+
conditions: []metav1.Condition{
53+
{Type: TypeSourceReady, Status: metav1.ConditionUnknown, Reason: "NotCloned", Message: "source not cloned yet"},
54+
{Type: TypeDeployed, Status: metav1.ConditionUnknown, Reason: "NotDeployed", Message: "not deployed yet"},
55+
{Type: TypeMiddlewareUpToDate, Status: metav1.ConditionTrue, Reason: "UpToDate"},
56+
},
57+
expectedStatus: metav1.ConditionFalse,
58+
expectedReason: "NotCloned",
59+
expectedMessage: "source not cloned yet",
60+
},
61+
{
62+
name: "false takes precedence over unknown",
63+
conditions: []metav1.Condition{
64+
{Type: TypeSourceReady, Status: metav1.ConditionUnknown, Reason: "NotCloned", Message: "source not cloned yet"},
65+
{Type: TypeDeployed, Status: metav1.ConditionFalse, Reason: "DeployFailed", Message: "deployment failed"},
66+
{Type: TypeMiddlewareUpToDate, Status: metav1.ConditionTrue, Reason: "UpToDate"},
67+
},
68+
expectedStatus: metav1.ConditionFalse,
69+
expectedReason: "DeployFailed",
70+
expectedMessage: "deployment failed",
71+
},
72+
{
73+
name: "all conditions unknown",
74+
conditions: []metav1.Condition{
75+
{Type: TypeSourceReady, Status: metav1.ConditionUnknown, Reason: "unknown", Message: ""},
76+
{Type: TypeDeployed, Status: metav1.ConditionUnknown, Reason: "unknown", Message: ""},
77+
{Type: TypeMiddlewareUpToDate, Status: metav1.ConditionUnknown, Reason: "unknown", Message: ""},
78+
},
79+
expectedStatus: metav1.ConditionFalse,
80+
expectedReason: "unknown",
81+
},
82+
}
83+
84+
for _, tt := range tests {
85+
t.Run(tt.name, func(t *testing.T) {
86+
f := &Function{
87+
ObjectMeta: metav1.ObjectMeta{
88+
Generation: 1,
89+
},
90+
}
91+
f.Status.Conditions = tt.conditions
92+
93+
f.CalculateReadyCondition()
94+
95+
readyCondition := meta.FindStatusCondition(f.Status.Conditions, TypeReady)
96+
if readyCondition == nil {
97+
t.Fatal("Ready condition not found")
98+
}
99+
100+
if readyCondition.Status != tt.expectedStatus {
101+
t.Errorf("expected status %v, got %v", tt.expectedStatus, readyCondition.Status)
102+
}
103+
104+
if readyCondition.Reason != tt.expectedReason {
105+
t.Errorf("expected reason %q, got %q", tt.expectedReason, readyCondition.Reason)
106+
}
107+
108+
if tt.expectedMessage != "" && readyCondition.Message != tt.expectedMessage {
109+
t.Errorf("expected message %q, got %q", tt.expectedMessage, readyCondition.Message)
110+
}
111+
})
112+
}
113+
}
114+
115+
func TestInitializeConditions(t *testing.T) {
116+
f := &Function{
117+
ObjectMeta: metav1.ObjectMeta{
118+
Generation: 1,
119+
},
120+
}
121+
122+
// Set some existing conditions
123+
f.Status.Conditions = []metav1.Condition{
124+
{Type: TypeSourceReady, Status: metav1.ConditionTrue, Reason: "CloneSucceeded"},
125+
{Type: TypeReady, Status: metav1.ConditionTrue, Reason: "ReconcileSucceeded"},
126+
}
127+
128+
f.InitializeConditions()
129+
130+
// Verify all conditions are reset to Unknown
131+
for _, condType := range FunctionsConditions {
132+
cond := meta.FindStatusCondition(f.Status.Conditions, condType)
133+
if cond == nil {
134+
t.Errorf("condition %s not found", condType)
135+
continue
136+
}
137+
if cond.Status != metav1.ConditionUnknown {
138+
t.Errorf("condition %s: expected status Unknown, got %v", condType, cond.Status)
139+
}
140+
if cond.Reason != "unknown" {
141+
t.Errorf("condition %s: expected reason 'unknown', got %q", condType, cond.Reason)
142+
}
143+
if cond.ObservedGeneration != f.Generation {
144+
t.Errorf("condition %s: expected generation %d, got %d", condType, f.Generation, cond.ObservedGeneration)
145+
}
146+
}
147+
148+
// Verify Ready condition is also set to Unknown
149+
readyCond := meta.FindStatusCondition(f.Status.Conditions, TypeReady)
150+
if readyCond == nil {
151+
t.Fatal("Ready condition not found")
152+
}
153+
if readyCond.Status != metav1.ConditionUnknown {
154+
t.Errorf("Ready condition: expected status Unknown, got %v", readyCond.Status)
155+
}
156+
}

‎internal/controller/function_controller.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,15 +91,17 @@ func (r *FunctionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
9191
}
9292

9393
function := original.DeepCopy()
94+
95+
// Create tracker and add to context
96+
statusTracker := NewStatusTracker(r.Client, function)
97+
ctx = WithStatusTracker(ctx, statusTracker)
98+
9499
reconcileErr := r.reconcile(ctx, function)
95-
function.CalculateReadyCondition()
96100

97-
// update status if required
98-
if !equality.Semantic.DeepEqual(original.Status, function.Status) {
99-
if err := r.Status().Update(ctx, function); err != nil {
100-
logger.Error(err, "Unable to update Function status")
101-
return ctrl.Result{}, err
102-
}
101+
// Final flush at the end (handles ready condition calculation)
102+
if err := statusTracker.Flush(ctx, function); err != nil {
103+
logger.Error(err, "Unable to update Function status")
104+
return ctrl.Result{}, err
103105
}
104106

105107
if reconcileErr != nil {
@@ -122,12 +124,18 @@ func (r *FunctionReconciler) reconcile(ctx context.Context, function *v1alpha1.F
122124
defer repo.Cleanup()
123125

124126
r.updateFunctionStatusGit(function, repo)
127+
if err := FlushStatus(ctx, function); err != nil {
128+
return fmt.Errorf("failed to update status: %w", err)
129+
}
125130

126131
if err := r.ensureDeployment(ctx, function, repo, metadata); err != nil {
127132
return fmt.Errorf("deploying function failed: %w", err)
128133
}
129134

130135
r.updateFunctionStatus(function, metadata)
136+
if err := FlushStatus(ctx, function); err != nil {
137+
return fmt.Errorf("failed to update status: %w", err)
138+
}
131139

132140
return nil
133141
}
@@ -199,6 +207,12 @@ func (r *FunctionReconciler) handleMiddlewareUpdate(ctx context.Context, functio
199207
if !isOnLatestMiddleware {
200208
logger.Info("Function is not on latest middleware. Will redeploy")
201209
function.MarkMiddlewareNotUpToDate("MiddlewareOutdated", "Middleware is outdated, redeploying")
210+
211+
// Checkpoint 2: Flush status before long deploy operation
212+
if err := FlushStatus(ctx, function); err != nil {
213+
logger.Error(err, "Failed to update status before redeployment")
214+
}
215+
202216
if err := r.deploy(ctx, function, repo); err != nil {
203217
function.MarkDeployNotReady("DeployFailed", "Redeployment failed: %s", err.Error())
204218
return fmt.Errorf("failed to redeploy function: %w", err)

‎internal/controller/status_tracker.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
Copyright 2025.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
"github.com/functions-dev/func-operator/api/v1alpha1"
24+
"k8s.io/apimachinery/pkg/api/equality"
25+
"k8s.io/apimachinery/pkg/types"
26+
"k8s.io/client-go/util/retry"
27+
"sigs.k8s.io/controller-runtime/pkg/client"
28+
)
29+
30+
// StatusTracker manages incremental status updates during reconciliation
31+
type StatusTracker struct {
32+
k8sClient client.Client
33+
original *v1alpha1.Function
34+
}
35+
36+
// NewStatusTracker creates a new status tracker with a snapshot of the current function state
37+
func NewStatusTracker(k8sClient client.Client, function *v1alpha1.Function) *StatusTracker {
38+
return &StatusTracker{
39+
k8sClient: k8sClient,
40+
original: function.DeepCopy(),
41+
}
42+
}
43+
44+
// Flush updates the function status if it has changed since the last flush
45+
func (t *StatusTracker) Flush(ctx context.Context, current *v1alpha1.Function) error {
46+
// Always calculate ready condition before comparing
47+
current.CalculateReadyCondition()
48+
49+
// Compare and update if changed
50+
if !equality.Semantic.DeepEqual(t.original.Status, current.Status) {
51+
// Retry on conflict with exponential backoff
52+
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
53+
// Get the latest version to ensure we have the most recent resourceVersion
54+
latest := &v1alpha1.Function{}
55+
if err := t.k8sClient.Get(ctx, types.NamespacedName{
56+
Name: current.Name,
57+
Namespace: current.Namespace,
58+
}, latest); err != nil {
59+
return err
60+
}
61+
62+
// Apply our status changes to the latest version
63+
latest.Status = current.Status
64+
65+
// Attempt the update
66+
return t.k8sClient.Status().Update(ctx, latest)
67+
})
68+
69+
if err != nil {
70+
return fmt.Errorf("failed to update status: %w", err)
71+
}
72+
73+
// Update our snapshot to the new state
74+
t.original = current.DeepCopy()
75+
}
76+
77+
return nil
78+
}
79+
80+
// statusTrackerKey is the context key for the status tracker
81+
type statusTrackerKey struct{}
82+
83+
// WithStatusTracker adds a status tracker to the context
84+
func WithStatusTracker(ctx context.Context, tracker *StatusTracker) context.Context {
85+
return context.WithValue(ctx, statusTrackerKey{}, tracker)
86+
}
87+
88+
// GetStatusTracker retrieves the tracker from context
89+
func GetStatusTracker(ctx context.Context) *StatusTracker {
90+
tracker, ok := ctx.Value(statusTrackerKey{}).(*StatusTracker)
91+
if !ok {
92+
return nil
93+
}
94+
return tracker
95+
}
96+
97+
// FlushStatus is a convenience helper that gets tracker from context and flushes
98+
func FlushStatus(ctx context.Context, function *v1alpha1.Function) error {
99+
tracker := GetStatusTracker(ctx)
100+
if tracker == nil {
101+
return nil // gracefully handle missing tracker
102+
}
103+
return tracker.Flush(ctx, function)
104+
}

0 commit comments

Comments
 (0)