Merge branch 'main' into kw-deprecating-dbt

Author: Startrekzky

Makefile

@@ -29,7 +29,7 @@ build-config-ui-image:
 	cd config-ui; docker build -t $(IMAGE_REPO)/devlake-config-ui:$(TAG) --file ./Dockerfile .
 
 build-grafana-image:
-	cd grafana; docker build -t $(IMAGE_REPO)/devlake-dashboard:$(TAG) --file ./backend/Dockerfile .
+	cd grafana; docker build -t $(IMAGE_REPO)/devlake-dashboard:$(TAG) --file ./Dockerfile .
 
 build-images: build-server-image build-config-ui-image build-grafana-image
 

backend/helpers/oidchelper/config.go

@@ -88,11 +88,11 @@ func (c *Config) ProviderNames() []string {
 }
 
 // LoadConfig reads auth env vars via Viper and validates required fields.
-// AUTH_ENABLED defaults to true unless it is explicitly set to false.
+// AUTH_ENABLED defaults to false unless it is explicitly set to true.
 func LoadConfig(basicRes context.BasicRes) (*Config, error) {
 	cfg := basicRes.GetConfigReader()
 
-	authEnabled := true
+	authEnabled := false
 	if cfg.IsSet("AUTH_ENABLED") {
 		authEnabled = cfg.GetBool("AUTH_ENABLED")
 	}

backend/helpers/oidchelper/config_test.go

@@ -105,15 +105,15 @@ func (b basicResStub) ReplaceLogger(log.Logger) corectx.BasicRes {
 }
 func (b basicResStub) GetDal() dal.Dal { return nil }
 
-func TestLoadConfigDefaultsAuthEnabled(t *testing.T) {
+func TestLoadConfigDefaultsAuthDisabled(t *testing.T) {
 	v := viper.New()
 
 	cfg, err := LoadConfig(basicResStub{cfg: v})
 	if err != nil {
 		t.Fatalf("LoadConfig returned error: %v", err)
 	}
-	if !cfg.AuthEnabled {
-		t.Fatal("AuthEnabled should default to true when AUTH_ENABLED is unset")
+	if cfg.AuthEnabled {
+		t.Fatal("AuthEnabled should default to false when AUTH_ENABLED is unset")
 	}
 	if cfg.OIDCEnabled {
 		t.Fatal("OIDCEnabled should default to false")
@@ -125,6 +125,7 @@ func TestLoadConfigDefaultsAuthEnabled(t *testing.T) {
 
 func TestLoadConfigRequiresSessionSecretForOIDC(t *testing.T) {
 	v := viper.New()
+	v.Set("AUTH_ENABLED", true)
 	v.Set("OIDC_ENABLED", true)
 
 	if _, err := LoadConfig(basicResStub{cfg: v}); err == nil {

backend/plugins/azuredevops_go/tasks/shared.go

@@ -153,8 +153,11 @@ func change203To401(res *http.Response) errors.Error {
 // that failed due to a YAML syntax error never produce a usable timeline), instead
 // of aborting the entire subtask.
 func ignoreInvalidTimelineResponse(res *http.Response) errors.Error {
-	// Keep existing behaviour: treat 404 as a graceful skip (build was deleted).
-	if res.StatusCode == http.StatusNotFound {
+	// Treat 404 (build deleted) and 204 (build has no timeline data) as
+	// graceful skips so the subtask continues instead of failing.
+	// The 204 guard must come before the body is read because a 204 response
+	// has an empty body by definition and would cause the JSON parser to fail.
+	if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusNoContent {
 		return api.ErrIgnoreAndContinue
 	}
 

backend/plugins/azuredevops_go/tasks/shared_test.go

@@ -41,6 +41,12 @@ func TestIgnoreInvalidTimelineResponse_404(t *testing.T) {
 	assert.Equal(t, api.ErrIgnoreAndContinue, err, "404 should return ErrIgnoreAndContinue")
 }
 
+func TestIgnoreInvalidTimelineResponse_204(t *testing.T) {
+	res := makeResponse(http.StatusNoContent, "")
+	err := ignoreInvalidTimelineResponse(res)
+	assert.Equal(t, api.ErrIgnoreAndContinue, err, "204 No Content should return ErrIgnoreAndContinue")
+}
+
 func TestIgnoreInvalidTimelineResponse_EmptyBody(t *testing.T) {
 	res := makeResponse(http.StatusOK, "")
 	err := ignoreInvalidTimelineResponse(res)

backend/plugins/circleci/tasks/job_collector.go

@@ -73,8 +73,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 					dal.Where("connection_id = ? and project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug),
 				}
 
-				if isIncremental {
-					clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+				// Incremental: workflows newer than last successful collectJobs.
+				// Full sync: workflows within SyncPolicy.TimeAfter window.
+				if createdAfter != nil {
+					clauses = append(clauses, dal.Where("created_date >= ?", createdAfter))
 				}
 
 				db := taskCtx.GetDal()
@@ -88,7 +90,7 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 				UrlTemplate:    "/v2/workflow/{{ .Input.Id }}/job",
 				Query:          BuildQueryParamsWithPageToken,
 				ResponseParser: ParseCircleciPageTokenResp,
-				AfterResponse:  ignoreDeletedBuilds, // Ignore the 404 response if a workflow has been deleted
+				AfterResponse:  ignoreDeletedOrBrokenBuilds,
 			},
 			GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
 				var job struct { // Individual job response lacks created_at field, so have to use started_at
@@ -105,7 +107,7 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 				UrlTemplate:    "/v2/workflow/{{ .Input.Id }}/job", // The individual job endpoint has different fields so need to recollect all jobs for a workflow
 				Query:          BuildQueryParamsWithPageToken,
 				ResponseParser: ParseCircleciPageTokenResp,
-				AfterResponse:  ignoreDeletedBuilds,
+				AfterResponse:  ignoreDeletedOrBrokenBuilds,
 			},
 			BuildInputIterator: func() (api.Iterator, errors.Error) {
 				db := taskCtx.GetDal()

backend/plugins/circleci/tasks/shared.go

@@ -123,10 +123,15 @@ func ParseCircleciPageTokenResp(res *http.Response) ([]json.RawMessage, errors.E
 	return data.Items, err
 }
 
-func ignoreDeletedBuilds(res *http.Response) errors.Error {
-	// CircleCI API will return a 404 response for a workflow/job that has been deleted
-	// due to their data retention policy. We should ignore these errors.
-	if res.StatusCode == http.StatusNotFound {
+// ignoreDeletedOrBrokenBuilds skips per-item API failures that should not
+// abort an entire collector subtask. 404 means the resource was deleted
+// (retention); 500 means the CircleCI Server record is corrupt/stuck
+// (e.g. pipeline exists but its /workflow endpoint errors).
+func ignoreDeletedOrBrokenBuilds(res *http.Response) errors.Error {
+	switch res.StatusCode {
+	case http.StatusNotFound:
+		return api.ErrIgnoreAndContinue
+	case http.StatusInternalServerError:
 		return api.ErrIgnoreAndContinue
 	}
 	return nil

backend/plugins/circleci/tasks/shared_test.go

@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func makeResponse(statusCode int) *http.Response {
+	return &http.Response{
+		StatusCode: statusCode,
+		Body:       io.NopCloser(bytes.NewBufferString("")),
+		Request:    &http.Request{},
+	}
+}
+
+func TestIgnoreDeletedOrBrokenBuilds(t *testing.T) {
+	tests := []struct {
+		name       string
+		statusCode int
+		want       error
+	}{
+		{"404 returns ErrIgnoreAndContinue", http.StatusNotFound, api.ErrIgnoreAndContinue},
+		{"500 returns ErrIgnoreAndContinue", http.StatusInternalServerError, api.ErrIgnoreAndContinue},
+		{"200 returns nil", http.StatusOK, nil},
+		{"403 returns nil", http.StatusForbidden, nil},
+		{"502 returns nil", http.StatusBadGateway, nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			res := makeResponse(tt.statusCode)
+			got := ignoreDeletedOrBrokenBuilds(res)
+			if tt.want == nil {
+				assert.Nil(t, got)
+			} else {
+				assert.Equal(t, tt.want, got)
+			}
+		})
+	}
+}
+

backend/plugins/circleci/tasks/workflow_collector.go

@@ -60,8 +60,10 @@ func CollectWorkflows(taskCtx plugin.SubTaskContext) errors.Error {
 					dal.Where("connection_id = ? AND project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug),
 				}
 
-				if isIncremental {
-					clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+				// Incremental: pipelines newer than last successful collectWorkflows.
+				// Full sync: pipelines within SyncPolicy.TimeAfter window.
+				if createdAfter != nil {
+					clauses = append(clauses, dal.Where("created_date >= ?", createdAfter))
 				}
 
 				db := taskCtx.GetDal()
@@ -75,7 +77,7 @@ func CollectWorkflows(taskCtx plugin.SubTaskContext) errors.Error {
 				UrlTemplate:    "/v2/pipeline/{{ .Input.Id }}/workflow",
 				Query:          BuildQueryParamsWithPageToken,
 				ResponseParser: ParseCircleciPageTokenResp,
-				AfterResponse:  ignoreDeletedBuilds, // Ignore the 404 response if a workflow has been deleted
+				AfterResponse:  ignoreDeletedOrBrokenBuilds,
 			},
 			GetCreated: extractCreatedAt,
 		},
@@ -88,7 +90,7 @@ func CollectWorkflows(taskCtx plugin.SubTaskContext) errors.Error {
 					err := api.UnmarshalResponse(res, &data)
 					return []json.RawMessage{data}, err
 				},
-				AfterResponse: ignoreDeletedBuilds,
+				AfterResponse: ignoreDeletedOrBrokenBuilds,
 			},
 			BuildInputIterator: func() (api.Iterator, errors.Error) {
 				clauses := []dal.Clause{

backend/plugins/claude_code/models/connection.go

@@ -62,6 +62,14 @@ func (conn *ClaudeCodeConn) HasUsableCustomHeaders() bool {
 	return false
 }
 
+// IsConsoleApiKey returns true when the token is a Claude Console Admin API key.
+// Console keys (sk-ant-admin01-...) have full Admin API access and use
+// /v1/organizations/usage_report/claude_code instead of the Enterprise
+// /v1/organizations/analytics/* endpoints.
+func (conn *ClaudeCodeConn) IsConsoleApiKey() bool {
+	return strings.HasPrefix(strings.TrimSpace(conn.Token), "sk-ant-admin01-")
+}
+
 func (conn *ClaudeCodeConn) HasIncompleteCustomHeaders() bool {
 	if conn == nil {
 		return false