l4: fix h2, master node cidr, add detailed termination log (beclab#2799)

* fix: h2, master node cidr, add detailed termination log

* update l4 image tag to v0.3.15

Author: hysyeah

cli/pkg/upgrade/base.go

@@ -142,7 +142,7 @@ func (u upgraderBase) UpgradeSystemComponents() []task.Interface {
 		},
 		&task.LocalTask{
 			Name:   "UpgradeL4BFLProxy",
-			Action: &upgradeL4BFLProxy{Tag: "v0.3.14"},
+			Action: &upgradeL4BFLProxy{Tag: "v0.3.15"},
 			Retry:  6,
 			Delay:  15 * time.Second,
 		},

framework/bfl/.olares/config/launcher/templates/bfl_deploy.yaml

@@ -304,7 +304,7 @@ spec:
         - name: BACKUP_SERVER
           value: backup-server.os-framework:8082
         - name: L4_PROXY_IMAGE_VERSION
-          value: v0.3.14
+          value: v0.3.15
         - name: L4_PROXY_SERVICE_ACCOUNT
           value: os-network-internal
         - name: L4_PROXY_NAMESPACE

framework/l4-bfl-proxy/.olares/Olares.yaml

@@ -3,5 +3,5 @@ target: prebuilt
 output:
   containers:
     - 
-      name: beclab/l4-bfl-proxy:v0.3.14
+      name: beclab/l4-bfl-proxy:v0.3.15
 # must have blank new line            

framework/l4-bfl-proxy/internal/message/types.go

@@ -28,6 +28,7 @@ type UserInfo struct {
 	SSL               *SSLConfig
 	Apps              []*AppInfo
 	FileserverNodes   []*FileserverNodeInfo
+	MasterNodeCIDR    string
 }
 
 type AppInfo struct {

framework/l4-bfl-proxy/internal/provider/provider.go

@@ -3,6 +3,7 @@ package provider
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sort"
 	"strconv"
@@ -495,6 +496,10 @@ func (p *Provider) listUsers(ctx context.Context, rawAppsMap map[string][]*appv1
 		if err != nil {
 			return nil, err
 		}
+		masterNodeCIDR, err := p.getMasterNodeCIDR(ctx)
+		if err != nil {
+			return nil, err
+		}
 
 		info := &message.UserInfo{
 			Name:              user.Name,
@@ -514,6 +519,7 @@ func (p *Provider) listUsers(ctx context.Context, rawAppsMap map[string][]*appv1
 			SSL:               sslConfig,
 			CustomDomainCerts: allCerts[user.Name],
 			FileserverNodes:   fileserverNodes,
+			MasterNodeCIDR:    masterNodeCIDR,
 		}
 		result = append(result, info)
 	}
@@ -674,6 +680,28 @@ func (p *Provider) getUsers(ctx context.Context) ([]iamv1alpha2.User, error) {
 	return users, nil
 }
 
+func (p *Provider) getMasterNodeCIDR(ctx context.Context) (string, error) {
+	var nodeList corev1.NodeList
+	if err := p.cache.List(ctx, &nodeList, client.HasLabels{"node-role.kubernetes.io/control-plane"}); err != nil {
+		klog.Errorf("provider: list node failed: %v", err)
+		return "", err
+	}
+	if len(nodeList.Items) == 0 {
+		return "", errors.New("no master node found")
+	}
+	node := nodeList.Items[0]
+	if len(node.Annotations) == 0 {
+		klog.Warningf("provider: node %s with empty annotations", node.Name)
+		return "", nil
+	}
+	cidr, ok := node.Annotations["projectcalico.org/IPv4Address"]
+	if !ok {
+		klog.Warningf("provider: node %s has no projectcalico.org/IPv4Address annotation", node.Name)
+		return "", nil
+	}
+	return cidr, nil
+}
+
 func getAnnotation(user *iamv1alpha2.User, key string) string {
 	if v, ok := user.Annotations[key]; ok && v != "" {
 		return v

framework/l4-bfl-proxy/internal/translator/translator.go

@@ -298,7 +298,10 @@ func (t *Translator) applyDenyAllRestrictions(user *message.UserInfo, vhosts []*
 	if user.LocalDomainIP != "" {
 		restrictCIDRs = append(restrictCIDRs, user.LocalDomainIP+"/32")
 	}
-	restrictCIDRs = append(restrictCIDRs, user.AllowCIDRs...)
+	if user.MasterNodeCIDR != "" {
+		restrictCIDRs = append(restrictCIDRs, user.MasterNodeCIDR)
+	}
+	//restrictCIDRs = append(restrictCIDRs, user.AllowCIDRs...)
 
 	for _, vh := range vhosts {
 		isAllowed := false

framework/l4-bfl-proxy/internal/xds/translator/translator.go

@@ -526,7 +526,7 @@ func buildMultiUserHTTPSListener(port uint32, proxyProtocol bool, httpListeners
 						Name:      httpIR.TLSCert.Name,
 						SdsConfig: adsSource,
 					}},
-					AlpnProtocols: []string{"http/1.1"},
+					AlpnProtocols: []string{"h2", "http/1.1"},
 				},
 			}
 			transportSocket = &corev3.TransportSocket{
@@ -1340,7 +1340,7 @@ func buildHTTPAccessLog() *accesslogv3.AccessLog {
 				Format: &corev3.SubstitutionFormatString_TextFormatSource{
 					TextFormatSource: &corev3.DataSource{
 						Specifier: &corev3.DataSource_InlineString{
-							InlineString: "[%START_TIME%] %DOWNSTREAM_REMOTE_ADDRESS% -> %UPSTREAM_HOST% %REQ(:AUTHORITY)% %REQ(:PATH)% %RESPONSE_CODE% duration=%DURATION%ms rx=%BYTES_RECEIVED% tx=%BYTES_SENT% flags=%RESPONSE_FLAGS% route=%ROUTE_NAME% cluster=%UPSTREAM_CLUSTER% details=%RESPONSE_CODE_DETAILS% ufail=%UPSTREAM_TRANSPORT_FAILURE_REASON%\n",
+							InlineString: "[%START_TIME%] %DOWNSTREAM_REMOTE_ADDRESS% -> %UPSTREAM_HOST% %REQ(:AUTHORITY)% %REQ(:PATH)% %RESPONSE_CODE% duration=%DURATION%ms rx=%BYTES_RECEIVED% tx=%BYTES_SENT% flags=%RESPONSE_FLAGS% route=%ROUTE_NAME% cluster=%UPSTREAM_CLUSTER% details=%RESPONSE_CODE_DETAILS% ufail=%UPSTREAM_TRANSPORT_FAILURE_REASON% upstream_connection_id=%UPSTREAM_CONNECTION_ID% connection_termination_details=%CONNECTION_TERMINATION_DETAILS%\n",
 						},
 					},
 				},