implement 2-way merge logic for deploying kubernetes resources

docker/aslan.Dockerfile

@@ -10,6 +10,8 @@ COPY go.mod go.sum ./
 COPY cmd cmd
 COPY pkg pkg
 
+RUN apk add --no-cache git
+
 RUN go mod download
 
 RUN --mount=type=cache,id=gobuild,target=/gocache \

docker/executor.Dockerfile

@@ -10,6 +10,8 @@ COPY go.mod go.sum ./
 COPY cmd cmd
 COPY pkg pkg
 
+RUN apk add --no-cache git
+
 RUN go mod download
 
 RUN --mount=type=cache,id=gobuild,target=/gocache \

pkg/microservice/aslan/config/consts.go

@@ -760,8 +760,15 @@ const (
 type ValueMergeStrategy string
 
 const (
-	ValueMergeStrategyReuseValue = "reuse-values"
-	ValueMergeStrategyOverride   = "override"
+	ValueMergeStrategyReuseValue ValueMergeStrategy = "reuse-values"
+	ValueMergeStrategyOverride   ValueMergeStrategy= "override"
+)
+
+type YAMLMergeStrategy string
+
+const (
+	YAMLMergeStrategyMerge    YAMLMergeStrategy = "merge"
+	YAMLMergeStrategyOverride YAMLMergeStrategy = "override"
 )
 
 type SystemLanguage string

pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go

@@ -233,6 +233,8 @@ type JobTaskDeploySpec struct {
 	Timeout            int                             `bson:"timeout"                          json:"timeout"                             yaml:"timeout"`
 	ReplaceResources   []Resource                      `bson:"replace_resources"                json:"replace_resources"                   yaml:"replace_resources"`
 	RelatedPodLabels   []map[string]string             `bson:"-"                                json:"-"                                   yaml:"-"`
+	// overrideResource is used to do a full yaml override instead of a 2-way merge patching for all the resources
+	OverrideResource   bool                            `bson:"override_resource"                json:"override_resource"                   yaml:"override_resource"`
 	// for compatibility
 	ServiceModule string `bson:"service_module"                   json:"service_module"                      yaml:"-"`
 	Image         string `bson:"image"                            json:"image"                               yaml:"-"`
@@ -264,6 +266,7 @@ type JobTaskDeployRevertSpec struct {
 	OverrideKVs             string       `bson:"override_kvs"                     json:"override_kvs"                        yaml:"override_kvs"`
 	Revision                int64        `bson:"revision"                         json:"revision"                            yaml:"revision"`
 	RevisionCreateTime      int64        `bson:"revision_create_time"             json:"revision_create_time"                yaml:"revision_create_time"`
+	OverrideResource        bool         `bson:"override_resource"                json:"override_resource"                   yaml:"override_resource"` 
 }
 
 type DeployServiceModule struct {

pkg/microservice/aslan/core/common/repository/models/workflow_v4.go

@@ -481,6 +481,9 @@ type ZadigDeployJobSpec struct {
 	ValueMergeStrategy  config.ValueMergeStrategy `bson:"value_merge_strategy"             json:"value_merge_strategy"                yaml:"value_merge_strategy"`
 	MergeStrategySource config.ParamSourceType    `bson:"merge_strategy_source"            json:"merge_strategy_source"               yaml:"merge_strategy_source"`
 
+	// YAML deploy only field
+	YAMLMergeStrategy config.YAMLMergeStrategy `bson:"yaml_merge_strategy"            json:"yaml_merge_strategy"               yaml:"yaml_merge_strategy"`
+
 	// TODO: Deprecated in 2.3.0, this field is now used for saving the default service module info for deployment.
 	DefaultServices []*ServiceAndImage `bson:"service_and_images" yaml:"service_and_images" json:"service_and_images"`
 }
@@ -576,6 +579,7 @@ type DeployServiceVariableConfigList []*DeployServiceVariableConfig
 
 type DeployVariableInfo struct {
 	ValueMergeStrategy config.ValueMergeStrategy `bson:"value_merge_strategy,omitempty" json:"value_merge_strategy,omitempty" yaml:"value_merge_strategy,omitempty"`
+	YAMLMergeStrategy  config.YAMLMergeStrategy  `bson:"yaml_merge_strategy,omitempty" json:"yaml_merge_strategy,omitempty" yaml:"yaml_merge_strategy,omitempty"`
 
 	VariableKVs []*commontypes.RenderVariableKV `bson:"variable_kvs"                     yaml:"variable_kvs"              json:"variable_kvs"`
 	OverrideKVs string                          `bson:"override_kvs"                     yaml:"override_kvs"              json:"override_kvs"` // used for helm services, json-encoded string of kv value

pkg/microservice/aslan/core/common/service/kube/actions.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
-	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -31,6 +30,8 @@ import (
 	"k8s.io/apimachinery/pkg/selection"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/koderover/zadig/v2/pkg/tool/clientmanager"
+
 	"github.com/koderover/zadig/v2/pkg/microservice/aslan/config"
 	commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models"
 	"github.com/koderover/zadig/v2/pkg/setting"
@@ -42,7 +43,7 @@ import (
 
 var registrySecretSuffix = "-registry-secret"
 
-func CreateNamespace(namespace string, customLabels map[string]string, enableIstioInjection bool, kubeClient client.Client) error {
+func CreateNamespace(namespace, clusterID string, customLabels map[string]string, enableIstioInjection bool) error {
 	nsLabels := map[string]string{
 		setting.EnvCreatedBy: setting.EnvCreator,
 	}
@@ -54,59 +55,52 @@ func CreateNamespace(namespace string, customLabels map[string]string, enableIst
 		customLabels = map[string]string{}
 	}
 	mergedLabels := labels.Merge(customLabels, nsLabels)
-	createErr := updater.CreateNamespaceByName(namespace, mergedLabels, kubeClient)
+	createErr := updater.CreateNamespaceByNameV2(context.TODO(), clusterID, namespace, mergedLabels)
 	if createErr != nil && !apierrors.IsAlreadyExists(createErr) {
 		return createErr
 	}
 
-	var err error
-	nsObj := &corev1.Namespace{}
-	// It may fail to obtain the namespace immediately after it is created due to synchronization delay.
-	// Try twice.
-	for i := 0; i < 2; i++ {
-		err = kubeClient.Get(context.TODO(), client.ObjectKey{
-			Name: namespace,
-		}, nsObj)
-		if err == nil {
-			break
-		}
-
-		time.Sleep(time.Second)
-	}
-	if err != nil {
-		return err
-	}
 	if enableIstioInjection && createErr != nil && apierrors.IsAlreadyExists(createErr) {
-		nsObj.Labels[zadigtypes.IstioLabelKeyInjection] = zadigtypes.IstioLabelValueInjection
-		err = updater.UpdateNamespace(nsObj, kubeClient)
+		err := updater.UpdateNamespaceV2(context.TODO(), clusterID, namespace, func(ns *corev1.Namespace) error {
+			if ns.Labels == nil {
+				ns.Labels = make(map[string]string)
+			}
+			ns.Labels[zadigtypes.IstioLabelKeyInjection] = zadigtypes.IstioLabelValueInjection
+			return nil
+		})
 		if err != nil {
 			return fmt.Errorf("failed to add istio-injection label and update namespace %s: %s", namespace, err)
 		}
 	}
 
-	if nsObj.Status.Phase == corev1.NamespaceTerminating {
-		return fmt.Errorf("namespace `%s` is in terminating state, please wait for a whilie and try again", namespace)
+	if createErr != nil && apierrors.IsAlreadyExists(createErr) {
+		c, err := clientmanager.NewKubeClientManager().GetKubernetesClientSet(clusterID)
+		if err != nil {
+			return fmt.Errorf("failed to get kube client: %w", err)
+		}
+		nsObj, err := c.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to get namespace %s: %w", namespace, err)
+		}
+		if nsObj.Status.Phase == corev1.NamespaceTerminating {
+			return fmt.Errorf("namespace `%s` is in terminating state, please wait for a while and try again", namespace)
+		}
 	}
 
 	return nil
 }
 
-func EnsureNamespaceLabels(namespace string, customLabels map[string]string, kubeClient client.Client) error {
-	nsObj := &corev1.Namespace{}
-	err := kubeClient.Get(context.TODO(), client.ObjectKey{
-		Name: namespace,
-	}, nsObj)
-	if err != nil {
-		return err
-	}
-	if labels.SelectorFromValidatedSet(customLabels).Matches(labels.Set(nsObj.Labels)) {
+func EnsureNamespaceLabels(namespace, clusterID string, customLabels map[string]string) error {
+	return updater.UpdateNamespaceV2(context.TODO(), clusterID, namespace, func(ns *corev1.Namespace) error {
+		if labels.SelectorFromValidatedSet(customLabels).Matches(labels.Set(ns.Labels)) {
+			return nil
+		}
+		ns.Labels = labels.Merge(ns.Labels, customLabels)
 		return nil
-	}
-	nsObj.Labels = labels.Merge(nsObj.Labels, customLabels)
-	return updater.UpdateNamespace(nsObj, kubeClient)
+	})
 }
 
-func CreateOrUpdateRSASecret(publicKey, privateKey []byte, kubeClient client.Client) error {
+func CreateOrUpdateRSASecret(publicKey, privateKey []byte, clusterID string) error {
 	data := make(map[string][]byte)
 
 	data["publicKey"] = publicKey
@@ -120,14 +114,14 @@ func CreateOrUpdateRSASecret(publicKey, privateKey []byte, kubeClient client.Cli
 		Data: data,
 		Type: corev1.SecretTypeOpaque,
 	}
-	return updater.UpdateOrCreateSecret(secret, kubeClient)
+	return updater.CreateOrUpdateSecretV2(context.TODO(), clusterID, secret)
 }
 
-func CreateOrUpdateDefaultRegistrySecret(namespace string, reg *commonmodels.RegistryNamespace, kubeClient client.Client) error {
-	return CreateOrUpdateRegistrySecret(namespace, reg, true, kubeClient)
+func CreateOrUpdateDefaultRegistrySecret(namespace, clusterID string, reg *commonmodels.RegistryNamespace) error {
+	return CreateOrUpdateRegistrySecret(namespace, clusterID, reg, true)
 }
 
-func CreateOrUpdateRegistrySecret(namespace string, reg *commonmodels.RegistryNamespace, isDefault bool, kubeClient client.Client) error {
+func CreateOrUpdateRegistrySecret(namespace, clusterID string, reg *commonmodels.RegistryNamespace, isDefault bool) error {
 	var secretName string
 	var err error
 	if !isDefault {
@@ -158,7 +152,7 @@ func CreateOrUpdateRegistrySecret(namespace string, reg *commonmodels.RegistryNa
 		Data: data,
 		Type: corev1.SecretTypeDockercfg,
 	}
-	return updater.UpdateOrCreateSecret(secret, kubeClient)
+	return updater.CreateOrUpdateSecretV2(context.TODO(), clusterID, secret)
 }
 
 func GenRegistrySecretName(reg *commonmodels.RegistryNamespace) (string, error) {