From a7cc65b4733802187e72182e7dc309a742993a71 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Thu, 25 Oct 2018 22:26:49 +0000 Subject: [PATCH 01/15] batch job as service provider --- common/types.go | 9 +- provider/aws/roleset.go | 6 + templates/assets/cloudformation/env-batch.yml | 0 .../assets/cloudformation/service-batch.yml | 60 +++++++ .../assets/cloudformation/service-iam.yml | 53 +++++++ workflows/environment_common.go | 6 + workflows/service_common.go | 13 ++ workflows/service_deploy.go | 146 +++++++++++++++--- 8 files changed, 268 insertions(+), 25 deletions(-) create mode 100644 templates/assets/cloudformation/env-batch.yml create mode 100644 templates/assets/cloudformation/service-batch.yml diff --git a/common/types.go b/common/types.go index 53630b98..8fb68b8b 100644 --- a/common/types.go +++ b/common/types.go @@ -157,6 +157,7 @@ type Service struct { HostPatterns []string `yaml:"hostPatterns,omitempty"` Priority int `yaml:"priority,omitempty" validate:"max=50000"` Pipeline Pipeline `yaml:"pipeline,omitempty"` + ProviderOverride string `yaml:"provider,omitempty"` Database Database `yaml:"database,omitempty"` Schedule []Schedule `yaml:"schedules,omitempty"` TargetCPUUtilization int `yaml:"targetCPUUtilization,omitempty" validate:"max=100"` @@ -168,6 +169,7 @@ type Service struct { EcsService string `yaml:"ecsService,omitempty" validate:"validateRoleARN"` EcsTask string `yaml:"ecsTask,omitempty" validate:"validateRoleARN"` ApplicationAutoScaling string `yaml:"applicationAutoScaling,omitempty" validate:"validateRoleARN"` + BatchJobRole string `yaml:"batchJobRole,omitempty" validate:"validateRoleARN"` } `yaml:"roles,omitempty"` } @@ -374,6 +376,7 @@ const ( TemplateEnvECS = "cloudformation/env-ecs.yml" TemplateEnvEKS = "cloudformation/env-eks.yml" TemplateEnvEKSBootstrap = "cloudformation/env-eks-bootstrap.yml" + TemplateEnvBatch = "cloudformation/env-batch.yml" TemplateEnvIAM = "cloudformation/env-iam.yml" TemplatePipelineIAM = "cloudformation/pipeline-iam.yml" TemplatePipeline = "cloudformation/pipeline.yml" @@ -381,6 +384,7 @@ const ( TemplateSchedule = "cloudformation/schedule.yml" TemplateServiceEC2 = "cloudformation/service-ec2.yml" TemplateServiceECS = "cloudformation/service-ecs.yml" + TemplateServiceBatch = "cloudformation/service-batch.yml" TemplateServiceIAM = "cloudformation/service-iam.yml" TemplateVPCTarget = "cloudformation/vpc-target.yml" TemplateVPC = "cloudformation/vpc.yml" @@ -396,8 +400,8 @@ type DeploymentStrategy string // List of supported deployment strategies const ( BlueGreenDeploymentStrategy DeploymentStrategy = "blue_green" - RollingDeploymentStrategy DeploymentStrategy = "rolling" - ReplaceDeploymentStrategy DeploymentStrategy = "replace" + RollingDeploymentStrategy = "rolling" + ReplaceDeploymentStrategy = "replace" ) // EnvProvider describes supported environment strategies @@ -410,6 +414,7 @@ const ( EnvProviderEc2 = "ec2" EnvProviderEks = "eks" EnvProviderEksFargate = "eks-fargate" + EnvProviderBatch = "batch" ) // InstanceTenancy describes supported tenancy options for EC2 diff --git a/provider/aws/roleset.go b/provider/aws/roleset.go index af56ab7c..0713086e 100644 --- a/provider/aws/roleset.go +++ b/provider/aws/roleset.go @@ -68,6 +68,7 @@ func (rolesetMgr *iamRolesetManager) GetServiceRoleset(environmentName string, s overrideRole(roleset, "EcsServiceRoleArn", rolesetMgr.context.Config.Service.Roles.EcsService) overrideRole(roleset, "EcsTaskRoleArn", rolesetMgr.context.Config.Service.Roles.EcsTask) overrideRole(roleset, "ApplicationAutoScalingRoleArn", rolesetMgr.context.Config.Service.Roles.ApplicationAutoScaling) + overrideRole(roleset, "BatchJobRoleArn", rolesetMgr.context.Config.Service.Roles.BatchJobRole) return roleset, nil } @@ -190,6 +191,11 @@ func (rolesetMgr *iamRolesetManager) UpsertServiceRoleset(environmentName string break } } + // allow to override provider (batch job definition can be deployed without environment) + if rolesetMgr.context.Config.Service.ProviderOverride != "" { + // todo: validate values + envProvider = rolesetMgr.context.Config.Service.ProviderOverride + } if envProvider == "" { log.Debugf("unable to find environment named '%s' in configuration...checking for existing stack", environmentName) envStackName := common.CreateStackName(rolesetMgr.context.Config.Namespace, common.StackTypeEnv, environmentName) diff --git a/templates/assets/cloudformation/env-batch.yml b/templates/assets/cloudformation/env-batch.yml new file mode 100644 index 00000000..e69de29b diff --git a/templates/assets/cloudformation/service-batch.yml b/templates/assets/cloudformation/service-batch.yml new file mode 100644 index 00000000..e53d0af1 --- /dev/null +++ b/templates/assets/cloudformation/service-batch.yml @@ -0,0 +1,60 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 +Description: MU AWS Batch job definition +Parameters: + Namespace: + Type: String + Description: Namespace for stack prefixes + EnvironmentName: + Type: String + Description: Name of environment used for resource namespace + ServiceName: + Type: String + Description: Name of service used for resource namespace + ServiceVCpu: + Type: String + Description: VCPUs to reserve for container + Default: '2' + ServiceMemory: + Type: String + Description: Memory to allocate to container (in MiB) + Default: '300' + ImageUrl: + Type: String + Description: Docker Image URL (or name, like 'amazonlinux') + BatchJobRoleArn: + Type: String + Description: IAM Role assumed by AWS Batch job (ecs task) + RetryAttemps: + Type: String + Description: Number of retry attempts + Default: '1' + VpcId: + Type: String + Description: Name of the value to import for the VpcId (not used for Batch) + +Resources: + + BatchJob: + Type: 'AWS::Batch::JobDefinition' + Properties: + JobDefinitionName: !Sub ${Namespace}-batch-${ServiceName}-${EnvironmentName} + Type: container + Parameters: + referenceFile: s3://path/to/a/reffile + targetFile: s3://path/to/a/targetfile + otherParams: 'params' + ContainerProperties: + Image: !Ref ImageUrl + Vcpus: !Ref ServiceVCpu + Memory: !Ref ServiceMemory + JobRoleArn: !Ref BatchJobRoleArn + Command: + - command-from-config + RetryStrategy: + Attempts: !Ref RetryAttemps + + +Outputs: + BatchJobDefinitionArn: + Value: !Ref BatchJob \ No newline at end of file diff --git a/templates/assets/cloudformation/service-iam.yml b/templates/assets/cloudformation/service-iam.yml index 935e2f82..e22fb5bd 100644 --- a/templates/assets/cloudformation/service-iam.yml +++ b/templates/assets/cloudformation/service-iam.yml @@ -21,6 +21,7 @@ Parameters: - "ec2" - "eks" - "eks-fargate" + - "batch" CodeDeployBucket: Type: String Description: Name of bucket to use for the CodeDeployBucket artifacts @@ -49,6 +50,10 @@ Conditions: - "Fn::Equals": - !Ref Provider - 'eks-fargate' + IsBatchService: + "Fn::Equals": + - !Ref Provider + - 'batch' HasDatabase: "Fn::Not": - "Fn::Equals": @@ -405,6 +410,47 @@ Resources: - logs:DescribeLogGroups - logs:DescribeLogStreams Resource: '*' + + BatchTaskRole: + Type: AWS::IAM::Role + Condition: IsBatchService + Properties: + RoleName: !Sub ${Namespace}-batch-${ServiceName}-${EnvironmentName}-task-${AWS::Region} + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: + - ecs-tasks.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: ecs-task + PolicyDocument: + Statement: + - Effect: Allow + Action: + - ecs:DescribeTasks + Resource: "*" + Condition: + ArnEquals: + "ecs:cluster": !Sub arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:cluster/${Namespace}-environment-${EnvironmentName} + - PolicyName: task-execution + PolicyDocument: + Statement: + - Effect: Allow + Action: + - ecr:GetAuthorizationToken + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - logs:DescribeLogGroups + - logs:DescribeLogStreams + Resource: '*' Outputs: DatabaseKeyArn: @@ -456,3 +502,10 @@ Outputs: - IsEcsService - !GetAtt ApplicationAutoScalingRole.Arn - '' + BatchJobRoleArn: + Description: Role assummed by AWS Batch job container + Value: + Fn::If: + - IsBatchService + - !GetAtt BatchTaskRole.Arn + - '' \ No newline at end of file diff --git a/workflows/environment_common.go b/workflows/environment_common.go index 38938f5d..c2876f80 100644 --- a/workflows/environment_common.go +++ b/workflows/environment_common.go @@ -57,6 +57,12 @@ func (workflow *environmentWorkflow) isEc2Provider() Conditional { } } +func (workflow *environmentWorkflow) isBatchProvider() Conditional { + return func() bool { + return strings.EqualFold(string(workflow.environment.Provider), string(common.EnvProviderBatch)) + } +} + func (workflow *environmentWorkflow) connectKubernetes(muNamespace string, provider common.KubernetesResourceManagerProvider) Executor { return func() error { clusterName := common.CreateStackName(muNamespace, common.StackTypeEnv, workflow.environment.Name) diff --git a/workflows/service_common.go b/workflows/service_common.go index 505a86f3..133b738a 100644 --- a/workflows/service_common.go +++ b/workflows/service_common.go @@ -13,6 +13,7 @@ import ( type serviceWorkflow struct { envStack *common.Stack lbStack *common.Stack + lbDisabled bool artifactProvider common.ArtifactProvider serviceName string serviceTag string @@ -29,6 +30,7 @@ type serviceWorkflow struct { microserviceTaskDefinitionArn string ecsEventsRoleArn string kubernetesResourceManager common.KubernetesResourceManager + providerOverride string } // Find a service in config, by name and set the reference @@ -54,6 +56,11 @@ func (workflow *serviceWorkflow) serviceLoader(ctx *common.Context, tag string, workflow.repoName = ctx.Config.Repo.Slug workflow.priority = ctx.Config.Service.Priority + // allow to override provider (i.e. deploy batch job definition) + // todo: validate for valid values + workflow.providerOverride = ctx.Config.Service.ProviderOverride + workflow.lbDisabled = false + if provider == "" { dockerfile := ctx.Config.Service.Dockerfile if dockerfile == "" { @@ -115,6 +122,12 @@ func (workflow *serviceWorkflow) isEc2Provider() Conditional { } } +func (workflow *serviceWorkflow) isBatchProvider() Conditional { + return func() bool { + return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderBatch)) + } +} + func (workflow *serviceWorkflow) serviceInput(ctx *common.Context, serviceName string) Executor { return func() error { // Repo Name diff --git a/workflows/service_deploy.go b/workflows/service_deploy.go index fc3d44fa..9f9dfbbc 100644 --- a/workflows/service_deploy.go +++ b/workflows/service_deploy.go @@ -49,6 +49,13 @@ func NewServiceDeployer(ctx *common.Context, environmentName string, tag string) workflow.serviceEksDeployer(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName), // TODO - placeholder for doing serviceCreateSchedules for EKS, leaving out-of-scope ), nil), + newConditionalExecutor(workflow.isBatchProvider(), + newPipelineExecutor( + workflow.serviceRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, environmentName), + workflow.serviceRepoUpserter(ctx.Config.Namespace, &ctx.Config.Service, ctx.StackManager, ctx.StackManager), + workflow.serviceApplyBatchParams(&ctx.Config.Service, stackParams, ctx.RolesetManager), + workflow.serviceBatchDeployer(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.StackManager), + ), nil), ) } @@ -90,17 +97,32 @@ func checkPriorityNotInUse(elbRuleLister common.ElbRuleLister, listenerArn strin func (workflow *serviceWorkflow) serviceEnvironmentLoader(namespace string, environmentName string, stackWaiter common.StackWaiter) Executor { return func() error { + lbStackName := common.CreateStackName(namespace, common.StackTypeLoadBalancer, environmentName) workflow.lbStack = stackWaiter.AwaitFinalStatus(lbStackName) envStackName := common.CreateStackName(namespace, common.StackTypeEnv, environmentName) workflow.envStack = stackWaiter.AwaitFinalStatus(envStackName) + // batch job definition can be deployed without a real environment, creating fake env (if needed) and overriding provider to 'batch' + if workflow.providerOverride == common.EnvProviderBatch { + workflow.createFakeBatchEnvironment(envStackName, namespace, environmentName) + } + if workflow.envStack == nil { return fmt.Errorf("Unable to find stack '%s' for environment '%s'", envStackName, environmentName) } - if workflow.isEcsProvider()() { + // // disable LB for Batch deployments + if workflow.isBatchProvider()() { + workflow.lbDisabled = true + } + + if workflow.lbStack == nil && workflow.lbDisabled != true { + return fmt.Errorf("Unable to find loadbalancer '%s' for environment '%s'", lbStackName, environmentName) + } + + if workflow.isEcsProvider()() || workflow.isBatchProvider()() { workflow.artifactProvider = common.ArtifactProviderEcr } else { workflow.artifactProvider = common.ArtifactProviderS3 @@ -110,6 +132,18 @@ func (workflow *serviceWorkflow) serviceEnvironmentLoader(namespace string, envi } } +func (workflow *serviceWorkflow) createFakeBatchEnvironment(envStackName string, namespace string, environmentName string) { + if workflow.envStack == nil { + outputs := make(map[string]string) + outputs["ElbHttpListenerArn"] = "foo" + outputs["ElbHttpsListenerArn"] = "foo" + tags := make(map[string]string) + workflow.envStack = &common.Stack{Name: envStackName, Status: common.StackStatusCreateComplete, Outputs: outputs, Tags: tags} + } + workflow.envStack.Tags["provider"] = common.EnvProviderBatch + workflow.envStack.Tags["environment"] = environmentName +} + func (workflow *serviceWorkflow) serviceRolesetUpserter(rolesetUpserter common.RolesetUpserter, rolesetGetter common.RolesetGetter, environmentName string) Executor { return func() error { err := rolesetUpserter.UpsertCommonRoleset() @@ -307,11 +341,44 @@ func (workflow *serviceWorkflow) serviceApplyEc2Params(params map[string]string, } } +func (workflow *serviceWorkflow) serviceApplyBatchParams(service *common.Service, params map[string]string, rolesetGetter common.RolesetGetter) Executor { + return func() error { + // ... + params["ImageUrl"] = workflow.serviceImage + + cpu := common.CPUMemorySupport[0] + if service.CPU != 0 { + params["ServiceVCpu"] = strconv.Itoa(service.CPU) + cpu = matchRequestedCPU(service.CPU, cpu) + } + + memory := cpu.Memory[0] + if service.Memory != 0 { + params["ServiceMemory"] = strconv.Itoa(service.Memory) + memory = matchRequestedMemory(service.Memory, cpu, memory) + } + + serviceRoleset, err := rolesetGetter.GetServiceRoleset(workflow.envStack.Tags["environment"], workflow.serviceName) + if err != nil { + return err + } + + params["BatchJobRoleArn"] = serviceRoleset["BatchJobRoleArn"] + + return nil + } +} + func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, service *common.Service, params map[string]string, environmentName string, stackWaiter common.StackWaiter, elbRuleLister common.ElbRuleLister, paramGetter common.ParamGetter) Executor { return func() error { + //if !workflow.isBatchProvider()() { params["VpcId"] = fmt.Sprintf("%s-VpcId", workflow.envStack.Name) + //} + + svcStackName := common.CreateStackName(namespace, common.StackTypeService, workflow.serviceName, environmentName) + svcStack := stackWaiter.AwaitFinalStatus(svcStackName) nextAvailablePriority := 0 if workflow.lbStack != nil { @@ -327,6 +394,26 @@ func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, serv nextAvailablePriority = 1 + getMaxPriority(elbRuleLister, workflow.lbStack.Outputs["ElbHttpsListenerArn"]) } } + + // assign priority + if workflow.priority > 0 { + // make sure manually specified priority is not already in use + err := checkPriorityNotInUse(elbRuleLister, workflow.lbStack.Outputs["ElbHttpListenerArn"], []int{workflow.priority, workflow.priority + 1}) + if err != nil { + return err + } + + params["PathListenerRulePriority"] = strconv.Itoa(workflow.priority) + params["HostListenerRulePriority"] = strconv.Itoa(workflow.priority + 1) + } else if svcStack != nil && svcStack.Status != "ROLLBACK_COMPLETE" { + // no value in config, and this is an update...use prior value + params["PathListenerRulePriority"] = "" + params["HostListenerRulePriority"] = "" + } else { + // no value in config, and this is a create...use next available + params["PathListenerRulePriority"] = strconv.Itoa(nextAvailablePriority) + params["HostListenerRulePriority"] = strconv.Itoa(nextAvailablePriority + 1) + } } common.NewMapElementIfNotZero(params, "TargetCPUUtilization", service.TargetCPUUtilization) @@ -346,28 +433,6 @@ func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, serv params["DatabaseMasterPassword"] = dbPass } - svcStackName := common.CreateStackName(namespace, common.StackTypeService, workflow.serviceName, environmentName) - svcStack := stackWaiter.AwaitFinalStatus(svcStackName) - - if workflow.priority > 0 { - // make sure manually specified priority is not already in use - err := checkPriorityNotInUse(elbRuleLister, workflow.lbStack.Outputs["ElbHttpListenerArn"], []int{workflow.priority, workflow.priority + 1}) - if err != nil { - return err - } - - params["PathListenerRulePriority"] = strconv.Itoa(workflow.priority) - params["HostListenerRulePriority"] = strconv.Itoa(workflow.priority + 1) - } else if svcStack != nil && svcStack.Status != "ROLLBACK_COMPLETE" { - // no value in config, and this is an update...use prior value - params["PathListenerRulePriority"] = "" - params["HostListenerRulePriority"] = "" - } else { - // no value in config, and this is a create...use next available - params["PathListenerRulePriority"] = strconv.Itoa(nextAvailablePriority) - params["HostListenerRulePriority"] = strconv.Itoa(nextAvailablePriority + 1) - } - params["Namespace"] = namespace params["EnvironmentName"] = environmentName params["ServiceName"] = workflow.serviceName @@ -532,6 +597,41 @@ func (workflow *serviceWorkflow) serviceEksDeployer(namespace string, service *c } } +func (workflow *serviceWorkflow) serviceBatchDeployer(namespace string, service *common.Service, stackParams map[string]string, environmentName string, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor { + return func() error { + log.Noticef("Deploying batch service '%s' to '%s' from '%s'", workflow.serviceName, environmentName, workflow.serviceImage) + + svcStackName := common.CreateStackName(namespace, common.StackTypeService, workflow.serviceName, environmentName) + + resolveServiceEnvironment(service, environmentName) + + tags := createTagMap(&ServiceTags{ + Service: workflow.serviceName, + Environment: environmentName, + Type: common.StackTypeService, + Provider: workflow.envStack.Outputs["provider"], + Revision: workflow.codeRevision, + Repo: workflow.repoName, + }) + + err := stackUpserter.UpsertStack(svcStackName, common.TemplateServiceBatch, service, stackParams, tags, "", workflow.cloudFormationRoleArn) + if err != nil { + return err + } + log.Debugf("Waiting for stack '%s' to complete", svcStackName) + stack := stackWaiter.AwaitFinalStatus(svcStackName) + if stack == nil { + return fmt.Errorf("Unable to create stack %s", svcStackName) + } + if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") { + return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason) + } + //workflow.batchJobDefinitionArn = stack.Outputs["BatchJobDefinitionArn"] + + return nil + } +} + func (workflow *serviceWorkflow) serviceCreateSchedules(namespace string, service *common.Service, environmentName string, stackWaiter common.StackWaiter, stackUpserter common.StackUpserter) Executor { return func() error { log.Noticef("Creating schedules for service '%s' to '%s'", workflow.serviceName, environmentName) From bea886ad48c420d524975ced30706d44aef49c40 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Fri, 26 Oct 2018 00:55:00 +0000 Subject: [PATCH 02/15] working POC --- templates/assets/cloudformation/common-iam.yml | 15 +++++++++++++++ templates/assets/cloudformation/service-iam.yml | 2 +- workflows/service_deploy.go | 4 +--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/templates/assets/cloudformation/common-iam.yml b/templates/assets/cloudformation/common-iam.yml index 4161300b..1e89c639 100644 --- a/templates/assets/cloudformation/common-iam.yml +++ b/templates/assets/cloudformation/common-iam.yml @@ -384,6 +384,21 @@ Resources: StringLike: iam:AWSServiceName: rds.amazonaws.com Effect: Allow + - PolicyName: aws-batch-jobs + PolicyDocument: + Version: '2012-10-17' + Statement: + - Action: + - batch:RegisterJobDefinition + - batch:DeregisterJobDefinition + - batch:DescribeJobDefinitions + Resource: '*' + Effect: Allow + - Action: + - iam:PassRole + Resource: + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${Namespace}-service-*-batchjob-${AWS::Region} + Effect: Allow Outputs: CloudFormationRoleArn: Description: Role assummed by CloudFormation diff --git a/templates/assets/cloudformation/service-iam.yml b/templates/assets/cloudformation/service-iam.yml index e22fb5bd..b687c64c 100644 --- a/templates/assets/cloudformation/service-iam.yml +++ b/templates/assets/cloudformation/service-iam.yml @@ -415,7 +415,7 @@ Resources: Type: AWS::IAM::Role Condition: IsBatchService Properties: - RoleName: !Sub ${Namespace}-batch-${ServiceName}-${EnvironmentName}-task-${AWS::Region} + RoleName: !Sub ${Namespace}-service-${ServiceName}-${EnvironmentName}-batchjob-${AWS::Region} AssumeRolePolicyDocument: Statement: - Effect: Allow diff --git a/workflows/service_deploy.go b/workflows/service_deploy.go index 9f9dfbbc..9607bdfd 100644 --- a/workflows/service_deploy.go +++ b/workflows/service_deploy.go @@ -373,15 +373,13 @@ func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, serv params map[string]string, environmentName string, stackWaiter common.StackWaiter, elbRuleLister common.ElbRuleLister, paramGetter common.ParamGetter) Executor { return func() error { - //if !workflow.isBatchProvider()() { params["VpcId"] = fmt.Sprintf("%s-VpcId", workflow.envStack.Name) - //} svcStackName := common.CreateStackName(namespace, common.StackTypeService, workflow.serviceName, environmentName) svcStack := stackWaiter.AwaitFinalStatus(svcStackName) nextAvailablePriority := 0 - if workflow.lbStack != nil { + if workflow.lbStack != nil && workflow.lbDisabled != true { if workflow.lbStack.Outputs["ElbHttpListenerArn"] != "" { params["ElbHttpListenerArn"] = fmt.Sprintf("%s-ElbHttpListenerArn", workflow.lbStack.Name) if workflow.priority < 1 { From ed68e6c603bfef71317b036d6aa0ac4a26d90735 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Fri, 26 Oct 2018 01:04:10 +0000 Subject: [PATCH 03/15] cleanup --- workflows/service_common.go | 2 +- workflows/service_deploy.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/workflows/service_common.go b/workflows/service_common.go index 133b738a..ec4caea0 100644 --- a/workflows/service_common.go +++ b/workflows/service_common.go @@ -57,7 +57,7 @@ func (workflow *serviceWorkflow) serviceLoader(ctx *common.Context, tag string, workflow.priority = ctx.Config.Service.Priority // allow to override provider (i.e. deploy batch job definition) - // todo: validate for valid values + // todo: validate values workflow.providerOverride = ctx.Config.Service.ProviderOverride workflow.lbDisabled = false diff --git a/workflows/service_deploy.go b/workflows/service_deploy.go index 9607bdfd..99c5bd42 100644 --- a/workflows/service_deploy.go +++ b/workflows/service_deploy.go @@ -118,9 +118,10 @@ func (workflow *serviceWorkflow) serviceEnvironmentLoader(namespace string, envi workflow.lbDisabled = true } - if workflow.lbStack == nil && workflow.lbDisabled != true { - return fmt.Errorf("Unable to find loadbalancer '%s' for environment '%s'", lbStackName, environmentName) - } + // this check is not needed (?) + // if workflow.lbStack == nil && workflow.lbDisabled != true { + // return fmt.Errorf("Unable to find loadbalancer '%s' for environment '%s'", lbStackName, environmentName) + // } if workflow.isEcsProvider()() || workflow.isBatchProvider()() { workflow.artifactProvider = common.ArtifactProviderEcr From 3e1f64293eccb738010bfb7adeb0b4093bbfaa13 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Fri, 26 Oct 2018 19:06:24 +0000 Subject: [PATCH 04/15] batch: command, parameters, enviroment, attempts, timeout support --- common/types.go | 9 ++- .../assets/cloudformation/service-batch.yml | 59 +++++++++++++++---- workflows/service_deploy.go | 3 + 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/common/types.go b/common/types.go index 8fb68b8b..72cb4193 100644 --- a/common/types.go +++ b/common/types.go @@ -162,7 +162,14 @@ type Service struct { Schedule []Schedule `yaml:"schedules,omitempty"` TargetCPUUtilization int `yaml:"targetCPUUtilization,omitempty" validate:"max=100"` DiscoveryTTL string `yaml:"discoveryTTL,omitempty"` - Roles struct { + + // Batch + Parameters map[string]interface{} `yaml:"parameters,omitempty"` + Command []string `yaml:"command,omitempty"` + Timeout int `yaml:"timeout,omitempty"` + RetryAttempts int `yaml:"retryAttempts,omitempty"` + + Roles struct { Ec2Instance string `yaml:"ec2Instance,omitempty" validate:"validateRoleARN"` CodeDeploy string `yaml:"codeDeploy,omitempty" validate:"validateRoleARN"` EcsEvents string `yaml:"ecsEvents,omitempty" validate:"validateRoleARN"` diff --git a/templates/assets/cloudformation/service-batch.yml b/templates/assets/cloudformation/service-batch.yml index e53d0af1..2100d317 100644 --- a/templates/assets/cloudformation/service-batch.yml +++ b/templates/assets/cloudformation/service-batch.yml @@ -18,21 +18,37 @@ Parameters: ServiceMemory: Type: String Description: Memory to allocate to container (in MiB) - Default: '300' + Default: '512' ImageUrl: Type: String Description: Docker Image URL (or name, like 'amazonlinux') BatchJobRoleArn: Type: String Description: IAM Role assumed by AWS Batch job (ecs task) - RetryAttemps: + RetryAttempts: Type: String Description: Number of retry attempts - Default: '1' + Default: '' + Timeout: + Type: String + Description: Job timeout in seconds. Job is terminated after timing out + Default: '' VpcId: Type: String Description: Name of the value to import for the VpcId (not used for Batch) +Conditions: + HasRetryAttempts: + "Fn::Not": + - "Fn::Equals": + - !Ref RetryAttempts + - '' + HasTimeout: + "Fn::Not": + - "Fn::Equals": + - !Ref Timeout + - '' + Resources: BatchJob: @@ -40,19 +56,42 @@ Resources: Properties: JobDefinitionName: !Sub ${Namespace}-batch-${ServiceName}-${EnvironmentName} Type: container - Parameters: - referenceFile: s3://path/to/a/reffile - targetFile: s3://path/to/a/targetfile - otherParams: 'params' + Parameters: + {{with .Parameters}} + {{range $key, $val := .}} + {{$key}}: !Sub {{$val}} + {{end}} + {{end}} ContainerProperties: Image: !Ref ImageUrl Vcpus: !Ref ServiceVCpu Memory: !Ref ServiceMemory JobRoleArn: !Ref BatchJobRoleArn - Command: - - command-from-config + Command: + {{with .Command}} + {{range $key, $val := .}} + - {{$val}} + {{end}} + {{end}} + Environment: + {{with .Environment}} + {{range $key, $val := .}} + - Name: {{$key}} + Value: !Sub {{$val}} + {{end}} + {{end}} RetryStrategy: - Attempts: !Ref RetryAttemps + Attempts: + Fn::If: + - HasRetryAttempts + - !Ref RetryAttempts + - !Ref AWS::NoValue + Timeout: + AttemptDurationSeconds: + Fn::If: + - HasTimeout + - !Ref Timeout + - !Ref AWS::NoValue Outputs: diff --git a/workflows/service_deploy.go b/workflows/service_deploy.go index 99c5bd42..12768443 100644 --- a/workflows/service_deploy.go +++ b/workflows/service_deploy.go @@ -345,6 +345,9 @@ func (workflow *serviceWorkflow) serviceApplyEc2Params(params map[string]string, func (workflow *serviceWorkflow) serviceApplyBatchParams(service *common.Service, params map[string]string, rolesetGetter common.RolesetGetter) Executor { return func() error { // ... + // Environment, Parameters, Command are handled by go templating engine within service-batch.yml + common.NewMapElementIfNotZero(params, "Timeout", service.Timeout) + common.NewMapElementIfNotZero(params, "RetryAttempts", service.RetryAttempts) params["ImageUrl"] = workflow.serviceImage cpu := common.CPUMemorySupport[0] From 097aced0fd30e78eae8391829280eea6facae2a1 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Thu, 1 Nov 2018 03:56:32 +0000 Subject: [PATCH 05/15] CF: check if parameter exist is prev stack before setting UsePreviousValue --- provider/aws/cloudformation.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/provider/aws/cloudformation.go b/provider/aws/cloudformation.go index 657892ce..59ed67d0 100644 --- a/provider/aws/cloudformation.go +++ b/provider/aws/cloudformation.go @@ -65,14 +65,22 @@ func newStackManager(sess *session.Session, extensionsManager common.ExtensionsM } -func buildStackParameters(parameters map[string]string) []*cloudformation.Parameter { +func buildStackParameters(parameters map[string]string, stack *common.Stack) []*cloudformation.Parameter { + stackParameters := make([]*cloudformation.Parameter, 0, len(parameters)) for key, value := range parameters { + // Stack exists, check if parameter exists. Needed to avoid: + // ValidationError: Invalid input for parameter key DatabaseName. Cannot specify usePreviousValue as true for a parameter key not in the previous template + usePrevVal := true + if stack != nil && stack.Status != "" { + _, usePrevVal = stack.Parameters[key] + } + stackParameters = append(stackParameters, &cloudformation.Parameter{ ParameterKey: aws.String(key), ParameterValue: aws.String(value), - UsePreviousValue: aws.Bool(value == ""), + UsePreviousValue: aws.Bool(value == "" && usePrevVal), }) } return stackParameters @@ -333,7 +341,7 @@ func (cfnMgr *cloudformationStackManager) UpsertStack(stackName string, template if err != nil { return err } - stackParameters := buildStackParameters(parameters) + stackParameters := buildStackParameters(parameters, stack) // stack tags tags, err = cfnMgr.extensionsManager.DecorateStackTags(stackName, tags) From a00ca0c7898e87aa520c813eca91c2e8e60d4ab2 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Thu, 1 Nov 2018 03:57:57 +0000 Subject: [PATCH 06/15] initial batch job exec (submit) code --- common/batch.go | 33 +++++++ provider/aws/batch.go | 207 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+) create mode 100644 common/batch.go create mode 100644 provider/aws/batch.go diff --git a/common/batch.go b/common/batch.go new file mode 100644 index 00000000..e0bafe27 --- /dev/null +++ b/common/batch.go @@ -0,0 +1,33 @@ +package common + +import ( + "github.com/aws/aws-sdk-go/service/batch" +) + +/* +// TaskContainerLister for listing tasks with containers +type TaskContainerLister interface { + ListTasks(namespace string, environment string, serviceName string) ([]Task, error) +} + +// TaskStopper for restarting tasks +type TaskStopper interface { + StopTask(namespace string, environment string, task string) error +} + +*/ + +// BatchJobSubmitResult describes the output result from Batch call to SubmitJob +type BatchJobSubmitResult *batch.SubmitJobOutput + +// BatchJobExecutor for executing batch jobs against an environment +type BatchJobExecutor interface { + ExecuteCommand(namespace string, task Task) (BatchJobSubmitResult, error) +} + +// BatchManager composite of all task capabilities +type BatchManager interface { + //TaskContainerLister + //TaskStopper + BatchJobExecutor +} diff --git a/provider/aws/batch.go b/provider/aws/batch.go new file mode 100644 index 00000000..af2a73db --- /dev/null +++ b/provider/aws/batch.go @@ -0,0 +1,207 @@ +package aws + +import ( + //"strings" + + //"github.com/aws/aws-sdk-go/aws" + //"github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/batch" + //"github.com/pkg/errors" + "github.com/stelligent/mu/common" +) + +type batchManager struct { + batchAPI *batch.Batch + stackManager common.StackGetter +} + +/* +func getFlagOrValue(flag string, value string) string { + var actual string + if len(flag) == Zero { + actual = value + } else { + actual = flag + } + return actual +} + + +func newBatchManager(sess *session.Session, stackManager *common.StackManager) (common.BatchManager, error) { + //log.Debug(EcsConnectionLog) + + batchAPI := batch.New(sess) + + return &batchManager{ + batchAPI: batchAPI, + stackManager: *stackManager, + }, nil +} + + + +func (taskMgr *ecsTaskManager) getTaskRunInput(namespace string, task common.Task) (*ecs.RunTaskInput, error) { + ecsStack, err := taskMgr.getECSStack(namespace, task.Service, task.Environment) + if err != nil { + return nil, err + } + + taskDefinitionOutput := ecsStack.Outputs[ECSTaskDefinitionOutputKey] + ecsClusterOutput := ecsStack.Outputs[ECSClusterOutputKey] + ecsTaskDefinition := getFlagOrValue(task.TaskDefinition, taskDefinitionOutput) + ecsCluster := getFlagOrValue(task.Cluster, ecsClusterOutput) + ecsServiceName := ecsStack.Parameters[ECSServiceNameParameterKey] + log.Debugf(ExecuteECSInputParameterLog, task.Environment, ecsServiceName, ecsCluster, ecsTaskDefinition) + + command := make([]*string, len(task.Command)) + for i, commandPart := range task.Command { + command[i] = aws.String(strings.TrimSpace(commandPart)) + } + + ecsRunTaskInput := &ecs.RunTaskInput{ + Cluster: aws.String(ecsCluster), + TaskDefinition: aws.String(ecsTaskDefinition), + Count: aws.Int64(1), + Overrides: &ecs.TaskOverride{ + ContainerOverrides: []*ecs.ContainerOverride{ + { + Name: aws.String(ecsServiceName), + Command: command, + }, + }, + }, + } + log.Debugf(ExecuteECSInputContentsLog, ecsRunTaskInput) + return ecsRunTaskInput, nil +} + +func (taskMgr *ecsTaskManager) runTask(runTaskInput *ecs.RunTaskInput) (common.ECSRunTaskResult, error) { + resp, err := taskMgr.ecsAPI.RunTask(runTaskInput) + log.Debugf(ExecuteECSResultContentsLog, resp, err) + log.Info(ExecuteCommandFinishLog) + if err != nil { + return nil, err + } + + return resp, nil +} + +// ExecuteCommand runs a command for a specific environment +func (taskMgr *ecsTaskManager) ExecuteCommand(namespace string, task common.Task) (common.ECSRunTaskResult, error) { + log.Infof(ExecuteCommandStartLog, task.Command, task.Environment, task.Service) + + ecsRunTaskInput, err := taskMgr.getTaskRunInput(namespace, task) + if err != nil { + return nil, err + } + + return taskMgr.runTask(ecsRunTaskInput) +} + +// ExecuteCommand runs a command for a specific environment +func (taskMgr *ecsTaskManager) ListTasks(namespace string, environment string, serviceName string) ([]common.Task, error) { + cluster := common.CreateStackName(namespace, common.StackTypeEnv, environment) + serviceInputParameters := &ecs.ListServicesInput{ + Cluster: aws.String(cluster), + } + tasks := []common.Task{} + + serviceOutput, err := taskMgr.ecsAPI.ListServices(serviceInputParameters) + if err != nil { + return nil, err + } + + for _, serviceARN := range serviceOutput.ServiceArns { + log.Debugf(SvcListTasksLog, environment, cluster, serviceName) + listTaskInput := &ecs.ListTasksInput{ + Cluster: aws.String(cluster), + ServiceName: aws.String(*serviceARN), + } + listTaskOutput, err := taskMgr.ecsAPI.ListTasks(listTaskInput) + if err != nil { + return nil, err + } + + for _, ecsTask := range listTaskOutput.TaskArns { + describeTaskParams := &ecs.DescribeTasksInput{ + Tasks: []*string{ + aws.String(*ecsTask), + }, + Cluster: aws.String(cluster), + } + taskOutput, err := taskMgr.ecsAPI.DescribeTasks(describeTaskParams) + if err != nil { + continue + } + + for _, ecsTask := range taskOutput.Tasks { + task, err := getTaskDetail(ecsTask, taskMgr, cluster, environment, serviceName) + if err != nil { + continue + } + tasks = append(tasks, *task) + } + } + } + return tasks, nil +} + +func (taskMgr *ecsTaskManager) StopTask(namespace string, environment string, task string) error { + cluster := common.CreateStackName(namespace, common.StackTypeEnv, environment) + stopTaskInput := &ecs.StopTaskInput{ + Cluster: &cluster, + Task: &task, + } + _, err := taskMgr.ecsAPI.StopTask(stopTaskInput) + + return err +} + +func getTaskDetail(ecsTask *ecs.Task, taskMgr *ecsTaskManager, cluster string, environment string, serviceName string) (*common.Task, error) { + containers := []common.Container{} + if len(ecsTask.Containers) > Zero { + for _, container := range ecsTask.Containers { + if *container.Name != serviceName && len(serviceName) != Zero { + return nil, errors.New(common.Empty) + } + if ecsTask.ContainerInstanceArn != nil { + containers = append(containers, getContainer(taskMgr, cluster, *ecsTask.ContainerInstanceArn, *container)) + } else { + containers = append(containers, common.Container{Name: *container.Name, Instance: "FARGATE"}) + } + } + } + task := common.Task{ + Name: strings.Split(*ecsTask.TaskArn, TaskARNSeparator)[1], + Environment: environment, + Service: serviceName, + Status: aws.StringValue(ecsTask.LastStatus), + Containers: containers, + } + log.Debugf(SvcTaskDetailLog, task) + return &task, nil +} + +func getContainer(taskMgr *ecsTaskManager, cluster string, instanceARN string, container ecs.Container) common.Container { + containerParams := &ecs.DescribeContainerInstancesInput{ + ContainerInstances: []*string{ + aws.String(instanceARN), + }, + Cluster: aws.String(cluster), + } + instanceOutput, err := taskMgr.ecsAPI.DescribeContainerInstances(containerParams) + if err != nil { + return common.Container{} + } + ec2InstanceID := *instanceOutput.ContainerInstances[0].Ec2InstanceId + return common.Container{Name: *container.Name, Instance: ec2InstanceID} +} + +func (taskMgr *ecsTaskManager) getECSStack(namespace string, serviceName string, environment string) (*common.Stack, error) { + envStackName := common.CreateStackName(namespace, common.StackTypeService, serviceName, environment) + log.Infof(SvcCmdStackLog, envStackName) + + return taskMgr.stackManager.GetStack(envStackName) +} + +*/ From dfb312738795ba03ea82c1b5df5721c6c8ff254a Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Thu, 1 Nov 2018 07:03:50 +0000 Subject: [PATCH 07/15] fix ServiceProtocol casing: CF expects upper --- workflows/service_deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/service_deploy.go b/workflows/service_deploy.go index 12768443..60d4aa53 100644 --- a/workflows/service_deploy.go +++ b/workflows/service_deploy.go @@ -439,7 +439,7 @@ func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, serv params["EnvironmentName"] = environmentName params["ServiceName"] = workflow.serviceName common.NewMapElementIfNotZero(params, "ServicePort", service.Port) - common.NewMapElementIfNotEmpty(params, "ServiceProtocol", string(service.Protocol)) + common.NewMapElementIfNotEmpty(params, "ServiceProtocol", strings.ToUpper(string(service.Protocol))) common.NewMapElementIfNotEmpty(params, "ServiceHealthEndpoint", service.HealthEndpoint) common.NewMapElementIfNotZero(params, "ServiceDesiredCount", service.DesiredCount) common.NewMapElementIfNotZero(params, "ServiceMinSize", service.MinSize) From 8d2d2856d532ecff7df649089ce10bc0153de2d0 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Mon, 26 Nov 2018 22:30:23 +0000 Subject: [PATCH 08/15] cleanup --- common/batch.go | 33 ------- provider/aws/batch.go | 207 ------------------------------------------ 2 files changed, 240 deletions(-) delete mode 100644 common/batch.go delete mode 100644 provider/aws/batch.go diff --git a/common/batch.go b/common/batch.go deleted file mode 100644 index e0bafe27..00000000 --- a/common/batch.go +++ /dev/null @@ -1,33 +0,0 @@ -package common - -import ( - "github.com/aws/aws-sdk-go/service/batch" -) - -/* -// TaskContainerLister for listing tasks with containers -type TaskContainerLister interface { - ListTasks(namespace string, environment string, serviceName string) ([]Task, error) -} - -// TaskStopper for restarting tasks -type TaskStopper interface { - StopTask(namespace string, environment string, task string) error -} - -*/ - -// BatchJobSubmitResult describes the output result from Batch call to SubmitJob -type BatchJobSubmitResult *batch.SubmitJobOutput - -// BatchJobExecutor for executing batch jobs against an environment -type BatchJobExecutor interface { - ExecuteCommand(namespace string, task Task) (BatchJobSubmitResult, error) -} - -// BatchManager composite of all task capabilities -type BatchManager interface { - //TaskContainerLister - //TaskStopper - BatchJobExecutor -} diff --git a/provider/aws/batch.go b/provider/aws/batch.go deleted file mode 100644 index af2a73db..00000000 --- a/provider/aws/batch.go +++ /dev/null @@ -1,207 +0,0 @@ -package aws - -import ( - //"strings" - - //"github.com/aws/aws-sdk-go/aws" - //"github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/batch" - //"github.com/pkg/errors" - "github.com/stelligent/mu/common" -) - -type batchManager struct { - batchAPI *batch.Batch - stackManager common.StackGetter -} - -/* -func getFlagOrValue(flag string, value string) string { - var actual string - if len(flag) == Zero { - actual = value - } else { - actual = flag - } - return actual -} - - -func newBatchManager(sess *session.Session, stackManager *common.StackManager) (common.BatchManager, error) { - //log.Debug(EcsConnectionLog) - - batchAPI := batch.New(sess) - - return &batchManager{ - batchAPI: batchAPI, - stackManager: *stackManager, - }, nil -} - - - -func (taskMgr *ecsTaskManager) getTaskRunInput(namespace string, task common.Task) (*ecs.RunTaskInput, error) { - ecsStack, err := taskMgr.getECSStack(namespace, task.Service, task.Environment) - if err != nil { - return nil, err - } - - taskDefinitionOutput := ecsStack.Outputs[ECSTaskDefinitionOutputKey] - ecsClusterOutput := ecsStack.Outputs[ECSClusterOutputKey] - ecsTaskDefinition := getFlagOrValue(task.TaskDefinition, taskDefinitionOutput) - ecsCluster := getFlagOrValue(task.Cluster, ecsClusterOutput) - ecsServiceName := ecsStack.Parameters[ECSServiceNameParameterKey] - log.Debugf(ExecuteECSInputParameterLog, task.Environment, ecsServiceName, ecsCluster, ecsTaskDefinition) - - command := make([]*string, len(task.Command)) - for i, commandPart := range task.Command { - command[i] = aws.String(strings.TrimSpace(commandPart)) - } - - ecsRunTaskInput := &ecs.RunTaskInput{ - Cluster: aws.String(ecsCluster), - TaskDefinition: aws.String(ecsTaskDefinition), - Count: aws.Int64(1), - Overrides: &ecs.TaskOverride{ - ContainerOverrides: []*ecs.ContainerOverride{ - { - Name: aws.String(ecsServiceName), - Command: command, - }, - }, - }, - } - log.Debugf(ExecuteECSInputContentsLog, ecsRunTaskInput) - return ecsRunTaskInput, nil -} - -func (taskMgr *ecsTaskManager) runTask(runTaskInput *ecs.RunTaskInput) (common.ECSRunTaskResult, error) { - resp, err := taskMgr.ecsAPI.RunTask(runTaskInput) - log.Debugf(ExecuteECSResultContentsLog, resp, err) - log.Info(ExecuteCommandFinishLog) - if err != nil { - return nil, err - } - - return resp, nil -} - -// ExecuteCommand runs a command for a specific environment -func (taskMgr *ecsTaskManager) ExecuteCommand(namespace string, task common.Task) (common.ECSRunTaskResult, error) { - log.Infof(ExecuteCommandStartLog, task.Command, task.Environment, task.Service) - - ecsRunTaskInput, err := taskMgr.getTaskRunInput(namespace, task) - if err != nil { - return nil, err - } - - return taskMgr.runTask(ecsRunTaskInput) -} - -// ExecuteCommand runs a command for a specific environment -func (taskMgr *ecsTaskManager) ListTasks(namespace string, environment string, serviceName string) ([]common.Task, error) { - cluster := common.CreateStackName(namespace, common.StackTypeEnv, environment) - serviceInputParameters := &ecs.ListServicesInput{ - Cluster: aws.String(cluster), - } - tasks := []common.Task{} - - serviceOutput, err := taskMgr.ecsAPI.ListServices(serviceInputParameters) - if err != nil { - return nil, err - } - - for _, serviceARN := range serviceOutput.ServiceArns { - log.Debugf(SvcListTasksLog, environment, cluster, serviceName) - listTaskInput := &ecs.ListTasksInput{ - Cluster: aws.String(cluster), - ServiceName: aws.String(*serviceARN), - } - listTaskOutput, err := taskMgr.ecsAPI.ListTasks(listTaskInput) - if err != nil { - return nil, err - } - - for _, ecsTask := range listTaskOutput.TaskArns { - describeTaskParams := &ecs.DescribeTasksInput{ - Tasks: []*string{ - aws.String(*ecsTask), - }, - Cluster: aws.String(cluster), - } - taskOutput, err := taskMgr.ecsAPI.DescribeTasks(describeTaskParams) - if err != nil { - continue - } - - for _, ecsTask := range taskOutput.Tasks { - task, err := getTaskDetail(ecsTask, taskMgr, cluster, environment, serviceName) - if err != nil { - continue - } - tasks = append(tasks, *task) - } - } - } - return tasks, nil -} - -func (taskMgr *ecsTaskManager) StopTask(namespace string, environment string, task string) error { - cluster := common.CreateStackName(namespace, common.StackTypeEnv, environment) - stopTaskInput := &ecs.StopTaskInput{ - Cluster: &cluster, - Task: &task, - } - _, err := taskMgr.ecsAPI.StopTask(stopTaskInput) - - return err -} - -func getTaskDetail(ecsTask *ecs.Task, taskMgr *ecsTaskManager, cluster string, environment string, serviceName string) (*common.Task, error) { - containers := []common.Container{} - if len(ecsTask.Containers) > Zero { - for _, container := range ecsTask.Containers { - if *container.Name != serviceName && len(serviceName) != Zero { - return nil, errors.New(common.Empty) - } - if ecsTask.ContainerInstanceArn != nil { - containers = append(containers, getContainer(taskMgr, cluster, *ecsTask.ContainerInstanceArn, *container)) - } else { - containers = append(containers, common.Container{Name: *container.Name, Instance: "FARGATE"}) - } - } - } - task := common.Task{ - Name: strings.Split(*ecsTask.TaskArn, TaskARNSeparator)[1], - Environment: environment, - Service: serviceName, - Status: aws.StringValue(ecsTask.LastStatus), - Containers: containers, - } - log.Debugf(SvcTaskDetailLog, task) - return &task, nil -} - -func getContainer(taskMgr *ecsTaskManager, cluster string, instanceARN string, container ecs.Container) common.Container { - containerParams := &ecs.DescribeContainerInstancesInput{ - ContainerInstances: []*string{ - aws.String(instanceARN), - }, - Cluster: aws.String(cluster), - } - instanceOutput, err := taskMgr.ecsAPI.DescribeContainerInstances(containerParams) - if err != nil { - return common.Container{} - } - ec2InstanceID := *instanceOutput.ContainerInstances[0].Ec2InstanceId - return common.Container{Name: *container.Name, Instance: ec2InstanceID} -} - -func (taskMgr *ecsTaskManager) getECSStack(namespace string, serviceName string, environment string) (*common.Stack, error) { - envStackName := common.CreateStackName(namespace, common.StackTypeService, serviceName, environment) - log.Infof(SvcCmdStackLog, envStackName) - - return taskMgr.stackManager.GetStack(envStackName) -} - -*/ From c8a42c8b1662f8048c56e292869bf22de3978f6f Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Mon, 26 Nov 2018 23:21:55 +0000 Subject: [PATCH 09/15] batch examples --- examples/batch/Dockerfile | 8 ++++++ examples/batch/README.md | 5 ++++ examples/batch/batch-job.sh | 13 ++++++++++ examples/batch/buildspec.yml | 12 +++++++++ examples/batch/mu.yml | 49 ++++++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+) create mode 100644 examples/batch/Dockerfile create mode 100644 examples/batch/README.md create mode 100644 examples/batch/batch-job.sh create mode 100644 examples/batch/buildspec.yml create mode 100644 examples/batch/mu.yml diff --git a/examples/batch/Dockerfile b/examples/batch/Dockerfile new file mode 100644 index 00000000..08a65817 --- /dev/null +++ b/examples/batch/Dockerfile @@ -0,0 +1,8 @@ +FROM amazonlinux:latest + +RUN yum -y install unzip aws-cli +ADD batch-job.sh /usr/local/bin/batch-job.sh +WORKDIR /tmp +USER nobody + +ENTRYPOINT ["/usr/local/bin/batch-job.sh"] \ No newline at end of file diff --git a/examples/batch/README.md b/examples/batch/README.md new file mode 100644 index 00000000..eed22f76 --- /dev/null +++ b/examples/batch/README.md @@ -0,0 +1,5 @@ +# Examples +These examples are not intended to be run directly. Rather, they serve as a reference that can be consulted when creating your own `mu.yml` files. + +For detailed steps to create your own project, check out the [quickstart](https://github.com/stelligent/mu/wiki/Quickstart#steps). + diff --git a/examples/batch/batch-job.sh b/examples/batch/batch-job.sh new file mode 100644 index 00000000..8a3b271f --- /dev/null +++ b/examples/batch/batch-job.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +date +# this will expose parameters passed from mu.yml command section +echo "Args: $@" +env +echo "This is my simple test job!." +echo "jobId: $AWS_BATCH_JOB_ID" +echo "jobQueue: $AWS_BATCH_JQ_NAME" +echo "computeEnvironment: $AWS_BATCH_CE_NAME" +sleep $1 +date +echo "bye bye!!" \ No newline at end of file diff --git a/examples/batch/buildspec.yml b/examples/batch/buildspec.yml new file mode 100644 index 00000000..2b6a64cb --- /dev/null +++ b/examples/batch/buildspec.yml @@ -0,0 +1,12 @@ +version: 0.1 + +phases: + build: + commands: + - echo "replace me with real build commands..." + +artifacts: + files: + - batch-job.sh + - mu.yml + - Dockerfile diff --git a/examples/batch/mu.yml b/examples/batch/mu.yml new file mode 100644 index 00000000..f2ab8a29 --- /dev/null +++ b/examples/batch/mu.yml @@ -0,0 +1,49 @@ +## See https://github.com/stelligent/mu/wiki/Services +service: + name: batch-sample + + # deployed as AWS Batch job + provider: batch + + # The relative path to the Dockerfile to build images (default: Dockerfile) + dockerfile: Dockerfile + + # The number of VCPUs to allocate to job (default: 2) + # Each vCPU is equivalent to 1,024 CPU shares. + cpu: 1 + + # The hard limit (in MiB) of memory to present to the container. If your container attempts to exceed the memory specified here, the container is killed. + # Read this: https://docs.aws.amazon.com/batch/latest/userguide/memory-management.html + memory: 128 + + # The time duration in seconds (measured from the job attempt's startedAt timestamp) + # after which AWS Batch terminates your jobs if they have not finished. + # If a job is terminated due to a timeout, it is not retried. + timeout: 0 + retryAttempts: 1 + + # our docker file already has a defined entry point script, these params + # will be appended to it + command: + - '--some-flag' + - Ref::param1 + + # Default parameters + # Parameter values replace the placeholders in the command, i.e. Ref::paramname. + # Usually final parameter values are specified on job submit + parameters: + param1: "Sample text passed to the batch job" + + ### Environment variables exposed to the containers + environment: + + #### Define this for each environment if you want APP_DEBUG=true + APP_DEBUG: + dev: 'true' + prod: 'false' + + APP_ENV: + dev: dev + prod: production + + \ No newline at end of file From e6d7ac09f49b42fab0eda5808d1352fe2b927cb8 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Tue, 27 Nov 2018 01:26:37 +0000 Subject: [PATCH 10/15] updated tests, reduced cyclo complexity, cleanup --- provider/aws/cloudformation_test.go | 26 ++++++++++- templates/assets/cloudformation/env-batch.yml | 0 .../assets/cloudformation/service-batch.yml | 1 - templates/template_test.go | 13 ++++++ workflows/service_deploy.go | 44 ++++++++++++------- workflows/service_deploy_test.go | 15 +++++-- 6 files changed, 76 insertions(+), 23 deletions(-) delete mode 100644 templates/assets/cloudformation/env-batch.yml diff --git a/provider/aws/cloudformation_test.go b/provider/aws/cloudformation_test.go index 52161762..75f9a628 100644 --- a/provider/aws/cloudformation_test.go +++ b/provider/aws/cloudformation_test.go @@ -342,17 +342,39 @@ func TestBuildParameters(t *testing.T) { paramMap := make(map[string]string) - parameters := buildStackParameters(paramMap) + parameters := buildStackParameters(paramMap, nil) assert.Equal(0, len(parameters)) paramMap["p1"] = "value 1" paramMap["p2"] = "value 2" - parameters = buildStackParameters(paramMap) + parameters = buildStackParameters(paramMap, nil) assert.Equal(2, len(parameters)) assert.Contains(*parameters[0].ParameterKey, "p") assert.Contains(*parameters[0].ParameterValue, "value") assert.Contains(*parameters[1].ParameterKey, "p") assert.Contains(*parameters[1].ParameterValue, "value") + + stackParameters := make([]*cloudformation.Parameter, 0, len(parameters)) + stackParameters = append(stackParameters, + &cloudformation.Parameter{ + ParameterKey: aws.String("ExistingParam"), + ParameterValue: aws.String("ExistingValue"), + }) + + stackDetails := cloudformation.Stack{ + StackName: aws.String("mu-environment-dev"), + CreationTime: aws.Time(time.Now()), + Tags: []*cloudformation.Tag{}, + StackStatus: aws.String("CREATE_COMPLETE"), + Parameters: stackParameters, + } + + stack := buildStack(&stackDetails) + paramMap["ExistingParam"] = "" // should have UsePreviousValue==true + paramMap["NewParam"] = "" // should have UsePreviousValue==false + parameters = buildStackParameters(paramMap, stack) + assert.Equal(*parameters[2].UsePreviousValue, true) + assert.Equal(*parameters[3].UsePreviousValue, false) } func TestTagParameters(t *testing.T) { diff --git a/templates/assets/cloudformation/env-batch.yml b/templates/assets/cloudformation/env-batch.yml deleted file mode 100644 index e69de29b..00000000 diff --git a/templates/assets/cloudformation/service-batch.yml b/templates/assets/cloudformation/service-batch.yml index 2100d317..af2613c6 100644 --- a/templates/assets/cloudformation/service-batch.yml +++ b/templates/assets/cloudformation/service-batch.yml @@ -1,4 +1,3 @@ ---- AWSTemplateFormatVersion: 2010-09-09 Description: MU AWS Batch job definition Parameters: diff --git a/templates/template_test.go b/templates/template_test.go index 46346f77..37a36aeb 100644 --- a/templates/template_test.go +++ b/templates/template_test.go @@ -68,6 +68,19 @@ func TestNewTemplate_assets(t *testing.T) { templateData = tdMap } + if templateName == "cloudformation/service-batch.yml" { + params := make(map[string]string) + params["testparam"] = "test" + env := make(map[string]string) + env["ENVVAR"] = "test" + command := []string{"test"} + tdMap := make(map[string]interface{}) + tdMap["Parameters"] = params + tdMap["Command"] = command + tdMap["Environment"] = env + templateData = tdMap + } + templateBody, err := GetAsset(templateName, ExecuteTemplate(templateData)) if err != nil { diff --git a/workflows/service_deploy.go b/workflows/service_deploy.go index 60d4aa53..aaa4098f 100644 --- a/workflows/service_deploy.go +++ b/workflows/service_deploy.go @@ -22,7 +22,8 @@ func NewServiceDeployer(ctx *common.Context, environmentName string, tag string) return newPipelineExecutor( workflow.serviceLoader(ctx, tag, ""), workflow.serviceEnvironmentLoader(ctx.Config.Namespace, environmentName, ctx.StackManager), - workflow.serviceApplyCommonParams(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.ElbManager, ctx.ParamManager), + workflow.serviceApplyCommonParams(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.ElbManager), + workflow.serviceDbParams(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.ParamManager), newConditionalExecutor(workflow.isEcsProvider(), newPipelineExecutor( workflow.serviceRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, environmentName), @@ -373,9 +374,33 @@ func (workflow *serviceWorkflow) serviceApplyBatchParams(service *common.Service } } +func (workflow *serviceWorkflow) serviceDbParams(namespace string, service *common.Service, + params map[string]string, environmentName string, stackWaiter common.StackWaiter, + paramGetter common.ParamGetter) Executor { + return func() error { + + dbStackName := common.CreateStackName(namespace, common.StackTypeDatabase, workflow.serviceName, environmentName) + dbStack := stackWaiter.AwaitFinalStatus(dbStackName) + if dbStack != nil { + params["DatabaseName"] = dbStack.Outputs["DatabaseName"] + params["DatabaseEndpointAddress"] = dbStack.Outputs["DatabaseEndpointAddress"] + params["DatabaseEndpointPort"] = dbStack.Outputs["DatabaseEndpointPort"] + params["DatabaseMasterUsername"] = dbStack.Outputs["DatabaseMasterUsername"] + + dbPass, err := paramGetter.GetParam(fmt.Sprintf("%s-%s", dbStackName, "DatabaseMasterPassword")) + if err != nil { + log.Warningf("Unable to get db password: %s", err) + } + params["DatabaseMasterPassword"] = dbPass + } + + return nil + } +} + func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, service *common.Service, params map[string]string, environmentName string, stackWaiter common.StackWaiter, - elbRuleLister common.ElbRuleLister, paramGetter common.ParamGetter) Executor { + elbRuleLister common.ElbRuleLister) Executor { return func() error { params["VpcId"] = fmt.Sprintf("%s-VpcId", workflow.envStack.Name) @@ -420,21 +445,6 @@ func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, serv common.NewMapElementIfNotZero(params, "TargetCPUUtilization", service.TargetCPUUtilization) - dbStackName := common.CreateStackName(namespace, common.StackTypeDatabase, workflow.serviceName, environmentName) - dbStack := stackWaiter.AwaitFinalStatus(dbStackName) - if dbStack != nil { - params["DatabaseName"] = dbStack.Outputs["DatabaseName"] - params["DatabaseEndpointAddress"] = dbStack.Outputs["DatabaseEndpointAddress"] - params["DatabaseEndpointPort"] = dbStack.Outputs["DatabaseEndpointPort"] - params["DatabaseMasterUsername"] = dbStack.Outputs["DatabaseMasterUsername"] - - dbPass, err := paramGetter.GetParam(fmt.Sprintf("%s-%s", dbStackName, "DatabaseMasterPassword")) - if err != nil { - log.Warningf("Unable to get db password: %s", err) - } - params["DatabaseMasterPassword"] = dbPass - } - params["Namespace"] = namespace params["EnvironmentName"] = environmentName params["ServiceName"] = workflow.serviceName diff --git a/workflows/service_deploy_test.go b/workflows/service_deploy_test.go index e16a67fe..eef2e0c2 100644 --- a/workflows/service_deploy_test.go +++ b/workflows/service_deploy_test.go @@ -51,7 +51,10 @@ func TestServiceApplyCommon_Create(t *testing.T) { workflow.serviceName = "myservice" workflow.envStack = &common.Stack{Name: "mu-environment-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.lbStack = &common.Stack{Name: "mu-loadbalancer-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} - err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister, paramManager)() + err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister)() + assert.Nil(err) + + err = workflow.serviceDbParams("mu", service, params, "dev", stackManager, paramManager)() assert.Nil(err) assert.Equal("mu-environment-dev-VpcId", params["VpcId"]) @@ -89,7 +92,10 @@ func TestServiceApplyCommon_Update(t *testing.T) { workflow.serviceName = "myservice" workflow.envStack = &common.Stack{Name: "mu-environment-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.lbStack = &common.Stack{Name: "mu-loadbalancer-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} - err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister, paramManager)() + err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister)() + assert.Nil(err) + + err = workflow.serviceDbParams("mu", service, params, "dev", stackManager, paramManager)() assert.Nil(err) assert.Equal("", params["ListenerRulePriority"]) @@ -124,7 +130,10 @@ func TestServiceApplyCommon_StaticPriority(t *testing.T) { workflow.envStack = &common.Stack{Name: "mu-environment-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.lbStack = &common.Stack{Name: "mu-loadbalancer-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.priority = 77 - err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister, paramManager)() + err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister)() + assert.Nil(err) + + err = workflow.serviceDbParams("mu", service, params, "dev", stackManager, paramManager)() assert.Nil(err) assert.Equal("77", params["PathListenerRulePriority"]) From cac17fb267d1fa079dbac27a79a2453680ba5f2c Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Wed, 9 Jan 2019 20:20:49 +0000 Subject: [PATCH 11/15] BatchTask IAM role update --- templates/assets/cloudformation/service-iam.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/assets/cloudformation/service-iam.yml b/templates/assets/cloudformation/service-iam.yml index b687c64c..92c79f89 100644 --- a/templates/assets/cloudformation/service-iam.yml +++ b/templates/assets/cloudformation/service-iam.yml @@ -122,6 +122,10 @@ Resources: - IsEksService - !GetAtt EksPodRole.Arn - !Ref AWS::NoValue + - Fn::If: + - IsBatchService + - !GetAtt BatchTaskRole.Arn + - !Ref AWS::NoValue Action: - kms:GenerateDataKey - kms:GenerateDataKeyWithoutPlaintext From bf54de375f4620b8386a54dd89566a485f998f79 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Tue, 15 Jan 2019 15:24:28 +0000 Subject: [PATCH 12/15] ecr image build pull auth --- common/docker.go | 9 +++++---- workflows/service_common.go | 15 +++++++++++++++ workflows/service_push.go | 4 ++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/common/docker.go b/common/docker.go index e8c454ea..48edab10 100644 --- a/common/docker.go +++ b/common/docker.go @@ -17,7 +17,7 @@ import ( // DockerImageBuilder for creating docker images type DockerImageBuilder interface { - ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, dockerOut io.Writer) error + ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerOut io.Writer) error } // DockerImagePusher for pushing docker images @@ -47,10 +47,11 @@ func newClientDockerManager() (DockerManager, error) { }, nil } -func (d *clientDockerManager) ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, dockerOut io.Writer) error { +func (d *clientDockerManager) ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerOut io.Writer) error { options := types.ImageBuildOptions{ - Tags: tags, - Labels: map[string]string{"SERVICE_NAME": serviceName}, + Tags: tags, + Labels: map[string]string{"SERVICE_NAME": serviceName}, + AuthConfigs: registryAuthConfig, } buildContext, err := createBuildContext(contextDir, relDockerfile) diff --git a/workflows/service_common.go b/workflows/service_common.go index ec4caea0..1753bc88 100644 --- a/workflows/service_common.go +++ b/workflows/service_common.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/docker/docker/api/types" "github.com/pkg/errors" "github.com/stelligent/mu/common" ) @@ -19,6 +20,7 @@ type serviceWorkflow struct { serviceTag string serviceImage string registryAuth string + registryAuthConfig map[string]types.AuthConfig priority int codeRevision string repoName string @@ -272,6 +274,19 @@ func (workflow *serviceWorkflow) serviceRegistryAuthenticator(authenticator comm authParts := strings.Split(string(data), ":") workflow.registryAuth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("{\"username\":\"%s\", \"password\":\"%s\"}", authParts[0], authParts[1]))) + + // ImageBuild pull auth + var authConfigs2 map[string]types.AuthConfig + authConfigs2 = make(map[string]types.AuthConfig) + + authConfigs2["ecr"] = types.AuthConfig{ + Username: authParts[0], + Password: authParts[1], + ServerAddress: fmt.Sprintf("https://%s", workflow.serviceImage), + } + + workflow.registryAuthConfig = authConfigs2 + return nil } } diff --git a/workflows/service_push.go b/workflows/service_push.go index b7d17429..11148ca5 100644 --- a/workflows/service_push.go +++ b/workflows/service_push.go @@ -22,8 +22,8 @@ func NewServicePusher(ctx *common.Context, tag string, provider string, kmsKey s newConditionalExecutor(workflow.isEcrProvider(), newPipelineExecutor( workflow.serviceRepoUpserter(ctx.Config.Namespace, &ctx.Config.Service, ctx.StackManager, ctx.StackManager), - workflow.serviceImageBuilder(ctx.DockerManager, &ctx.Config, dockerWriter), workflow.serviceRegistryAuthenticator(ctx.ClusterManager), + workflow.serviceImageBuilder(ctx.DockerManager, &ctx.Config, dockerWriter), workflow.serviceImagePusher(ctx.DockerManager, dockerWriter), ), newPipelineExecutor( @@ -36,7 +36,7 @@ func NewServicePusher(ctx *common.Context, tag string, provider string, kmsKey s func (workflow *serviceWorkflow) serviceImageBuilder(imageBuilder common.DockerImageBuilder, config *common.Config, dockerWriter io.Writer) Executor { return func() error { log.Noticef("Building service:'%s' as image:%s'", workflow.serviceName, workflow.serviceImage) - return imageBuilder.ImageBuild(config.Basedir, workflow.serviceName, config.Service.Dockerfile, []string{workflow.serviceImage}, dockerWriter) + return imageBuilder.ImageBuild(config.Basedir, workflow.serviceName, config.Service.Dockerfile, []string{workflow.serviceImage}, workflow.registryAuthConfig, dockerWriter) } } From e2a696258881ae28983a3f6619444444094997ea Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Tue, 15 Jan 2019 15:27:24 +0000 Subject: [PATCH 13/15] gitignore bak --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4106b75e..d87d9ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ vendor/ homebrew-tap/ *-packr.go CHANGELOG.md +*.bak \ No newline at end of file From 0b59f53b0cac669e71785fda3f24fa19d933c0d7 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Tue, 15 Jan 2019 16:33:06 +0000 Subject: [PATCH 14/15] image build ecr auth fix --- workflows/service_common.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/service_common.go b/workflows/service_common.go index 1753bc88..4d67fd6f 100644 --- a/workflows/service_common.go +++ b/workflows/service_common.go @@ -279,10 +279,10 @@ func (workflow *serviceWorkflow) serviceRegistryAuthenticator(authenticator comm var authConfigs2 map[string]types.AuthConfig authConfigs2 = make(map[string]types.AuthConfig) - authConfigs2["ecr"] = types.AuthConfig{ + authConfigs2[strings.Split(workflow.serviceImage, ":")[0]] = types.AuthConfig{ Username: authParts[0], Password: authParts[1], - ServerAddress: fmt.Sprintf("https://%s", workflow.serviceImage), + ServerAddress: fmt.Sprintf("https://%s", strings.Split(workflow.serviceImage, ":")[0]), } workflow.registryAuthConfig = authConfigs2 From b6104ece0569dd4bfcb3ec69d934ca218fdc73e0 Mon Sep 17 00:00:00 2001 From: AndreyMarchuk Date: Tue, 12 Feb 2019 05:30:39 +0000 Subject: [PATCH 15/15] test fix --- workflows/service_push_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflows/service_push_test.go b/workflows/service_push_test.go index 1620d9a2..e18c18c1 100644 --- a/workflows/service_push_test.go +++ b/workflows/service_push_test.go @@ -1,6 +1,7 @@ package workflows import ( + "github.com/docker/docker/api/types" "github.com/stelligent/mu/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -21,7 +22,7 @@ type mockServiceBuilder struct { common.DockerImageBuilder } -func (m *mockServiceBuilder) ImageBuild(basedir string, serviceName string, dockerfile string, tags []string, dockerWriter io.Writer) error { +func (m *mockServiceBuilder) ImageBuild(basedir string, serviceName string, dockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerWriter io.Writer) error { args := m.Called() return args.Error(0) }