diff --git a/.gitignore b/.gitignore index 4106b75e..d87d9ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ vendor/ homebrew-tap/ *-packr.go CHANGELOG.md +*.bak \ No newline at end of file diff --git a/common/docker.go b/common/docker.go index e8c454ea..48edab10 100644 --- a/common/docker.go +++ b/common/docker.go @@ -17,7 +17,7 @@ import ( // DockerImageBuilder for creating docker images type DockerImageBuilder interface { - ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, dockerOut io.Writer) error + ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerOut io.Writer) error } // DockerImagePusher for pushing docker images @@ -47,10 +47,11 @@ func newClientDockerManager() (DockerManager, error) { }, nil } -func (d *clientDockerManager) ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, dockerOut io.Writer) error { +func (d *clientDockerManager) ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerOut io.Writer) error { options := types.ImageBuildOptions{ - Tags: tags, - Labels: map[string]string{"SERVICE_NAME": serviceName}, + Tags: tags, + Labels: map[string]string{"SERVICE_NAME": serviceName}, + AuthConfigs: registryAuthConfig, } buildContext, err := createBuildContext(contextDir, relDockerfile) diff --git a/common/types.go b/common/types.go index 945af60d..9a7bb913 100644 --- a/common/types.go +++ b/common/types.go @@ -169,17 +169,26 @@ type Service struct { HostPatterns []string `yaml:"hostPatterns,omitempty"` Priority int `yaml:"priority,omitempty" validate:"max=50000"` Pipeline Pipeline `yaml:"pipeline,omitempty"` + ProviderOverride string `yaml:"provider,omitempty"` Database Database `yaml:"database,omitempty"` Schedule []Schedule `yaml:"schedules,omitempty"` TargetCPUUtilization int `yaml:"targetCPUUtilization,omitempty" validate:"max=100"` DiscoveryTTL string `yaml:"discoveryTTL,omitempty"` - Roles struct { + + // Batch + Parameters map[string]interface{} `yaml:"parameters,omitempty"` + Command []string `yaml:"command,omitempty"` + Timeout int `yaml:"timeout,omitempty"` + RetryAttempts int `yaml:"retryAttempts,omitempty"` + + Roles struct { Ec2Instance string `yaml:"ec2Instance,omitempty" validate:"validateRoleARN"` CodeDeploy string `yaml:"codeDeploy,omitempty" validate:"validateRoleARN"` EcsEvents string `yaml:"ecsEvents,omitempty" validate:"validateRoleARN"` EcsService string `yaml:"ecsService,omitempty" validate:"validateRoleARN"` EcsTask string `yaml:"ecsTask,omitempty" validate:"validateRoleARN"` ApplicationAutoScaling string `yaml:"applicationAutoScaling,omitempty" validate:"validateRoleARN"` + BatchJobRole string `yaml:"batchJobRole,omitempty" validate:"validateRoleARN"` } `yaml:"roles,omitempty"` } @@ -395,6 +404,7 @@ const ( TemplateEnvECS = "cloudformation/env-ecs.yml" TemplateEnvEKS = "cloudformation/env-eks.yml" TemplateEnvEKSBootstrap = "cloudformation/env-eks-bootstrap.yml" + TemplateEnvBatch = "cloudformation/env-batch.yml" TemplateEnvIAM = "cloudformation/env-iam.yml" TemplatePipelineIAM = "cloudformation/pipeline-iam.yml" TemplatePipeline = "cloudformation/pipeline.yml" @@ -402,6 +412,7 @@ const ( TemplateSchedule = "cloudformation/schedule.yml" TemplateServiceEC2 = "cloudformation/service-ec2.yml" TemplateServiceECS = "cloudformation/service-ecs.yml" + TemplateServiceBatch = "cloudformation/service-batch.yml" TemplateServiceIAM = "cloudformation/service-iam.yml" TemplateVPCTarget = "cloudformation/vpc-target.yml" TemplateVPC = "cloudformation/vpc.yml" @@ -418,8 +429,8 @@ type DeploymentStrategy string // List of supported deployment strategies const ( BlueGreenDeploymentStrategy DeploymentStrategy = "blue_green" - RollingDeploymentStrategy DeploymentStrategy = "rolling" - ReplaceDeploymentStrategy DeploymentStrategy = "replace" + RollingDeploymentStrategy = "rolling" + ReplaceDeploymentStrategy = "replace" ) // EnvProvider describes supported environment strategies @@ -432,6 +443,7 @@ const ( EnvProviderEc2 = "ec2" EnvProviderEks = "eks" EnvProviderEksFargate = "eks-fargate" + EnvProviderBatch = "batch" ) // InstanceTenancy describes supported tenancy options for EC2 diff --git a/examples/batch/Dockerfile b/examples/batch/Dockerfile new file mode 100644 index 00000000..08a65817 --- /dev/null +++ b/examples/batch/Dockerfile @@ -0,0 +1,8 @@ +FROM amazonlinux:latest + +RUN yum -y install unzip aws-cli +ADD batch-job.sh /usr/local/bin/batch-job.sh +WORKDIR /tmp +USER nobody + +ENTRYPOINT ["/usr/local/bin/batch-job.sh"] \ No newline at end of file diff --git a/examples/batch/README.md b/examples/batch/README.md new file mode 100644 index 00000000..eed22f76 --- /dev/null +++ b/examples/batch/README.md @@ -0,0 +1,5 @@ +# Examples +These examples are not intended to be run directly. Rather, they serve as a reference that can be consulted when creating your own `mu.yml` files. + +For detailed steps to create your own project, check out the [quickstart](https://github.com/stelligent/mu/wiki/Quickstart#steps). + diff --git a/examples/batch/batch-job.sh b/examples/batch/batch-job.sh new file mode 100644 index 00000000..8a3b271f --- /dev/null +++ b/examples/batch/batch-job.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +date +# this will expose parameters passed from mu.yml command section +echo "Args: $@" +env +echo "This is my simple test job!." +echo "jobId: $AWS_BATCH_JOB_ID" +echo "jobQueue: $AWS_BATCH_JQ_NAME" +echo "computeEnvironment: $AWS_BATCH_CE_NAME" +sleep $1 +date +echo "bye bye!!" \ No newline at end of file diff --git a/examples/batch/buildspec.yml b/examples/batch/buildspec.yml new file mode 100644 index 00000000..2b6a64cb --- /dev/null +++ b/examples/batch/buildspec.yml @@ -0,0 +1,12 @@ +version: 0.1 + +phases: + build: + commands: + - echo "replace me with real build commands..." + +artifacts: + files: + - batch-job.sh + - mu.yml + - Dockerfile diff --git a/examples/batch/mu.yml b/examples/batch/mu.yml new file mode 100644 index 00000000..f2ab8a29 --- /dev/null +++ b/examples/batch/mu.yml @@ -0,0 +1,49 @@ +## See https://github.com/stelligent/mu/wiki/Services +service: + name: batch-sample + + # deployed as AWS Batch job + provider: batch + + # The relative path to the Dockerfile to build images (default: Dockerfile) + dockerfile: Dockerfile + + # The number of VCPUs to allocate to job (default: 2) + # Each vCPU is equivalent to 1,024 CPU shares. + cpu: 1 + + # The hard limit (in MiB) of memory to present to the container. If your container attempts to exceed the memory specified here, the container is killed. + # Read this: https://docs.aws.amazon.com/batch/latest/userguide/memory-management.html + memory: 128 + + # The time duration in seconds (measured from the job attempt's startedAt timestamp) + # after which AWS Batch terminates your jobs if they have not finished. + # If a job is terminated due to a timeout, it is not retried. + timeout: 0 + retryAttempts: 1 + + # our docker file already has a defined entry point script, these params + # will be appended to it + command: + - '--some-flag' + - Ref::param1 + + # Default parameters + # Parameter values replace the placeholders in the command, i.e. Ref::paramname. + # Usually final parameter values are specified on job submit + parameters: + param1: "Sample text passed to the batch job" + + ### Environment variables exposed to the containers + environment: + + #### Define this for each environment if you want APP_DEBUG=true + APP_DEBUG: + dev: 'true' + prod: 'false' + + APP_ENV: + dev: dev + prod: production + + \ No newline at end of file diff --git a/provider/aws/cloudformation.go b/provider/aws/cloudformation.go index 657892ce..59ed67d0 100644 --- a/provider/aws/cloudformation.go +++ b/provider/aws/cloudformation.go @@ -65,14 +65,22 @@ func newStackManager(sess *session.Session, extensionsManager common.ExtensionsM } -func buildStackParameters(parameters map[string]string) []*cloudformation.Parameter { +func buildStackParameters(parameters map[string]string, stack *common.Stack) []*cloudformation.Parameter { + stackParameters := make([]*cloudformation.Parameter, 0, len(parameters)) for key, value := range parameters { + // Stack exists, check if parameter exists. Needed to avoid: + // ValidationError: Invalid input for parameter key DatabaseName. Cannot specify usePreviousValue as true for a parameter key not in the previous template + usePrevVal := true + if stack != nil && stack.Status != "" { + _, usePrevVal = stack.Parameters[key] + } + stackParameters = append(stackParameters, &cloudformation.Parameter{ ParameterKey: aws.String(key), ParameterValue: aws.String(value), - UsePreviousValue: aws.Bool(value == ""), + UsePreviousValue: aws.Bool(value == "" && usePrevVal), }) } return stackParameters @@ -333,7 +341,7 @@ func (cfnMgr *cloudformationStackManager) UpsertStack(stackName string, template if err != nil { return err } - stackParameters := buildStackParameters(parameters) + stackParameters := buildStackParameters(parameters, stack) // stack tags tags, err = cfnMgr.extensionsManager.DecorateStackTags(stackName, tags) diff --git a/provider/aws/cloudformation_test.go b/provider/aws/cloudformation_test.go index 52161762..75f9a628 100644 --- a/provider/aws/cloudformation_test.go +++ b/provider/aws/cloudformation_test.go @@ -342,17 +342,39 @@ func TestBuildParameters(t *testing.T) { paramMap := make(map[string]string) - parameters := buildStackParameters(paramMap) + parameters := buildStackParameters(paramMap, nil) assert.Equal(0, len(parameters)) paramMap["p1"] = "value 1" paramMap["p2"] = "value 2" - parameters = buildStackParameters(paramMap) + parameters = buildStackParameters(paramMap, nil) assert.Equal(2, len(parameters)) assert.Contains(*parameters[0].ParameterKey, "p") assert.Contains(*parameters[0].ParameterValue, "value") assert.Contains(*parameters[1].ParameterKey, "p") assert.Contains(*parameters[1].ParameterValue, "value") + + stackParameters := make([]*cloudformation.Parameter, 0, len(parameters)) + stackParameters = append(stackParameters, + &cloudformation.Parameter{ + ParameterKey: aws.String("ExistingParam"), + ParameterValue: aws.String("ExistingValue"), + }) + + stackDetails := cloudformation.Stack{ + StackName: aws.String("mu-environment-dev"), + CreationTime: aws.Time(time.Now()), + Tags: []*cloudformation.Tag{}, + StackStatus: aws.String("CREATE_COMPLETE"), + Parameters: stackParameters, + } + + stack := buildStack(&stackDetails) + paramMap["ExistingParam"] = "" // should have UsePreviousValue==true + paramMap["NewParam"] = "" // should have UsePreviousValue==false + parameters = buildStackParameters(paramMap, stack) + assert.Equal(*parameters[2].UsePreviousValue, true) + assert.Equal(*parameters[3].UsePreviousValue, false) } func TestTagParameters(t *testing.T) { diff --git a/provider/aws/roleset.go b/provider/aws/roleset.go index 3b147896..f0d6c853 100644 --- a/provider/aws/roleset.go +++ b/provider/aws/roleset.go @@ -68,6 +68,7 @@ func (rolesetMgr *iamRolesetManager) GetServiceRoleset(environmentName string, s overrideRole(roleset, "EcsServiceRoleArn", rolesetMgr.context.Config.Service.Roles.EcsService) overrideRole(roleset, "EcsTaskRoleArn", rolesetMgr.context.Config.Service.Roles.EcsTask) overrideRole(roleset, "ApplicationAutoScalingRoleArn", rolesetMgr.context.Config.Service.Roles.ApplicationAutoScaling) + overrideRole(roleset, "BatchJobRoleArn", rolesetMgr.context.Config.Service.Roles.BatchJobRole) return roleset, nil } @@ -185,6 +186,11 @@ func (rolesetMgr *iamRolesetManager) GetEnvironmentProvider(environmentName stri break } } + // allow to override provider (batch job definition can be deployed without environment) + if rolesetMgr.context.Config.Service.ProviderOverride != "" { + // todo: validate values + envProvider = rolesetMgr.context.Config.Service.ProviderOverride + } if envProvider == "" { log.Debugf("unable to find environment named '%s' in configuration...checking for existing stack", environmentName) envStackName := common.CreateStackName(rolesetMgr.context.Config.Namespace, common.StackTypeEnv, environmentName) diff --git a/templates/assets/cloudformation/common-iam.yml b/templates/assets/cloudformation/common-iam.yml index 4161300b..1e89c639 100644 --- a/templates/assets/cloudformation/common-iam.yml +++ b/templates/assets/cloudformation/common-iam.yml @@ -384,6 +384,21 @@ Resources: StringLike: iam:AWSServiceName: rds.amazonaws.com Effect: Allow + - PolicyName: aws-batch-jobs + PolicyDocument: + Version: '2012-10-17' + Statement: + - Action: + - batch:RegisterJobDefinition + - batch:DeregisterJobDefinition + - batch:DescribeJobDefinitions + Resource: '*' + Effect: Allow + - Action: + - iam:PassRole + Resource: + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${Namespace}-service-*-batchjob-${AWS::Region} + Effect: Allow Outputs: CloudFormationRoleArn: Description: Role assummed by CloudFormation diff --git a/templates/assets/cloudformation/service-batch.yml b/templates/assets/cloudformation/service-batch.yml new file mode 100644 index 00000000..af2613c6 --- /dev/null +++ b/templates/assets/cloudformation/service-batch.yml @@ -0,0 +1,98 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: MU AWS Batch job definition +Parameters: + Namespace: + Type: String + Description: Namespace for stack prefixes + EnvironmentName: + Type: String + Description: Name of environment used for resource namespace + ServiceName: + Type: String + Description: Name of service used for resource namespace + ServiceVCpu: + Type: String + Description: VCPUs to reserve for container + Default: '2' + ServiceMemory: + Type: String + Description: Memory to allocate to container (in MiB) + Default: '512' + ImageUrl: + Type: String + Description: Docker Image URL (or name, like 'amazonlinux') + BatchJobRoleArn: + Type: String + Description: IAM Role assumed by AWS Batch job (ecs task) + RetryAttempts: + Type: String + Description: Number of retry attempts + Default: '' + Timeout: + Type: String + Description: Job timeout in seconds. Job is terminated after timing out + Default: '' + VpcId: + Type: String + Description: Name of the value to import for the VpcId (not used for Batch) + +Conditions: + HasRetryAttempts: + "Fn::Not": + - "Fn::Equals": + - !Ref RetryAttempts + - '' + HasTimeout: + "Fn::Not": + - "Fn::Equals": + - !Ref Timeout + - '' + +Resources: + + BatchJob: + Type: 'AWS::Batch::JobDefinition' + Properties: + JobDefinitionName: !Sub ${Namespace}-batch-${ServiceName}-${EnvironmentName} + Type: container + Parameters: + {{with .Parameters}} + {{range $key, $val := .}} + {{$key}}: !Sub {{$val}} + {{end}} + {{end}} + ContainerProperties: + Image: !Ref ImageUrl + Vcpus: !Ref ServiceVCpu + Memory: !Ref ServiceMemory + JobRoleArn: !Ref BatchJobRoleArn + Command: + {{with .Command}} + {{range $key, $val := .}} + - {{$val}} + {{end}} + {{end}} + Environment: + {{with .Environment}} + {{range $key, $val := .}} + - Name: {{$key}} + Value: !Sub {{$val}} + {{end}} + {{end}} + RetryStrategy: + Attempts: + Fn::If: + - HasRetryAttempts + - !Ref RetryAttempts + - !Ref AWS::NoValue + Timeout: + AttemptDurationSeconds: + Fn::If: + - HasTimeout + - !Ref Timeout + - !Ref AWS::NoValue + + +Outputs: + BatchJobDefinitionArn: + Value: !Ref BatchJob \ No newline at end of file diff --git a/templates/assets/cloudformation/service-iam.yml b/templates/assets/cloudformation/service-iam.yml index 935e2f82..92c79f89 100644 --- a/templates/assets/cloudformation/service-iam.yml +++ b/templates/assets/cloudformation/service-iam.yml @@ -21,6 +21,7 @@ Parameters: - "ec2" - "eks" - "eks-fargate" + - "batch" CodeDeployBucket: Type: String Description: Name of bucket to use for the CodeDeployBucket artifacts @@ -49,6 +50,10 @@ Conditions: - "Fn::Equals": - !Ref Provider - 'eks-fargate' + IsBatchService: + "Fn::Equals": + - !Ref Provider + - 'batch' HasDatabase: "Fn::Not": - "Fn::Equals": @@ -117,6 +122,10 @@ Resources: - IsEksService - !GetAtt EksPodRole.Arn - !Ref AWS::NoValue + - Fn::If: + - IsBatchService + - !GetAtt BatchTaskRole.Arn + - !Ref AWS::NoValue Action: - kms:GenerateDataKey - kms:GenerateDataKeyWithoutPlaintext @@ -405,6 +414,47 @@ Resources: - logs:DescribeLogGroups - logs:DescribeLogStreams Resource: '*' + + BatchTaskRole: + Type: AWS::IAM::Role + Condition: IsBatchService + Properties: + RoleName: !Sub ${Namespace}-service-${ServiceName}-${EnvironmentName}-batchjob-${AWS::Region} + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: + - ecs-tasks.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: ecs-task + PolicyDocument: + Statement: + - Effect: Allow + Action: + - ecs:DescribeTasks + Resource: "*" + Condition: + ArnEquals: + "ecs:cluster": !Sub arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:cluster/${Namespace}-environment-${EnvironmentName} + - PolicyName: task-execution + PolicyDocument: + Statement: + - Effect: Allow + Action: + - ecr:GetAuthorizationToken + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - logs:DescribeLogGroups + - logs:DescribeLogStreams + Resource: '*' Outputs: DatabaseKeyArn: @@ -456,3 +506,10 @@ Outputs: - IsEcsService - !GetAtt ApplicationAutoScalingRole.Arn - '' + BatchJobRoleArn: + Description: Role assummed by AWS Batch job container + Value: + Fn::If: + - IsBatchService + - !GetAtt BatchTaskRole.Arn + - '' \ No newline at end of file diff --git a/templates/template_test.go b/templates/template_test.go index 46346f77..37a36aeb 100644 --- a/templates/template_test.go +++ b/templates/template_test.go @@ -68,6 +68,19 @@ func TestNewTemplate_assets(t *testing.T) { templateData = tdMap } + if templateName == "cloudformation/service-batch.yml" { + params := make(map[string]string) + params["testparam"] = "test" + env := make(map[string]string) + env["ENVVAR"] = "test" + command := []string{"test"} + tdMap := make(map[string]interface{}) + tdMap["Parameters"] = params + tdMap["Command"] = command + tdMap["Environment"] = env + templateData = tdMap + } + templateBody, err := GetAsset(templateName, ExecuteTemplate(templateData)) if err != nil { diff --git a/workflows/environment_common.go b/workflows/environment_common.go index 38938f5d..c2876f80 100644 --- a/workflows/environment_common.go +++ b/workflows/environment_common.go @@ -57,6 +57,12 @@ func (workflow *environmentWorkflow) isEc2Provider() Conditional { } } +func (workflow *environmentWorkflow) isBatchProvider() Conditional { + return func() bool { + return strings.EqualFold(string(workflow.environment.Provider), string(common.EnvProviderBatch)) + } +} + func (workflow *environmentWorkflow) connectKubernetes(muNamespace string, provider common.KubernetesResourceManagerProvider) Executor { return func() error { clusterName := common.CreateStackName(muNamespace, common.StackTypeEnv, workflow.environment.Name) diff --git a/workflows/service_common.go b/workflows/service_common.go index 505a86f3..4d67fd6f 100644 --- a/workflows/service_common.go +++ b/workflows/service_common.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/docker/docker/api/types" "github.com/pkg/errors" "github.com/stelligent/mu/common" ) @@ -13,11 +14,13 @@ import ( type serviceWorkflow struct { envStack *common.Stack lbStack *common.Stack + lbDisabled bool artifactProvider common.ArtifactProvider serviceName string serviceTag string serviceImage string registryAuth string + registryAuthConfig map[string]types.AuthConfig priority int codeRevision string repoName string @@ -29,6 +32,7 @@ type serviceWorkflow struct { microserviceTaskDefinitionArn string ecsEventsRoleArn string kubernetesResourceManager common.KubernetesResourceManager + providerOverride string } // Find a service in config, by name and set the reference @@ -54,6 +58,11 @@ func (workflow *serviceWorkflow) serviceLoader(ctx *common.Context, tag string, workflow.repoName = ctx.Config.Repo.Slug workflow.priority = ctx.Config.Service.Priority + // allow to override provider (i.e. deploy batch job definition) + // todo: validate values + workflow.providerOverride = ctx.Config.Service.ProviderOverride + workflow.lbDisabled = false + if provider == "" { dockerfile := ctx.Config.Service.Dockerfile if dockerfile == "" { @@ -115,6 +124,12 @@ func (workflow *serviceWorkflow) isEc2Provider() Conditional { } } +func (workflow *serviceWorkflow) isBatchProvider() Conditional { + return func() bool { + return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderBatch)) + } +} + func (workflow *serviceWorkflow) serviceInput(ctx *common.Context, serviceName string) Executor { return func() error { // Repo Name @@ -259,6 +274,19 @@ func (workflow *serviceWorkflow) serviceRegistryAuthenticator(authenticator comm authParts := strings.Split(string(data), ":") workflow.registryAuth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("{\"username\":\"%s\", \"password\":\"%s\"}", authParts[0], authParts[1]))) + + // ImageBuild pull auth + var authConfigs2 map[string]types.AuthConfig + authConfigs2 = make(map[string]types.AuthConfig) + + authConfigs2[strings.Split(workflow.serviceImage, ":")[0]] = types.AuthConfig{ + Username: authParts[0], + Password: authParts[1], + ServerAddress: fmt.Sprintf("https://%s", strings.Split(workflow.serviceImage, ":")[0]), + } + + workflow.registryAuthConfig = authConfigs2 + return nil } } diff --git a/workflows/service_deploy.go b/workflows/service_deploy.go index fc3d44fa..aaa4098f 100644 --- a/workflows/service_deploy.go +++ b/workflows/service_deploy.go @@ -22,7 +22,8 @@ func NewServiceDeployer(ctx *common.Context, environmentName string, tag string) return newPipelineExecutor( workflow.serviceLoader(ctx, tag, ""), workflow.serviceEnvironmentLoader(ctx.Config.Namespace, environmentName, ctx.StackManager), - workflow.serviceApplyCommonParams(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.ElbManager, ctx.ParamManager), + workflow.serviceApplyCommonParams(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.ElbManager), + workflow.serviceDbParams(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.ParamManager), newConditionalExecutor(workflow.isEcsProvider(), newPipelineExecutor( workflow.serviceRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, environmentName), @@ -49,6 +50,13 @@ func NewServiceDeployer(ctx *common.Context, environmentName string, tag string) workflow.serviceEksDeployer(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName), // TODO - placeholder for doing serviceCreateSchedules for EKS, leaving out-of-scope ), nil), + newConditionalExecutor(workflow.isBatchProvider(), + newPipelineExecutor( + workflow.serviceRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, environmentName), + workflow.serviceRepoUpserter(ctx.Config.Namespace, &ctx.Config.Service, ctx.StackManager, ctx.StackManager), + workflow.serviceApplyBatchParams(&ctx.Config.Service, stackParams, ctx.RolesetManager), + workflow.serviceBatchDeployer(ctx.Config.Namespace, &ctx.Config.Service, stackParams, environmentName, ctx.StackManager, ctx.StackManager), + ), nil), ) } @@ -90,17 +98,33 @@ func checkPriorityNotInUse(elbRuleLister common.ElbRuleLister, listenerArn strin func (workflow *serviceWorkflow) serviceEnvironmentLoader(namespace string, environmentName string, stackWaiter common.StackWaiter) Executor { return func() error { + lbStackName := common.CreateStackName(namespace, common.StackTypeLoadBalancer, environmentName) workflow.lbStack = stackWaiter.AwaitFinalStatus(lbStackName) envStackName := common.CreateStackName(namespace, common.StackTypeEnv, environmentName) workflow.envStack = stackWaiter.AwaitFinalStatus(envStackName) + // batch job definition can be deployed without a real environment, creating fake env (if needed) and overriding provider to 'batch' + if workflow.providerOverride == common.EnvProviderBatch { + workflow.createFakeBatchEnvironment(envStackName, namespace, environmentName) + } + if workflow.envStack == nil { return fmt.Errorf("Unable to find stack '%s' for environment '%s'", envStackName, environmentName) } - if workflow.isEcsProvider()() { + // // disable LB for Batch deployments + if workflow.isBatchProvider()() { + workflow.lbDisabled = true + } + + // this check is not needed (?) + // if workflow.lbStack == nil && workflow.lbDisabled != true { + // return fmt.Errorf("Unable to find loadbalancer '%s' for environment '%s'", lbStackName, environmentName) + // } + + if workflow.isEcsProvider()() || workflow.isBatchProvider()() { workflow.artifactProvider = common.ArtifactProviderEcr } else { workflow.artifactProvider = common.ArtifactProviderS3 @@ -110,6 +134,18 @@ func (workflow *serviceWorkflow) serviceEnvironmentLoader(namespace string, envi } } +func (workflow *serviceWorkflow) createFakeBatchEnvironment(envStackName string, namespace string, environmentName string) { + if workflow.envStack == nil { + outputs := make(map[string]string) + outputs["ElbHttpListenerArn"] = "foo" + outputs["ElbHttpsListenerArn"] = "foo" + tags := make(map[string]string) + workflow.envStack = &common.Stack{Name: envStackName, Status: common.StackStatusCreateComplete, Outputs: outputs, Tags: tags} + } + workflow.envStack.Tags["provider"] = common.EnvProviderBatch + workflow.envStack.Tags["environment"] = environmentName +} + func (workflow *serviceWorkflow) serviceRolesetUpserter(rolesetUpserter common.RolesetUpserter, rolesetGetter common.RolesetGetter, environmentName string) Executor { return func() error { err := rolesetUpserter.UpsertCommonRoleset() @@ -307,29 +343,41 @@ func (workflow *serviceWorkflow) serviceApplyEc2Params(params map[string]string, } } -func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, service *common.Service, - params map[string]string, environmentName string, stackWaiter common.StackWaiter, - elbRuleLister common.ElbRuleLister, paramGetter common.ParamGetter) Executor { +func (workflow *serviceWorkflow) serviceApplyBatchParams(service *common.Service, params map[string]string, rolesetGetter common.RolesetGetter) Executor { return func() error { - params["VpcId"] = fmt.Sprintf("%s-VpcId", workflow.envStack.Name) + // ... + // Environment, Parameters, Command are handled by go templating engine within service-batch.yml + common.NewMapElementIfNotZero(params, "Timeout", service.Timeout) + common.NewMapElementIfNotZero(params, "RetryAttempts", service.RetryAttempts) + params["ImageUrl"] = workflow.serviceImage - nextAvailablePriority := 0 - if workflow.lbStack != nil { - if workflow.lbStack.Outputs["ElbHttpListenerArn"] != "" { - params["ElbHttpListenerArn"] = fmt.Sprintf("%s-ElbHttpListenerArn", workflow.lbStack.Name) - if workflow.priority < 1 { - nextAvailablePriority = 1 + getMaxPriority(elbRuleLister, workflow.lbStack.Outputs["ElbHttpListenerArn"]) - } - } - if workflow.lbStack.Outputs["ElbHttpsListenerArn"] != "" { - params["ElbHttpsListenerArn"] = fmt.Sprintf("%s-ElbHttpsListenerArn", workflow.lbStack.Name) - if workflow.priority < 1 && nextAvailablePriority == 0 { - nextAvailablePriority = 1 + getMaxPriority(elbRuleLister, workflow.lbStack.Outputs["ElbHttpsListenerArn"]) - } - } + cpu := common.CPUMemorySupport[0] + if service.CPU != 0 { + params["ServiceVCpu"] = strconv.Itoa(service.CPU) + cpu = matchRequestedCPU(service.CPU, cpu) } - common.NewMapElementIfNotZero(params, "TargetCPUUtilization", service.TargetCPUUtilization) + memory := cpu.Memory[0] + if service.Memory != 0 { + params["ServiceMemory"] = strconv.Itoa(service.Memory) + memory = matchRequestedMemory(service.Memory, cpu, memory) + } + + serviceRoleset, err := rolesetGetter.GetServiceRoleset(workflow.envStack.Tags["environment"], workflow.serviceName) + if err != nil { + return err + } + + params["BatchJobRoleArn"] = serviceRoleset["BatchJobRoleArn"] + + return nil + } +} + +func (workflow *serviceWorkflow) serviceDbParams(namespace string, service *common.Service, + params map[string]string, environmentName string, stackWaiter common.StackWaiter, + paramGetter common.ParamGetter) Executor { + return func() error { dbStackName := common.CreateStackName(namespace, common.StackTypeDatabase, workflow.serviceName, environmentName) dbStack := stackWaiter.AwaitFinalStatus(dbStackName) @@ -346,33 +394,62 @@ func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, serv params["DatabaseMasterPassword"] = dbPass } + return nil + } +} + +func (workflow *serviceWorkflow) serviceApplyCommonParams(namespace string, service *common.Service, + params map[string]string, environmentName string, stackWaiter common.StackWaiter, + elbRuleLister common.ElbRuleLister) Executor { + return func() error { + params["VpcId"] = fmt.Sprintf("%s-VpcId", workflow.envStack.Name) + svcStackName := common.CreateStackName(namespace, common.StackTypeService, workflow.serviceName, environmentName) svcStack := stackWaiter.AwaitFinalStatus(svcStackName) - if workflow.priority > 0 { - // make sure manually specified priority is not already in use - err := checkPriorityNotInUse(elbRuleLister, workflow.lbStack.Outputs["ElbHttpListenerArn"], []int{workflow.priority, workflow.priority + 1}) - if err != nil { - return err + nextAvailablePriority := 0 + if workflow.lbStack != nil && workflow.lbDisabled != true { + if workflow.lbStack.Outputs["ElbHttpListenerArn"] != "" { + params["ElbHttpListenerArn"] = fmt.Sprintf("%s-ElbHttpListenerArn", workflow.lbStack.Name) + if workflow.priority < 1 { + nextAvailablePriority = 1 + getMaxPriority(elbRuleLister, workflow.lbStack.Outputs["ElbHttpListenerArn"]) + } + } + if workflow.lbStack.Outputs["ElbHttpsListenerArn"] != "" { + params["ElbHttpsListenerArn"] = fmt.Sprintf("%s-ElbHttpsListenerArn", workflow.lbStack.Name) + if workflow.priority < 1 && nextAvailablePriority == 0 { + nextAvailablePriority = 1 + getMaxPriority(elbRuleLister, workflow.lbStack.Outputs["ElbHttpsListenerArn"]) + } } - params["PathListenerRulePriority"] = strconv.Itoa(workflow.priority) - params["HostListenerRulePriority"] = strconv.Itoa(workflow.priority + 1) - } else if svcStack != nil && svcStack.Status != "ROLLBACK_COMPLETE" { - // no value in config, and this is an update...use prior value - params["PathListenerRulePriority"] = "" - params["HostListenerRulePriority"] = "" - } else { - // no value in config, and this is a create...use next available - params["PathListenerRulePriority"] = strconv.Itoa(nextAvailablePriority) - params["HostListenerRulePriority"] = strconv.Itoa(nextAvailablePriority + 1) + // assign priority + if workflow.priority > 0 { + // make sure manually specified priority is not already in use + err := checkPriorityNotInUse(elbRuleLister, workflow.lbStack.Outputs["ElbHttpListenerArn"], []int{workflow.priority, workflow.priority + 1}) + if err != nil { + return err + } + + params["PathListenerRulePriority"] = strconv.Itoa(workflow.priority) + params["HostListenerRulePriority"] = strconv.Itoa(workflow.priority + 1) + } else if svcStack != nil && svcStack.Status != "ROLLBACK_COMPLETE" { + // no value in config, and this is an update...use prior value + params["PathListenerRulePriority"] = "" + params["HostListenerRulePriority"] = "" + } else { + // no value in config, and this is a create...use next available + params["PathListenerRulePriority"] = strconv.Itoa(nextAvailablePriority) + params["HostListenerRulePriority"] = strconv.Itoa(nextAvailablePriority + 1) + } } + common.NewMapElementIfNotZero(params, "TargetCPUUtilization", service.TargetCPUUtilization) + params["Namespace"] = namespace params["EnvironmentName"] = environmentName params["ServiceName"] = workflow.serviceName common.NewMapElementIfNotZero(params, "ServicePort", service.Port) - common.NewMapElementIfNotEmpty(params, "ServiceProtocol", string(service.Protocol)) + common.NewMapElementIfNotEmpty(params, "ServiceProtocol", strings.ToUpper(string(service.Protocol))) common.NewMapElementIfNotEmpty(params, "ServiceHealthEndpoint", service.HealthEndpoint) common.NewMapElementIfNotZero(params, "ServiceDesiredCount", service.DesiredCount) common.NewMapElementIfNotZero(params, "ServiceMinSize", service.MinSize) @@ -532,6 +609,41 @@ func (workflow *serviceWorkflow) serviceEksDeployer(namespace string, service *c } } +func (workflow *serviceWorkflow) serviceBatchDeployer(namespace string, service *common.Service, stackParams map[string]string, environmentName string, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor { + return func() error { + log.Noticef("Deploying batch service '%s' to '%s' from '%s'", workflow.serviceName, environmentName, workflow.serviceImage) + + svcStackName := common.CreateStackName(namespace, common.StackTypeService, workflow.serviceName, environmentName) + + resolveServiceEnvironment(service, environmentName) + + tags := createTagMap(&ServiceTags{ + Service: workflow.serviceName, + Environment: environmentName, + Type: common.StackTypeService, + Provider: workflow.envStack.Outputs["provider"], + Revision: workflow.codeRevision, + Repo: workflow.repoName, + }) + + err := stackUpserter.UpsertStack(svcStackName, common.TemplateServiceBatch, service, stackParams, tags, "", workflow.cloudFormationRoleArn) + if err != nil { + return err + } + log.Debugf("Waiting for stack '%s' to complete", svcStackName) + stack := stackWaiter.AwaitFinalStatus(svcStackName) + if stack == nil { + return fmt.Errorf("Unable to create stack %s", svcStackName) + } + if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") { + return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason) + } + //workflow.batchJobDefinitionArn = stack.Outputs["BatchJobDefinitionArn"] + + return nil + } +} + func (workflow *serviceWorkflow) serviceCreateSchedules(namespace string, service *common.Service, environmentName string, stackWaiter common.StackWaiter, stackUpserter common.StackUpserter) Executor { return func() error { log.Noticef("Creating schedules for service '%s' to '%s'", workflow.serviceName, environmentName) diff --git a/workflows/service_deploy_test.go b/workflows/service_deploy_test.go index e16a67fe..eef2e0c2 100644 --- a/workflows/service_deploy_test.go +++ b/workflows/service_deploy_test.go @@ -51,7 +51,10 @@ func TestServiceApplyCommon_Create(t *testing.T) { workflow.serviceName = "myservice" workflow.envStack = &common.Stack{Name: "mu-environment-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.lbStack = &common.Stack{Name: "mu-loadbalancer-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} - err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister, paramManager)() + err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister)() + assert.Nil(err) + + err = workflow.serviceDbParams("mu", service, params, "dev", stackManager, paramManager)() assert.Nil(err) assert.Equal("mu-environment-dev-VpcId", params["VpcId"]) @@ -89,7 +92,10 @@ func TestServiceApplyCommon_Update(t *testing.T) { workflow.serviceName = "myservice" workflow.envStack = &common.Stack{Name: "mu-environment-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.lbStack = &common.Stack{Name: "mu-loadbalancer-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} - err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister, paramManager)() + err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister)() + assert.Nil(err) + + err = workflow.serviceDbParams("mu", service, params, "dev", stackManager, paramManager)() assert.Nil(err) assert.Equal("", params["ListenerRulePriority"]) @@ -124,7 +130,10 @@ func TestServiceApplyCommon_StaticPriority(t *testing.T) { workflow.envStack = &common.Stack{Name: "mu-environment-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.lbStack = &common.Stack{Name: "mu-loadbalancer-dev", Status: common.StackStatusCreateComplete, Outputs: outputs} workflow.priority = 77 - err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister, paramManager)() + err := workflow.serviceApplyCommonParams("mu", service, params, "dev", stackManager, elbRuleLister)() + assert.Nil(err) + + err = workflow.serviceDbParams("mu", service, params, "dev", stackManager, paramManager)() assert.Nil(err) assert.Equal("77", params["PathListenerRulePriority"]) diff --git a/workflows/service_push.go b/workflows/service_push.go index b7d17429..11148ca5 100644 --- a/workflows/service_push.go +++ b/workflows/service_push.go @@ -22,8 +22,8 @@ func NewServicePusher(ctx *common.Context, tag string, provider string, kmsKey s newConditionalExecutor(workflow.isEcrProvider(), newPipelineExecutor( workflow.serviceRepoUpserter(ctx.Config.Namespace, &ctx.Config.Service, ctx.StackManager, ctx.StackManager), - workflow.serviceImageBuilder(ctx.DockerManager, &ctx.Config, dockerWriter), workflow.serviceRegistryAuthenticator(ctx.ClusterManager), + workflow.serviceImageBuilder(ctx.DockerManager, &ctx.Config, dockerWriter), workflow.serviceImagePusher(ctx.DockerManager, dockerWriter), ), newPipelineExecutor( @@ -36,7 +36,7 @@ func NewServicePusher(ctx *common.Context, tag string, provider string, kmsKey s func (workflow *serviceWorkflow) serviceImageBuilder(imageBuilder common.DockerImageBuilder, config *common.Config, dockerWriter io.Writer) Executor { return func() error { log.Noticef("Building service:'%s' as image:%s'", workflow.serviceName, workflow.serviceImage) - return imageBuilder.ImageBuild(config.Basedir, workflow.serviceName, config.Service.Dockerfile, []string{workflow.serviceImage}, dockerWriter) + return imageBuilder.ImageBuild(config.Basedir, workflow.serviceName, config.Service.Dockerfile, []string{workflow.serviceImage}, workflow.registryAuthConfig, dockerWriter) } } diff --git a/workflows/service_push_test.go b/workflows/service_push_test.go index 1620d9a2..e18c18c1 100644 --- a/workflows/service_push_test.go +++ b/workflows/service_push_test.go @@ -1,6 +1,7 @@ package workflows import ( + "github.com/docker/docker/api/types" "github.com/stelligent/mu/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -21,7 +22,7 @@ type mockServiceBuilder struct { common.DockerImageBuilder } -func (m *mockServiceBuilder) ImageBuild(basedir string, serviceName string, dockerfile string, tags []string, dockerWriter io.Writer) error { +func (m *mockServiceBuilder) ImageBuild(basedir string, serviceName string, dockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerWriter io.Writer) error { args := m.Called() return args.Error(0) }