Batch support (as a service provider)

.gitignore

@@ -8,3 +8,4 @@ vendor/
 homebrew-tap/
 *-packr.go
 CHANGELOG.md
+*.bak

common/docker.go

@@ -17,7 +17,7 @@ import (
 
 // DockerImageBuilder for creating docker images
 type DockerImageBuilder interface {
-	ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, dockerOut io.Writer) error
+	ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerOut io.Writer) error
 }
 
 // DockerImagePusher for pushing docker images
@@ -47,10 +47,11 @@ func newClientDockerManager() (DockerManager, error) {
 	}, nil
 }
 
-func (d *clientDockerManager) ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, dockerOut io.Writer) error {
+func (d *clientDockerManager) ImageBuild(contextDir string, serviceName string, relDockerfile string, tags []string, registryAuthConfig map[string]types.AuthConfig, dockerOut io.Writer) error {
 	options := types.ImageBuildOptions{
-		Tags:   tags,
-		Labels: map[string]string{"SERVICE_NAME": serviceName},
+		Tags:        tags,
+		Labels:      map[string]string{"SERVICE_NAME": serviceName},
+		AuthConfigs: registryAuthConfig,
 	}
 
 	buildContext, err := createBuildContext(contextDir, relDockerfile)

common/types.go

@@ -169,17 +169,26 @@ type Service struct {
 	HostPatterns         []string               `yaml:"hostPatterns,omitempty"`
 	Priority             int                    `yaml:"priority,omitempty" validate:"max=50000"`
 	Pipeline             Pipeline               `yaml:"pipeline,omitempty"`
+	ProviderOverride     string                 `yaml:"provider,omitempty"`
 	Database             Database               `yaml:"database,omitempty"`
 	Schedule             []Schedule             `yaml:"schedules,omitempty"`
 	TargetCPUUtilization int                    `yaml:"targetCPUUtilization,omitempty" validate:"max=100"`
 	DiscoveryTTL         string                 `yaml:"discoveryTTL,omitempty"`
-	Roles                struct {
+
+	// Batch
+	Parameters    map[string]interface{} `yaml:"parameters,omitempty"`
+	Command       []string               `yaml:"command,omitempty"`
+	Timeout       int                    `yaml:"timeout,omitempty"`
+	RetryAttempts int                    `yaml:"retryAttempts,omitempty"`
+
+	Roles struct {
 		Ec2Instance            string `yaml:"ec2Instance,omitempty" validate:"validateRoleARN"`
 		CodeDeploy             string `yaml:"codeDeploy,omitempty" validate:"validateRoleARN"`
 		EcsEvents              string `yaml:"ecsEvents,omitempty" validate:"validateRoleARN"`
 		EcsService             string `yaml:"ecsService,omitempty" validate:"validateRoleARN"`
 		EcsTask                string `yaml:"ecsTask,omitempty" validate:"validateRoleARN"`
 		ApplicationAutoScaling string `yaml:"applicationAutoScaling,omitempty" validate:"validateRoleARN"`
+		BatchJobRole           string `yaml:"batchJobRole,omitempty" validate:"validateRoleARN"`
 	} `yaml:"roles,omitempty"`
 }
 
@@ -395,13 +404,15 @@ const (
 	TemplateEnvECS                  = "cloudformation/env-ecs.yml"
 	TemplateEnvEKS                  = "cloudformation/env-eks.yml"
 	TemplateEnvEKSBootstrap         = "cloudformation/env-eks-bootstrap.yml"
+	TemplateEnvBatch                = "cloudformation/env-batch.yml"
 	TemplateEnvIAM                  = "cloudformation/env-iam.yml"
 	TemplatePipelineIAM             = "cloudformation/pipeline-iam.yml"
 	TemplatePipeline                = "cloudformation/pipeline.yml"
 	TemplateRepo                    = "cloudformation/repo.yml"
 	TemplateSchedule                = "cloudformation/schedule.yml"
 	TemplateServiceEC2              = "cloudformation/service-ec2.yml"
 	TemplateServiceECS              = "cloudformation/service-ecs.yml"
+	TemplateServiceBatch            = "cloudformation/service-batch.yml"
 	TemplateServiceIAM              = "cloudformation/service-iam.yml"
 	TemplateVPCTarget               = "cloudformation/vpc-target.yml"
 	TemplateVPC                     = "cloudformation/vpc.yml"
@@ -418,8 +429,8 @@ type DeploymentStrategy string
 // List of supported deployment strategies
 const (
 	BlueGreenDeploymentStrategy DeploymentStrategy = "blue_green"
-	RollingDeploymentStrategy   DeploymentStrategy = "rolling"
-	ReplaceDeploymentStrategy   DeploymentStrategy = "replace"
+	RollingDeploymentStrategy                      = "rolling"
+	ReplaceDeploymentStrategy                      = "replace"
 )
 
 // EnvProvider describes supported environment strategies
@@ -432,6 +443,7 @@ const (
 	EnvProviderEc2                    = "ec2"
 	EnvProviderEks                    = "eks"
 	EnvProviderEksFargate             = "eks-fargate"
+	EnvProviderBatch                  = "batch"
 )
 
 // InstanceTenancy describes supported tenancy options for EC2

examples/batch/Dockerfile

@@ -0,0 +1,8 @@
+FROM amazonlinux:latest
+
+RUN yum -y install unzip aws-cli
+ADD batch-job.sh /usr/local/bin/batch-job.sh
+WORKDIR /tmp
+USER nobody
+
+ENTRYPOINT ["/usr/local/bin/batch-job.sh"]

examples/batch/README.md

@@ -0,0 +1,5 @@
+# Examples
+These examples are not intended to be run directly.  Rather, they serve as a reference that can be consulted when creating your own `mu.yml` files.
+
+For detailed steps to create your own project, check out the [quickstart](https://github.com/stelligent/mu/wiki/Quickstart#steps).
+

examples/batch/batch-job.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+date
+# this will expose parameters passed from mu.yml command section
+echo "Args: $@"
+env
+echo "This is my simple test job!."
+echo "jobId: $AWS_BATCH_JOB_ID"
+echo "jobQueue: $AWS_BATCH_JQ_NAME"
+echo "computeEnvironment: $AWS_BATCH_CE_NAME"
+sleep $1
+date
+echo "bye bye!!"

examples/batch/buildspec.yml

@@ -0,0 +1,12 @@
+version: 0.1
+
+phases:
+  build:
+    commands:
+      - echo "replace me with real build commands..."
+
+artifacts:
+  files:
+    - batch-job.sh
+    - mu.yml
+    - Dockerfile

examples/batch/mu.yml

@@ -0,0 +1,49 @@
+## See https://github.com/stelligent/mu/wiki/Services
+service:
+  name: batch-sample
+
+  # deployed as AWS Batch job
+  provider: batch
+
+  # The relative path to the Dockerfile to build images (default: Dockerfile)
+  dockerfile: Dockerfile
+
+  # The number of VCPUs to allocate to job (default: 2)  
+  # Each vCPU is equivalent to 1,024 CPU shares.
+  cpu: 1                             
+
+  # The hard limit (in MiB) of memory to present to the container. If your container attempts to exceed the memory specified here, the container is killed.
+  # Read this: https://docs.aws.amazon.com/batch/latest/userguide/memory-management.html
+  memory: 128                       
+
+  # The time duration in seconds (measured from the job attempt's startedAt timestamp) 
+  # after which AWS Batch terminates your jobs if they have not finished. 
+  # If a job is terminated due to a timeout, it is not retried. 
+  timeout: 0
+  retryAttempts: 1
+
+  # our docker file already has a defined entry point script, these params
+  # will be appended to it
+  command:
+    - '--some-flag'
+    - Ref::param1
+
+  # Default parameters
+  # Parameter values replace the placeholders in the command, i.e. Ref::paramname.
+  # Usually final parameter values are specified on job submit
+  parameters:
+    param1: "Sample text passed to the batch job"
+
+  ### Environment variables exposed to the containers
+  environment:
+
+    #### Define this for each environment if you want APP_DEBUG=true
+    APP_DEBUG:
+      dev: 'true'
+      prod: 'false'
+
+    APP_ENV:
+      dev: dev
+      prod: production
+
+

provider/aws/cloudformation.go

@@ -65,14 +65,22 @@ func newStackManager(sess *session.Session, extensionsManager common.ExtensionsM
 
 }
 
-func buildStackParameters(parameters map[string]string) []*cloudformation.Parameter {
+func buildStackParameters(parameters map[string]string, stack *common.Stack) []*cloudformation.Parameter {
+
 	stackParameters := make([]*cloudformation.Parameter, 0, len(parameters))
 	for key, value := range parameters {
+		// Stack exists, check if parameter exists. Needed to avoid:
+		// ValidationError: Invalid input for parameter key DatabaseName. Cannot specify usePreviousValue as true for a parameter key not in the previous template
+		usePrevVal := true
+		if stack != nil && stack.Status != "" {
+			_, usePrevVal = stack.Parameters[key]
+		}
+
 		stackParameters = append(stackParameters,
 			&cloudformation.Parameter{
 				ParameterKey:     aws.String(key),
 				ParameterValue:   aws.String(value),
-				UsePreviousValue: aws.Bool(value == ""),
+				UsePreviousValue: aws.Bool(value == "" && usePrevVal),
 			})
 	}
 	return stackParameters
@@ -333,7 +341,7 @@ func (cfnMgr *cloudformationStackManager) UpsertStack(stackName string, template
 	if err != nil {
 		return err
 	}
-	stackParameters := buildStackParameters(parameters)
+	stackParameters := buildStackParameters(parameters, stack)
 
 	// stack tags
 	tags, err = cfnMgr.extensionsManager.DecorateStackTags(stackName, tags)

provider/aws/cloudformation_test.go

@@ -342,17 +342,39 @@ func TestBuildParameters(t *testing.T) {
 
 	paramMap := make(map[string]string)
 
-	parameters := buildStackParameters(paramMap)
+	parameters := buildStackParameters(paramMap, nil)
 	assert.Equal(0, len(parameters))
 
 	paramMap["p1"] = "value 1"
 	paramMap["p2"] = "value 2"
-	parameters = buildStackParameters(paramMap)
+	parameters = buildStackParameters(paramMap, nil)
 	assert.Equal(2, len(parameters))
 	assert.Contains(*parameters[0].ParameterKey, "p")
 	assert.Contains(*parameters[0].ParameterValue, "value")
 	assert.Contains(*parameters[1].ParameterKey, "p")
 	assert.Contains(*parameters[1].ParameterValue, "value")
+
+	stackParameters := make([]*cloudformation.Parameter, 0, len(parameters))
+	stackParameters = append(stackParameters,
+		&cloudformation.Parameter{
+			ParameterKey:   aws.String("ExistingParam"),
+			ParameterValue: aws.String("ExistingValue"),
+		})
+
+	stackDetails := cloudformation.Stack{
+		StackName:    aws.String("mu-environment-dev"),
+		CreationTime: aws.Time(time.Now()),
+		Tags:         []*cloudformation.Tag{},
+		StackStatus:  aws.String("CREATE_COMPLETE"),
+		Parameters:   stackParameters,
+	}
+
+	stack := buildStack(&stackDetails)
+	paramMap["ExistingParam"] = "" // should have UsePreviousValue==true
+	paramMap["NewParam"] = ""      // should have UsePreviousValue==false
+	parameters = buildStackParameters(paramMap, stack)
+	assert.Equal(*parameters[2].UsePreviousValue, true)
+	assert.Equal(*parameters[3].UsePreviousValue, false)
 }
 
 func TestTagParameters(t *testing.T) {

provider/aws/roleset.go

@@ -68,6 +68,7 @@ func (rolesetMgr *iamRolesetManager) GetServiceRoleset(environmentName string, s
 	overrideRole(roleset, "EcsServiceRoleArn", rolesetMgr.context.Config.Service.Roles.EcsService)
 	overrideRole(roleset, "EcsTaskRoleArn", rolesetMgr.context.Config.Service.Roles.EcsTask)
 	overrideRole(roleset, "ApplicationAutoScalingRoleArn", rolesetMgr.context.Config.Service.Roles.ApplicationAutoScaling)
+	overrideRole(roleset, "BatchJobRoleArn", rolesetMgr.context.Config.Service.Roles.BatchJobRole)
 	return roleset, nil
 }
 
@@ -185,6 +186,11 @@ func (rolesetMgr *iamRolesetManager) GetEnvironmentProvider(environmentName stri
 			break
 		}
 	}
+	// allow to override provider (batch job definition can be deployed without environment)
+	if rolesetMgr.context.Config.Service.ProviderOverride != "" {
+		// todo: validate values
+		envProvider = rolesetMgr.context.Config.Service.ProviderOverride
+	}
 	if envProvider == "" {
 		log.Debugf("unable to find environment named '%s' in configuration...checking for existing stack", environmentName)
 		envStackName := common.CreateStackName(rolesetMgr.context.Config.Namespace, common.StackTypeEnv, environmentName)

templates/assets/cloudformation/common-iam.yml

@@ -384,6 +384,21 @@ Resources:
               StringLike:
                 iam:AWSServiceName: rds.amazonaws.com
             Effect: Allow
+      - PolicyName: aws-batch-jobs
+        PolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+          - Action:
+            - batch:RegisterJobDefinition
+            - batch:DeregisterJobDefinition
+            - batch:DescribeJobDefinitions
+            Resource: '*'
+            Effect: Allow
+          - Action:
+            - iam:PassRole
+            Resource:
+            - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${Namespace}-service-*-batchjob-${AWS::Region}
+            Effect: Allow
 Outputs:
   CloudFormationRoleArn:
     Description: Role assummed by CloudFormation