I have created a cloudformation template to launch an emr cluster using a lambda function as a trigger. The stack creation is successful but when I go to the lambda function it says "Lambda can't find the file lambda_function.py. Make sure that your handler upholds the format: file-name.method." The format I have given is: "lambda_function.lambda_handler" The s3 bucket from where the file is being accessed also has the zip folder with the lambda_function.py file.
And this is the lambda function created and how it looks when I create a cloudwatch event to trigger the same: enter image description here
What might be the issue here? Here is my code:
AWSTemplateFormatVersion: 2010-09-09
Description: EMR Cluster for Spark
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
- Label:
default: Common Parameters
Parameters:
- SystemValue
- SubSystemValue
- RevenueStreamValue
- EnvironmentValue
- KMSArn
- Label:
default: EMR Parameters
default: Lambda Parameters
Parameters:
- EMRClusterName
- KeyName
- MasterInstanceType
- CoreInstanceType
- NumberOfCoreInstances
- VpcId
- VPCSubnetIds
- LogUriFolder
- S3DataUri
- ReleaseLabel
- ApplicationsList
- PackageBucketName
- EbsRootVolumeSize
- FunctionName1Value
- Label:
default: S3 Parameters
Parameters:
- EDWBucketName
ParameterLabels:
SystemValue:
default: System
SubSystemValue:
default: SubSystem
RevenueStreamValue:
default: Revenue Stream
EnvironmentValue:
default: Environment Value
FunctionName1Value:
default: Lambda1Name
EMRClusterName:
default: EMR Cluster Name
KeyName:
default: Key Name
MasterInstanceType:
default: Master Instance Type
CoreInstanceType:
default: Core Instance Type
NumberOfCoreInstances:
default: Number Of Core Instances
VpcId:
default: VPC ID
VPCSubnetIds:
default: VPC Subnet ID
LogUriFolder:
default: Log Uri Folder
S3DataUri:
default: S3 Data Uri
ReleaseLabel:
default: Release Label
ApplicationsList:
default: Applications List
KMSArn:
default: KMS Arn
EDWBucketName:
default: EDW Bucket Name
PackageBucketName:
default: Package Bucket Name
EbsRootVolumeSize:
default: Ebs Root Volume Size
#########################################################################
Parameters:
KMSArn:
Type: String
Description: Enter KMS ARN based on your subsystem.
Default: 'arn:aws:kms:us-east-1:084067650016:key/1b84929f-8fea-4d0a-bfa4-dba0a69bd9a0'
SystemValue:
Type: String
Description: Enter System Name
Default: 'Messaging'
SubSystemValue:
Type: String
Description: Enter SubSystem Name
Default: 'edw'
RevenueStreamValue:
Type: String
Description: Enter Revenue Stream Name
Default: 'edw'
FunctionName1Value:
Type: String
Default: 'EMRCluster'
Description: Enter 1st Lambda Function Name(Do not append System & Sub-System Name).
EDWBucketName:
Type: String
Default: 'crx-dev-messaging-edw'
Description: Enter crx-[env]-messaging-edw
PackageBucketName:
Type: String
Default: 'crx-dev-deployments'
Description: Enter crx-[env]-deployments
EnvironmentValue:
Type: String
Default: 'dev'
LambdaRuntime:
Type: String
Default: 'python2.7'
TagEnvironmentValue:
Type: String
Default: 'dev'
VpcId:
Type: 'AWS::EC2::VPC::Id'
Default: vpc-7c368507
VpcCIDR:
Type: String
Default: '10.10.16.0/20'
VPCSubnetIds:
Type: CommaDelimitedList
Default: subnet-4d527c62
Env:
Type: String
Default: 'dev'
KeyName:
Type: 'AWS::EC2::KeyPair::KeyName'
Description: Ane<Env>-Messaging-EDW
Default: development
MasterInstanceType:
Type: String
Description: Instance type to be used for the master instance.
Default: 'm5.xlarge'
AllowedValues:
- m5.xlarge
- m1.medium
CoreInstanceType:
Type: String
Description: Instance type to be used for core instances.
Default: 'm5.xlarge'
AllowedValues:
- m5.xlarge
- m1.medium
NumberOfCoreInstances:
Description: Must be a valid number - 2
Type: Number
Default: '2'
LogUriFolder:
Type: String
Description: S3 Folder name for spark logs (spark-logs)
Default: 's3n://crx-dev-messaging-edw/spark-logs/'
S3DataUri:
Type: String
Description: Must be a valid S3 bucket URL
Default: 's3n://crx-messaging-edw'
EbsRootVolumeSize:
Type: String
Description: Specify the volume size
Default: '200'
ReleaseLabel:
Type: String
Description: Must be a valid EMR release version emr-6.0.0
Default: 'emr-6.0.0'
ApplicationsList:
Type: String
Description: Please select which application will be installed on the cluster
this would be either Ganglia,spark,Hadoop etc.,
Default: 'Spark'
AllowedValues:
- Hadoop
- Ganglia
- Spark
#########################################################################
Resources:
Bucket1:
Type: 'AWS::S3::Bucket'
Properties:
BucketName: !Sub 'crx-test-${EnvironmentValue}-${SubSystemValue}'
BucketEncryption:
ServerSideEncryptionConfiguration:
- ServerSideEncryptionByDefault:
SSEAlgorithm: aws:kms
KMSMasterKeyID: !Ref KMSArn
PublicAccessBlockConfiguration:
BlockPublicAcls: true
BlockPublicPolicy: true
IgnorePublicAcls: true
RestrictPublicBuckets: true
#EbsRootVolumeSize: !Ref EbsRootVolumeSize
SvcAccessSecurityGroup:
Type: 'AWS::EC2::SecurityGroup'
Properties:
GroupName: !Sub SG-${Env}-${SystemValue}-${SubSystemValue}-ServiceAccess
VpcId: !Ref VpcId
GroupDescription: !Sub 'SG-${SystemValue}-${SubSystemValue}-Service Access Security Group'
SecurityGroupIngress:
- IpProtocol: tcp
FromPort: 443
ToPort: 443
CidrIp: !Ref VpcCIDR
SecurityGroupEgress:
- IpProtocol: -1
CidrIp: 0.0.0.0/0
Tags:
- Key: Environment
Value: !Ref TagEnvironmentValue
- Key: RevenueStream
Value: !Ref RevenueStreamValue
- Key: SubSystem
Value: !Ref SubSystemValue
- Key: System
Value: !Ref SystemValue
- Key: Name
Value: !Sub 'SG-${Env}-${SystemValue}-${SubSystemValue}-ServiceAccess'
EmrMasterSecurityGroup:
Type: 'AWS::EC2::SecurityGroup'
Properties:
GroupName: !Sub 'SG-${Env}-${SystemValue}-${SubSystemValue}-EMRMaster'
VpcId: !Ref VpcId
GroupDescription: !Sub 'SG-${SystemValue}-${SubSystemValue}-EMR Managed Master Security Group'
SecurityGroupIngress:
- IpProtocol: tcp
FromPort: 443
ToPort: 443
CidrIp: !Ref VpcCIDR
SecurityGroupEgress:
- IpProtocol: -1
CidrIp: 0.0.0.0/0
Tags:
- Key: Environment
Value: !Ref TagEnvironmentValue
- Key: RevenueStream
Value: !Ref RevenueStreamValue
- Key: SubSystem
Value: !Ref SubSystemValue
- Key: System
Value: !Ref SystemValue
- Key: Name
Value: !Sub 'SG-${Env}-${SystemValue}-${SubSystemValue}-EMRMaster'
EmrSlaveSecurityGroup:
Type: 'AWS::EC2::SecurityGroup'
Properties:
GroupName: !Sub 'SG-${Env}-${SystemValue}-${SubSystemValue}-EMRSlave'
VpcId: !Ref VpcId
GroupDescription: !Sub 'SG-${SystemValue}-${SubSystemValue}-EMR Managed Slave Security Group'
SecurityGroupIngress:
- IpProtocol: tcp
FromPort: 443
ToPort: 443
CidrIp: !Ref VpcCIDR
SecurityGroupEgress:
- IpProtocol: -1
CidrIp: 0.0.0.0/0
Tags:
- Key: Environment
Value: !Ref TagEnvironmentValue
- Key: RevenueStream
Value: !Ref RevenueStreamValue
- Key: SubSystem
Value: !Ref SubSystemValue
- Key: System
Value: !Ref SystemValue
- Key: Name
Value: !Sub 'SG-${Env}-${SystemValue}-${SubSystemValue}-EMRSlave'
LambdaSecurityGroup:
Type: 'AWS::EC2::SecurityGroup'
Properties:
GroupName: !Sub 'SG-${Env}-${SystemValue}-${SubSystemValue}'
VpcId: !Ref VpcId
GroupDescription: !Sub 'SG-${SystemValue}-${SubSystemValue}-Lambda Security Group'
SecurityGroupIngress:
- IpProtocol: tcp
FromPort: 443
ToPort: 443
CidrIp: !Ref VpcCIDR
SecurityGroupEgress:
- IpProtocol: -1
CidrIp: 0.0.0.0/0
Tags:
- Key: Environment
Value: !Ref TagEnvironmentValue
- Key: RevenueStream
Value: !Ref RevenueStreamValue
- Key: SubSystem
Value: !Ref SubSystemValue
- Key: System
Value: !Ref SystemValue
- Key: Name
Value: !Sub 'SG-${Env}-${SystemValue}-${SubSystemValue}'
Lambda1:
Type: 'AWS::Lambda::Function'
DependsOn:
- LambdaExecutionRole
- ManagerApiPolicy
Properties:
Description: Script to launch EMR
FunctionName: !Sub '${SystemValue}-${SubSystemValue}-${FunctionName1Value}'
Handler: lambda_function.lambda_handler
Code:
S3Bucket: !Ref PackageBucketName
S3Key: emr-launch.zip
Role: !GetAtt
- LambdaExecutionRole
- Arn
Runtime: !Ref LambdaRuntime
Timeout: '900'
MemorySize: 512
Environment:
Variables:
parameterPath: !Sub '/crx/${SystemValue}/${SubSystemValue}/'
region: !Ref 'AWS::Region'
VpcConfig:
SubnetIds: !Ref VPCSubnetIds
SecurityGroupIds:
- !GetAtt
- LambdaSecurityGroup
- GroupId
Tags:
- Value: !Sub '${SystemValue}-${SubSystemValue}-${FunctionName1Value}'
Key: Name
- Key: SubSystem
Value: !Ref SubSystemValue
- Key: System
Value: !Ref SystemValue
- Value: !Ref RevenueStreamValue
Key: RevenueStream
- Value: !Ref TagEnvironmentValue
Key: Environment
LambdaExecutionRole:
Type: 'AWS::IAM::Role'
DependsOn:
- LambdaSecurityGroup
Properties:
RoleName: !Sub '${SystemValue}-${SubSystemValue}'
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service:
- lambda.amazonaws.com
- states.amazonaws.com
- events.amazonaws.com
Action:
- 'sts:AssumeRole'
ManagerApiPolicy:
Type: 'AWS::IAM::ManagedPolicy'
DependsOn:
- LambdaExecutionRole
Properties:
ManagedPolicyName: !Sub '${SystemValue}-${SubSystemValue}'
PolicyDocument:
Version: 2012-10-17
Statement:
- Sid: common
Effect: Allow
Action:
- 'ec2:CreateNetworkInterface'
- 'ec2:Describe*'
- 'ec2:Get*'
- 'ec2:DeleteNetworkInterface'
- 'kms:GenerateDataKey'
- 'kms:ListAliases'
- 'kms:ListKeys'
- 'kms:ReEncryptTo'
- 'kms:CreateKey'
- 'kms:GenerateRandom'
- 'kms:ReEncryptFrom'
- 'logs:CreateLogGroup'
- 'logs:PutLogEvents'
- 'logs:CreateLogStream'
- 'ssm:DescribeParameters'
- 'lambda:InvokeFunction'
Resource: '*'
- Sid: ssm
Effect: Allow
Action:
- 'ssm:GetParametersByPath'
- 'ssm:GetParameters'
- 'ssm:GetParameter'
Resource:
- !Join
- ''
- - 'arn:aws:ssm:'
- !Ref 'AWS::Region'
- ':'
- !Ref 'AWS::AccountId'
- ':parameter/crx/'
- !Sub '${SystemValue}/${SubSystemValue}*'
- Sid: kms
Effect: Allow
Action: 'kms:*'
Resource:
- !Ref KMSArn
- !Join
- ''
- - 'arn:aws:kms:'
- !Ref 'AWS::Region'
- ':'
- !Ref 'AWS::AccountId'
- ':alias/'
- !Sub '${SystemValue}-${SubSystemValue}'
- Sid: s3
Effect: Allow
Action:
- 's3:PutObject'
- 's3:GetObject'
- 's3:List*'
Resource:
- !Join
- ''
- - 'arn:aws:s3:'
- '::'
- !Sub 'crx-${EnvironmentValue}-${PackageBucketName}'
- !Join
- ''
- - 'arn:aws:s3:'
- '::'
- !Sub 'crx-${EnvironmentValue}-${PackageBucketName}/*'
- !Join
- ''
- - 'arn:aws:s3:'
- '::'
- !Sub 'crx-${EnvironmentValue}-public-assets'
- !Join
- ''
- - 'arn:aws:s3:'
- '::'
- !Sub 'crx-${EnvironmentValue}-public-assets/*'
Roles:
- !Ref LambdaExecutionRole
LambdaExecutionRoleforEMRPolicy:
DependsOn:
- LambdaExecutionRole
Type: 'AWS::IAM::Policy'
Properties:
PolicyName: LambdaforEMRRolePolicy
Roles:
- !Ref LambdaExecutionRole
PolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Action:
- 'logs:CreateLogGroup'
- 'logs:CreateLogStream'
- 'logs:PutLogEvents'
Resource: '*'
- Effect: Allow
Action:
- 'elasticmapreduce:*'
Resource: '*'
LambdaPermission1:
Type: 'AWS::Lambda::Permission'
Properties:
Action: 'lambda:invokeFunction'
FunctionName: !GetAtt
- Lambda1
- Arn
Principal: events.amazonaws.com
SourceArn: !Join
- ''
- - 'arn:aws:events:'
- !Ref 'AWS::Region'
- ':'
- !Ref 'AWS::AccountId'
- ':'
- 'rule/'
- !Sub '${SystemValue}-${SubSystemValue}-${FunctionName1Value}'
# PermissionForEventsToInvokeLambda:
# Type: 'AWS::Lambda::Permission'
# Properties:
# FunctionName:
# !Ref LambdaExecutionRole
# Action: lambda:InvokeFunction
# Principal: apigateway.amazonaws.com
# SourceArn: !GetAtt
# - Lambda1
# - Arn
EMRClusterServiceRole:
Type: 'AWS::IAM::Role'
Properties:
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service:
- ec2.amazonaws.com
Action:
- 'sts:AssumeRole'
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
Path: /
EMRClusterinstanceProfileRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service:
- ec2.amazonaws.com
Action:
- 'sts:AssumeRole'
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
Path: /
EMRClusterinstanceProfile:
Type: AWS::IAM::InstanceProfile
Properties:
Path: /
Roles:
- !Ref EMRClusterinstanceProfileRole
