first commit

.gitignore

@@ -0,0 +1,4 @@
+.terraform
+.terraform.lock.hcl
+terraform.tfstate
+terraform.tfstate.backup

README.md

@@ -0,0 +1,34 @@
+## S3 bucket with a Lambda function as a listener and append to a DynamoDB table With Terraform
+
+This is a simple example of how to use a Lambda function as a listener for an S3 bucket and append the data to a DynamoDB table.
+
+### Prerequisites
+
+- Terraform 1.4.x
+- AWS CLI
+- AWS account with permissions to create the resources
+- AWS credentials configured
+
+## Usage
+
+1. Clone this repository
+2. go to ``` locals.tf ``` in the root directory and change the account_id to your own account id  
+3. need to make sure the javascript file inside ``` module/lambda/files/lambda_function.zip ``` is wil the right region and table name
+4. Run `terraform init`
+5. Run `terraform apply`
+
+## Example playlod file *.json
+
+``` json
+{
+  "id": "00a6fa25-df29-4701-9077-557932591766",
+  "timestamp": 23764783264,
+  "data": "CONTENT OF THE FILE"
+}
+```
+
+
+### Test
+1. Upload a file to the S3 bucket
+2. Check the DynamoDB table for the new item
+3. check the CloudWatch logs for the Lambda function

locals.tf

@@ -0,0 +1,7 @@
+locals {
+   account_id = "539634357948"
+   default_tags = {
+     Project = "my-project"
+     Owner   = "my-team"
+   }
+}

main.tf

@@ -0,0 +1,29 @@
+provider "aws" {
+  region = var.regoin
+}
+
+module "role" {
+  source              = "./modules/iam/role"
+  account_id          = local.account_id
+  s3_bucket           = var.s3_bucket
+  dynamodb_table_name = var.dynamodb_table_name
+}
+
+module "dynamodb" {
+  source     = "./modules/dynamodb"
+  table_name = var.dynamodb_table_name
+}
+
+module "s3" {
+  source    = "./modules/s3"
+  s3_bucket = var.s3_bucket
+}
+
+module "lambda_function" {
+  source = "./modules/lambda"
+
+  lambda_function_role = module.role.arn
+  s3_bucket            = var.s3_bucket
+  project              = var.project
+  s3_arn               = module.s3.s3_bucket_arn
+}

modules/dynamodb/main.tf

@@ -0,0 +1,67 @@
+resource "aws_dynamodb_table" "lambda_table" {
+  name         = var.table_name
+  billing_mode = var.billing_mode
+  hash_key     = var.hash_key
+  range_key    = var.range_key
+
+  attribute {
+    name = var.hash_key
+    type = var.hash_key_type
+  }
+
+  attribute {
+    name = var.range_key
+    type = "N"
+  }
+
+  attribute {
+    name = "data"
+    type = "S"
+  }
+
+  attribute {
+    name = "timestampIndex"
+    type = var.sort_key_type
+  }
+
+  global_secondary_index {
+    name               = "data-index"
+    hash_key           = "data"
+    projection_type    = "ALL"
+    read_capacity      = var.read_capacity_units
+    write_capacity     = var.write_capacity_units
+    non_key_attributes = []
+  }
+
+  global_secondary_index {
+    name               = "id-index"
+    hash_key           = "id"
+    projection_type    = "ALL"
+    read_capacity      = var.read_capacity_units
+    write_capacity     = var.write_capacity_units
+    non_key_attributes = []
+  }
+
+  global_secondary_index {
+    name               = "timestamp-index"
+    hash_key           = "timestampIndex"
+    range_key          = var.range_key
+    read_capacity      = var.read_capacity_units
+    write_capacity     = var.write_capacity_units
+    projection_type    = "INCLUDE"
+    non_key_attributes = ["id", "data"]
+  }
+}
+
+
+output "name" {
+  value = aws_dynamodb_table.lambda_table.name
+}
+
+output "arn" {
+  value = aws_dynamodb_table.lambda_table.arn
+}
+
+output "stream_arn" {
+  value = aws_dynamodb_table.lambda_table.stream_arn
+}

modules/dynamodb/variables.tf

@@ -0,0 +1,53 @@
+variable "table_name" {
+  description = "The name of the table"
+  type        = string
+  default     = "example"
+}
+
+variable "billing_mode" {
+  description = "The billing mode of the table"
+  type        = string
+  default     = "PAY_PER_REQUEST"
+}
+
+variable "range_key" {
+  description = "The name of the range key in the table"
+  type        = string
+  default     = "timestamp"
+}
+
+variable "hash_key" {
+  description = "The name of the hash key in the table"
+  type        = string
+  default     = "id"
+}
+
+variable "hash_key_type" {
+  description = "The type of the hash key in the table"
+  type        = string
+  default     = "S"
+}
+
+variable "sort_key" {
+  description = "The name of the sort key in the table"
+  type        = string
+  default     = "timestamp"
+}
+
+variable "sort_key_type" {
+  description = "The type of the sort key in the table"
+  type        = string
+  default     = "N"
+}
+
+variable "read_capacity_units" {
+  description = "The number of read capacity units for the DynamoDB table"
+  type        = number
+  default     = 5
+}
+
+variable "write_capacity_units" {
+  description = "The number of write capacity units for the DynamoDB table"
+  type        = number
+  default     = 5
+}

modules/iam/role/main.tf

@@ -0,0 +1,90 @@
+data "aws_iam_policy_document" "assume_role_policy" {
+  statement {
+    effect = "Allow"
+
+    actions = ["sts:AssumeRole"]
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role" "role_for_lambda" {
+  name               = var.role_name
+  assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
+
+  depends_on = [data.aws_iam_policy_document.assume_role_policy]
+}
+
+resource "aws_iam_role_policy" "lambda_policy" {
+
+  name        = "lambda_policy"
+  role        = aws_iam_role.role_for_lambda.id
+  policy      = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "S3Access",
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${var.s3_bucket}/files/*.json"
+      ]
+    },
+    {
+      "Sid": "CloudWatchLogsAccess",
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": [
+        "arn:aws:logs:us-east-1:${var.account_id}:*"
+      ]
+    }
+  ]
+}
+EOF
+  depends_on  = [aws_iam_role.role_for_lambda]
+}
+
+
+resource "aws_iam_policy" "lambda_dynamodb_policy" {
+  name        = "lambda_dynamodb_policy"
+  description = "Policy for Lambda to write to DynamoDB"
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "dynamodb:PutItem"
+      ],
+      "Resource": [
+        "arn:aws:dynamodb:us-east-1:${var.account_id}:table/${var.dynamodb_table_name}"
+      ]
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_role_policy_attachment" {
+  policy_arn = aws_iam_policy.lambda_dynamodb_policy.arn
+  role       = aws_iam_role.role_for_lambda.name
+}
+
+output "arn" {
+  value = aws_iam_role.role_for_lambda.arn
+}
+
+output "role_name" {
+  value = aws_iam_role.role_for_lambda.name
+}

modules/iam/role/variables.tf

@@ -0,0 +1,20 @@
+variable "role_name" {
+  type = string
+  default = "amdocs-example-role"
+}
+
+variable "account_id" {
+  type = string
+  default = "123456789012"
+}
+
+variable "s3_bucket" {
+  type = string
+  default = "amdocs-example-bucket"
+}
+
+variable "dynamodb_table_name" {
+  description = "The name of the table"
+  type        = string
+  default     = "example"
+}

modules/lambda/locals.tf

@@ -0,0 +1,6 @@
+locals { 
+  default_tags = {
+    Project = "my-project"
+    Owner   = "my-team"
+    }
+}

modules/lambda/main.tf

@@ -0,0 +1,60 @@
+resource "aws_lambda_function" "lambda_function" {
+  filename      = "${path.module}/files/${var.filename}"
+  function_name = var.lambda_function_name
+  role          = var.lambda_function_role
+  handler       = var.lambda_function_handler
+  runtime       = var.lambda_function_runtime
+  timeout       = var.lambda_function_timeout
+  memory_size   = var.lambda_function_memory_size
+
+  environment {
+    variables = var.lambda_function_environment_variables
+  }
+
+  vpc_config {
+    subnet_ids         = var.lambda_function_subnet_ids
+    security_group_ids = var.lambda_function_security_group_ids
+  }
+
+  tags = merge(local.default_tags, var.tags, { "Project" = var.project }, { "Environment" = var.environment })
+}
+
+resource "aws_cloudwatch_log_group" "lambda_function_log_group" {
+  name              = "/aws/lambda/${var.lambda_function_name}"
+  retention_in_days = 14
+
+  tags = merge(local.default_tags, var.tags, { "Project" = var.project }, { "Environment" = var.environment })
+}
+
+resource "aws_lambda_permission" "s3_trigger" {
+  statement_id  = "AllowExecutionFromS3"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.lambda_function.arn
+  principal     = "s3.amazonaws.com"
+
+  source_arn = var.s3_arn
+}
+
+resource "aws_lambda_permission" "cloudwatch_logs" {
+  statement_id  = "AllowExecutionFromCloudWatch"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.lambda_function.arn
+  principal     = "logs.amazonaws.com"
+
+  source_arn = aws_cloudwatch_log_group.lambda_function_log_group.arn
+}
+
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  bucket = var.s3_bucket 
+
+  lambda_function {
+    lambda_function_arn = aws_lambda_function.lambda_function.arn
+    events              = ["s3:ObjectCreated:*"]
+    filter_prefix       = var.s3_bucket_prefix
+  }
+}
+
+output "lambda_function_arn" {
+  value       = aws_lambda_function.lambda_function.arn
+  description = "ARN of the Lambda function"
+}

modules/lambda/variables.tf

@@ -0,0 +1,79 @@
+variable "lambda_function_name" {
+  type = string
+  default = "s3_to_dynamodb"
+}
+
+variable "filename" {
+  type = string
+  default = "lambda_function.zip"
+}
+
+variable "lambda_function_role" {
+  type = string
+  default = ""
+}
+
+variable "lambda_function_handler" {
+  type = string
+  default = "index.handler"
+}
+
+variable "lambda_function_runtime" {
+  type = string
+  default = "nodejs14.x"
+}
+
+variable "lambda_function_timeout" {
+  type = number
+  default = 60
+}
+
+variable "lambda_function_memory_size" {
+  type = number
+  default = 128
+}
+
+variable "lambda_function_environment_variables" {
+  type = map
+  default = {}
+}
+
+variable "lambda_function_subnet_ids" {
+  type = list
+  default = []
+}
+
+variable "lambda_function_security_group_ids" {
+  type = list
+  default = []
+}
+
+variable "tags" {
+  type = map
+  default = {}
+}
+
+variable "project" {
+  type = string
+  default = "s3_to_dynamodb"
+}
+
+variable "s3_bucket_prefix" {
+  type = string
+  default = "files/"
+}
+
+variable "environment" {
+  type = string
+  default = "dev"
+}
+
+variable "s3_bucket" {
+  type = string
+  default = "s3-to-dynamodb"
+}
+
+variable "s3_arn" {
+  type = string
+  default = ""
+}

modules/s3/locals.tf

@@ -0,0 +1,6 @@
+locals { 
+  default_tags = {
+    Project = "my-project"
+    Owner   = "my-team"
+    }
+}

modules/s3/main.tf

@@ -0,0 +1,19 @@
+resource "aws_s3_bucket" "bucket" {
+  bucket = var.s3_bucket
+  
+  tags = merge(local.default_tags, var.tags, { "Project" = var.project }, { "Environment" = var.environment })
+}
+
+
+resource "aws_s3_object" "files_folder" {
+  bucket = aws_s3_bucket.bucket.id
+  key    = "files/"
+}
+
+output "s3_bucket_name" {
+  value = aws_s3_bucket.bucket.id
+}
+
+output "s3_bucket_arn" {
+  value = aws_s3_bucket.bucket.arn
+}

modules/s3/variables.tf

@@ -0,0 +1,19 @@
+variable "s3_bucket" {
+  type = string
+  default = "amdocs-example-project"
+}
+
+variable "environment" {
+  type = string
+  default = "dev"
+}
+
+variable "project" {
+  type = string
+  default = "amdocs-example"
+}
+
+variable "tags" {
+  type = map
+  default = {}
+}

variables.tf

@@ -0,0 +1,18 @@
+variable "regoin" {
+  description = "AWS regoin"
+  default = "us-east-1"
+}
+
+variable "s3_bucket" {
+  description = "value of s3 bucket name"
+  default = "amdocs-example-project"
+}
+
+variable "project" {
+  description = "value of project name"
+  default = "s3_to_dynamodb"
+}
+
+variable "dynamodb_table_name" {
+  default = "example_dynamo_table"
+}