varnitgoyal95/infra-provisioner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

INFRA-1227 | Sandeep Sogani | Adding dynamodb module

Browse Source
This commit is contained in:
sandeepsogani
2022-08-18 12:57:44 +05:30
parent 5c2bf03742
commit 4b5de02553
9 changed files with 593 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

486
bindata/bindata.go
View File

File diff suppressed because one or more lines are too long

12
main.go
View File

@@ -221,6 +221,18 @@ func main() {
return nil
},
},
{
Name: "dynamo-db",
Usage: "Provision dynamo db tables",
Action: func(c *cli.Context) error {
err := executeCommand(c, "dynamodb", "dynamo-db-tf")
if err != nil {
log.Fatalf("\nErr: %v", err)
return err
}
return nil
},
},
{
Name: "all",
Usage: "Provision all resources",

7
main_test.go
View File

@@ -138,3 +138,10 @@ func TestTemplates_DocumentDb_CompareWithOutput(t *testing.T) {
t.Error(err)
}
}
func TestTemplates_DynamoDb_CompareWithOutput(t *testing.T) {
err := CompareResourceWithOutput("dynamo-db-tf", "dynamodb")
if err != nil {
t.Error(err)
}
}

68
sample_infra_manifest.json
View File

@@ -111,7 +111,73 @@
"awsInstanceClass": "cache.t3.small",
"instanceName": "smaple-redis-cache",
"multiAZ": true
}
},
"dynamodb": {
"tables": [
{
"table_name": "message",
"read_capacity": 15,
"read_target_value": 70,
"min_read_capacity": 35,
"max_read_capacity": 100,
"write_capacity": 35,
"write_target_value": 70,
"min_write_capacity": 15,
"max_write_capacity": 100,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": "subBatchId",
"range_key": "messageId",
"attributes": [
{
"name": "messageId",
"type": "S"
},
{
"name": "subBatchId",
"type": "S"
}
],
"tags": {
"Name": "message",
"Product": "sa",
"Team": "CRM",
"Owner": "crm",
"Environment": "nonprod"
}
},
{
"table_name": "batch",
"read_capacity": 15,
"read_target_value": 70,
"min_read_capacity": 35,
"max_read_capacity": 100,
"write_capacity": 35,
"write_target_value": 70,
"min_write_capacity": 15,
"max_write_capacity": 100,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": "batchId",
"range_key": "subBatchId",
"attributes": [
{
"name": "batchId",
"type": "S"
},
{
"name": "subBatchId",
"type": "N"
}
],
"tags": {
"Name": "batch",
"Product": "sa",
"Team": "CRM",
"Owner": "crm",
"Environment": "nonprod"
}
}
]
}
},
"team": {
"name": "Infra"

18
templates/dynamo-db-tf/deploy.sh Normal file
View File

@@ -0,0 +1,18 @@
#!/bin/sh
set -e
additional_terraform_options=""
{{ if .Actions.Plan }} terraform_action=plan {{ else if .Actions.Destroy}} terraform_action=destroy {{ else }} terraform_action=apply {{ end }}
{{- if and .Actions.Plan .Actions.Destroy }}
additional_terraform_options="${additional_terraform_options} -destroy"
{{- else if or .Actions.Destroy .Actions.Apply }}
additional_terraform_options="${additional_terraform_options} -auto-approve"
{{- end }}
terraform init
terraform workspace select {{ .ExtraResources.Workspace }} || terraform workspace new {{ .ExtraResources.Workspace }}
terraform $terraform_action $additional_terraform_options

13
testdata/m1_basic/expected_output/dynamo-db-tf/deploy.sh vendored Executable file
View File

@@ -0,0 +1,13 @@
#!/bin/sh
set -e
additional_terraform_options=""
terraform_action=apply
additional_terraform_options="${additional_terraform_options} -auto-approve"
terraform init
terraform workspace select nonprod.np.navi-tech.in || terraform workspace new nonprod.np.navi-tech.in
terraform $terraform_action $additional_terraform_options

24
testdata/m1_basic/expected_output/dynamo-db-tf/main.tf vendored Executable file
View File

@@ -0,0 +1,24 @@
terraform {
backend "s3" {
bucket = "navi-bank-terraform-command-state"
region = "ap-south-1"
key = "dynamodb"
workspace_key_prefix = "dynamodb-states/lending/dev/foo"
profile = "cmd"
acl = "bucket-owner-full-control"
}
}
module "dynamodb" {
source = "git@github.cmd.navi-tech.in:navi-infra/dynamo-db-module.git"
environment = "dev"
infra_vertical = "lending"
tags = {
Team = "Infra"
Owner = "lending"
Product = "lending"
Environment = "dev"
Name = "foo"
}
tables = [{"table_name":"message","read_capacity":15,"read_target_value":70,"min_read_capacity":35,"max_read_capacity":100,"write_capacity":35,"write_target_value":70,"min_write_capacity":15,"max_write_capacity":100,"billing_mode":"PAY_PER_REQUEST","hash_key":"subBatchId","range_key":"messageId","attributes":[{"name":"messageId","type":"S"},{"name":"subBatchId","type":"S"}]},{"table_name":"batch","read_capacity":15,"read_target_value":70,"min_read_capacity":35,"max_read_capacity":100,"write_capacity":35,"write_target_value":70,"min_write_capacity":15,"max_write_capacity":100,"billing_mode":"PAY_PER_REQUEST","hash_key":"batchId","range_key":"subBatchId","attributes":[{"name":"batchId","type":"S"},{"name":"subBatchId","type":"N"}]}]
}

66
testdata/m1_basic/sample_infra_manifest.json vendored
View File

@@ -104,6 +104,72 @@
"volumeUsageTooHigh": 107374,
"freeMemoryTooLowInMB": 200
}
},
"dynamodb": {
"tables": [
{
"table_name": "message",
"read_capacity": 15,
"read_target_value": 70,
"min_read_capacity": 35,
"max_read_capacity": 100,
"write_capacity": 35,
"write_target_value": 70,
"min_write_capacity": 15,
"max_write_capacity": 100,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": "subBatchId",
"range_key": "messageId",
"attributes": [
{
"name": "messageId",
"type": "S"
},
{
"name": "subBatchId",
"type": "S"
}
],
"tags": {
"Name": "message",
"Product": "sa",
"Team": "CRM",
"Owner": "crm",
"Environment": "nonprod"
}
},
{
"table_name": "batch",
"read_capacity": 15,
"read_target_value": 70,
"min_read_capacity": 35,
"max_read_capacity": 100,
"write_capacity": 35,
"write_target_value": 70,
"min_write_capacity": 15,
"max_write_capacity": 100,
"billing_mode": "PAY_PER_REQUEST",
"hash_key": "batchId",
"range_key": "subBatchId",
"attributes": [
{
"name": "batchId",
"type": "S"
},
{
"name": "subBatchId",
"type": "N"
}
],
"tags": {
"Name": "batch",
"Product": "sa",
"Team": "CRM",
"Owner": "crm",
"Environment": "nonprod"
}
}
]
}
},
"team": {

128
types.go
View File

@@ -24,20 +24,48 @@ type Manifest struct {
Name string `json:"name"`
NameSuffix string
StateStoreBackend *StateStoreBackend
InfraVertical string `json:"infraVertical"`
InfraVertical string `json:"infraVertical"`
Actions *Actions
}
type ExtraResources struct {
//TODO: Remove environment from ExtraResources once all apps have migrated to new schema
Environment string `json:"environment"`
Workspace string `json:"workspace"`
Database *Database `json:"database" module:"rds" moduleDir:"rds-tf"`
Environment string `json:"environment"`
Workspace string `json:"workspace"`
Database *Database `json:"database" module:"rds" moduleDir:"rds-tf"`
// TBD: remove undersocre from json key. to keep naming conventions same across board
ServiceRole *ServiceRole `json:"aws_access" module:"roles" moduleDir:"aws-roles-tf"`
S3Buckets []S3Bucket `json:"s3_buckets" module:"s3" moduleDir:"aws-s3-bucket-tf"`
ElasticCache *ElasticCache `json:"elasticCache" module:"elasticCache" moduleDir:"elastic-cache-tf"`
DocDb *DocDb `json:"docdb" module:"docdb" moduleDir:"document-db-tf"`
ServiceRole *ServiceRole `json:"aws_access" module:"roles" moduleDir:"aws-roles-tf"`
S3Buckets []S3Bucket `json:"s3_buckets" module:"s3" moduleDir:"aws-s3-bucket-tf"`
ElasticCache *ElasticCache `json:"elasticCache" module:"elasticCache" moduleDir:"elastic-cache-tf"`
DocDb *DocDb `json:"docdb" module:"docdb" moduleDir:"document-db-tf"`
Dynamodb *Dynamodb `json:"dynamodb" module:"dynamodb" moduleDir:"dynamo-db-tf"`
}
type Dynamodb struct {
Tables []DynamodbTables `json:"tables"`
}
type DynamodbTables struct {
TableName string `json:"table_name"`
ReadCapacity int `json:"read_capacity"`
ReadTargetValue int `json:"read_target_value"`
MinReadCapacity int `json:"min_read_capacity"`
MaxReadCapacity int `json:"max_read_capacity"`
WriteCapacity int `json:"write_capacity"`
WriteTargetValue int `json:"write_target_value"`
MinWriteCapacity int `json:"min_write_capacity"`
MaxWriteCapacity int `json:"max_write_capacity"`
BillingMode string `json:"billing_mode"`
HashKey string `json:"hash_key"`
RangeKey string `json:"range_key"`
Attributes []struct {
Name string `json:"name"`
Type string `json:"type"`
} `json:"attributes"`
Ttl []struct {
AttributeName string `json:"attribute_name"`
Enabled bool `json:"enabled"`
}
}
//We provide defaults in respective terraforms instead of here to keep all values at one place
@@ -76,36 +104,36 @@ type ElasticCache struct {
}
type DocDb struct {
AwsInstanceClass string `json:"awsInstanceClass"`
InstanceName string `json:"instanceName" valid:"required"`
ApplyImmediately bool `json:"applyImmediately"`
MasterUser string `json:"masterUser" valid:"required"`
MasterPassword string `json:"masterPassword" valid:"required"`
Parameters []Parameter `json:"parameters"`
KmsKeyId string `json:"kmsKeyId"`
ClusterSize int `json:"clusterSize"`
DocDBAlertThresholds *DocDBAlertThresholds `json:"docDBAlertThresholds"`
DocDBAlertDurations *DocDBAlertDurations `json:"docDBAlertDurations"`
AwsInstanceClass string `json:"awsInstanceClass"`
InstanceName string `json:"instanceName" valid:"required"`
ApplyImmediately bool `json:"applyImmediately"`
MasterUser string `json:"masterUser" valid:"required"`
MasterPassword string `json:"masterPassword" valid:"required"`
Parameters []Parameter `json:"parameters"`
KmsKeyId string `json:"kmsKeyId"`
ClusterSize int `json:"clusterSize"`
DocDBAlertThresholds *DocDBAlertThresholds `json:"docDBAlertThresholds"`
DocDBAlertDurations *DocDBAlertDurations `json:"docDBAlertDurations"`
}
type DocDBAlertThresholds struct {
ReadLatency float64 `json:"readLatency"`
WriteLatency float64 `json:"writeLatency"`
DBConnections int `json:"dbConnections"`
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
VolumeUsageTooHigh int `json:"volumeUsageTooHigh"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
ReadLatency float64 `json:"readLatency"`
WriteLatency float64 `json:"writeLatency"`
DBConnections int `json:"dbConnections"`
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
VolumeUsageTooHigh int `json:"volumeUsageTooHigh"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
}
type DocDBAlertDurations struct {
ReadLatency int `json:"readLatency"`
WriteLatency int `json:"writeLatency"`
DBConnections int `json:"dbConnections"`
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
VolumeUsageTooHigh int `json:"volumeUsageTooHigh"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
ReadLatency int `json:"readLatency"`
WriteLatency int `json:"writeLatency"`
DBConnections int `json:"dbConnections"`
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
VolumeUsageTooHigh int `json:"volumeUsageTooHigh"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
}
type Parameter struct {
@@ -144,27 +172,27 @@ type S3Bucket struct {
}
type RdsAlertThresholds struct {
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
BurstBalance int `json:"burstBalance"`
DBConnections int `json:"dbConnections"`
QueueDepth int `json:"queueDepth"`
FreeStorageSpacePercent int `json:"freeStorageSpacePercent"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
ReadLatency float64 `json:"readLatency"`
WriteLatency float64 `json:"writeLatency"`
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
BurstBalance int `json:"burstBalance"`
DBConnections int `json:"dbConnections"`
QueueDepth int `json:"queueDepth"`
FreeStorageSpacePercent int `json:"freeStorageSpacePercent"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
ReadLatency float64 `json:"readLatency"`
WriteLatency float64 `json:"writeLatency"`
}
type RdsAlertDurations struct {
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
BurstBalance int `json:"burstBalance"`
DBConnections int `json:"dbConnections"`
QueueDepth int `json:"queueDepth"`
FreeStorageSpacePercent int `json:"freeStorageSpacePercent"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
ReadLatency int `json:"readLatency"`
WriteLatency int `json:"writeLatency"`
CpuUtilization int `json:"cpuUtilization"`
CpuCreditBalance int `json:"cpuCreditBalance"`
BurstBalance int `json:"burstBalance"`
DBConnections int `json:"dbConnections"`
QueueDepth int `json:"queueDepth"`
FreeStorageSpacePercent int `json:"freeStorageSpacePercent"`
FreeMemoryTooLowInMB int `json:"freeMemoryTooLowInMB"`
ReadLatency int `json:"readLatency"`
WriteLatency int `json:"writeLatency"`
}
type LifecycleRule struct {