diff --git a/.gitignore b/.gitignore index 331c58f..751eb03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea -vendor \ No newline at end of file +vendor +actual_output/ diff --git a/go.mod b/go.mod index daabd79..3a00cfe 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/caarlos0/env/v6 v6.2.1 github.com/huandu/xstrings v1.2.1 // indirect github.com/imdario/mergo v0.3.8 // indirect + github.com/sergi/go-diff v1.1.0 github.com/urfave/cli/v2 v2.1.1 golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 // indirect ) diff --git a/go.sum b/go.sum index b9ffdfa..f9fb1c3 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,9 @@ github.com/huandu/xstrings v1.2.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= @@ -33,6 +36,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= @@ -55,5 +60,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..be03a78 --- /dev/null +++ b/main_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.cmd.navi-tech.in/navi-infra/infra-provisioner/bindata" + "github.com/sergi/go-diff/diffmatchpatch" +) + +const ActualOutputDir = "actual_output" +const TestDataDir = "testdata" +const ExpectedOutputDir = "expected_output" +const ManifestFile = "sample_infra_manifest.json" + +func textDiff(text1, text2 string) string { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(text1, text2, false) + return dmp.DiffPrettyText(diffs) +} + +func CompareResourceWithOutput(resouceDir string, resource string) error { + dirs, err := ioutil.ReadDir(TestDataDir) + if err != nil { + return err + } + for _, dir := range dirs { + fmt.Print(dir.Name()) + manifest, err := parseManifest(filepath.Join(TestDataDir, dir.Name(), ManifestFile)) + if err != nil { + return err + } + + err = templateResourceTf(resource, resouceDir, manifest, filepath.Join(TestDataDir, dir.Name(), ActualOutputDir, resouceDir)) + if err != nil { + return err + } + err = filepath.Walk(filepath.Join(TestDataDir, dir.Name(), ExpectedOutputDir, resouceDir), + func(path string, fileInfo os.FileInfo, err error) error { + if err != nil { + return err + } + if fileInfo.IsDir() { + return nil + } + expectedOutput, err := ioutil.ReadFile(path) + if err != nil { + return err + } + actualOutput, err := ioutil.ReadFile(filepath.Join(TestDataDir, dir.Name(), ActualOutputDir, resouceDir, filepath.Base(path))) + if err != nil { + return err + } + if bytes.Compare(expectedOutput, actualOutput) != 0 { + return fmt.Errorf("Mismatch for %s, diff: %s\n", path, textDiff(string(actualOutput), string(expectedOutput))) + } + return nil + }) + if err != nil { + return err + } + + } + return nil +} + +func TestBinData_CompareWithTemplates(t *testing.T) { + err := filepath.Walk(TemplatesDir, + func(path string, fileInfo os.FileInfo, err error) error { + if err != nil { + t.Error(err) + } + if fileInfo.IsDir() { + return nil + } + actualTemplate, err := ioutil.ReadFile(path) + if err != nil { + t.Error(err) + } + binDataTemplate, err := bindata.Asset(path) + if err != nil { + t.Error(err) + } + + if bytes.Compare(actualTemplate, binDataTemplate) != 0 { + t.Errorf("Found outdated bindata for %s", path) + } + return nil + }) + if err != nil { + t.Error(err) + } +} + +func TestTemplates_Rds_CompareWithOutput(t *testing.T) { + err := CompareResourceWithOutput("rds-tf", "rds") + if err != nil { + t.Error(err) + } +} + +func TestTemplates_S3_CompareWithOutput(t *testing.T) { + err := CompareResourceWithOutput("aws-s3-bucket-tf", "s3-bucket") + if err != nil { + t.Error(err) + } +} + +func TestTemplates_AwsRole_CompareWithOutput(t *testing.T) { + err := CompareResourceWithOutput("aws-roles-tf", "iam-role") + if err != nil { + t.Error(err) + } +} diff --git a/resource.go b/resource.go index 6d65776..b63276b 100644 --- a/resource.go +++ b/resource.go @@ -1,13 +1,14 @@ package main import ( - "github.cmd.navi-tech.in/navi-infra/infra-provisioner/bindata" "log" "os" "os/exec" "strings" "text/template" + "github.cmd.navi-tech.in/navi-infra/infra-provisioner/bindata" + "github.com/Masterminds/sprig/v3" ) @@ -15,7 +16,7 @@ const TemplatesDir = "templates" const InitScript = "./deploy.sh" func provisionResource(resourceName, resourceDir string, manifest *Manifest, templateOnly, plan bool) error { - err := templateResourceTf(resourceName, resourceDir, manifest) + err := templateResourceTf(resourceName, resourceDir, manifest, resourceDir) if err != nil { log.Fatalf("\nErr: %v", err) return err @@ -32,7 +33,7 @@ func provisionResource(resourceName, resourceDir string, manifest *Manifest, tem return nil } -func templateResourceTf(templateName, resourceDir string, manifest *Manifest) error { +func templateResourceTf(templateName, resourceDir string, manifest *Manifest, destinationDir string) error { log.Printf("Creating templates for %s in %s", templateName, resourceDir) tfFiles, err := bindata.AssetDir(strings.Join([]string{TemplatesDir, resourceDir}, "/")) @@ -44,7 +45,7 @@ func templateResourceTf(templateName, resourceDir string, manifest *Manifest) er tfBytes := bindata.MustAsset(strings.Join([]string{TemplatesDir, resourceDir, tfFile}, "/")) t := template.Must(template.New(templateName).Funcs(sprig.TxtFuncMap()).Parse(string(tfBytes))) - tfOut, err := createFile(resourceDir, tfFile) + tfOut, err := createFile(destinationDir, tfFile) if err != nil { log.Fatalf("\nErr: %v", err) return err diff --git a/testdata/m1_basic/expected_output/aws-roles-tf/deploy.sh b/testdata/m1_basic/expected_output/aws-roles-tf/deploy.sh new file mode 100755 index 0000000..4ef57c7 --- /dev/null +++ b/testdata/m1_basic/expected_output/aws-roles-tf/deploy.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# exit when any command fails +set -e + +additional_terraform_options="" +additional_kube_options="" + +terraform_action=${1:-apply} +if [ $terraform_action == "apply" ];then + additional_terraform_options="${additional_terraform_options} -auto-approve" +else + additional_kube_options="${additional_kube_options} --dry-run" +fi + +terraform init +terraform workspace select nonprod.np.navi-tech.in || terraform workspace new nonprod.np.navi-tech.in +terraform $terraform_action $additional_terraform_options + +kubectl config use-context ${CLUSTER} +kubectl apply -f foo-navi-service-dev.yaml -n ${NAMESPACE} $additional_kube_options diff --git a/testdata/m1_basic/expected_output/aws-roles-tf/main.tf b/testdata/m1_basic/expected_output/aws-roles-tf/main.tf new file mode 100755 index 0000000..8340437 --- /dev/null +++ b/testdata/m1_basic/expected_output/aws-roles-tf/main.tf @@ -0,0 +1,17 @@ +terraform { + backend "s3" { + bucket = "navi-bank-terraform-nonprod-state" + region = "ap-south-1" + key = "service-iam-roles" + workspace_key_prefix = "iamroles/dev/foo-navi-service" + profile = "nonprod" + acl = "bucket-owner-full-control" + } +} + +module "iam-role" { + source = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/iam-roles.git" + environment = "dev" + service_role = {"policies":[{"actions":["s3:GetObject","s3:PutObject"],"resource":"arn:aws:s3:::navi-e3e2a9bfd88566b05001b02a3f51d286/*"},{"actions":["s3:GetObject","s3:PutObject"],"resource":"*"},{"actions":["sns:Publish","sns:SetSMSAttributes"],"resource":"arn:aws:s3:::arn:aws:s3:::test-bucket-to-be-deleted/*"}]} + role_name = "foo-navi-service" +} diff --git a/testdata/m1_basic/expected_output/aws-s3-bucket-tf/deploy.sh b/testdata/m1_basic/expected_output/aws-s3-bucket-tf/deploy.sh new file mode 100755 index 0000000..739ff87 --- /dev/null +++ b/testdata/m1_basic/expected_output/aws-s3-bucket-tf/deploy.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# exit when any command fails +set -e + +additional_terraform_options="" +terraform_action=${1:-apply} + +if [ $terraform_action == "apply" ];then + additional_terraform_options="${additional_terraform_options} -auto-approve" +fi + +terraform init +terraform workspace select nonprod.np.navi-tech.in || terraform workspace new nonprod.np.navi-tech.in +terraform $terraform_action $additional_terraform_options diff --git a/testdata/m1_basic/expected_output/aws-s3-bucket-tf/main.tf b/testdata/m1_basic/expected_output/aws-s3-bucket-tf/main.tf new file mode 100755 index 0000000..fb34aea --- /dev/null +++ b/testdata/m1_basic/expected_output/aws-s3-bucket-tf/main.tf @@ -0,0 +1,16 @@ +terraform { + backend "s3" { + bucket = "navi-bank-terraform-nonprod-state" + region = "ap-south-1" + key = "s3-buckets" + workspace_key_prefix = "s3-buckets/dev/foo-navi-service" + profile = "nonprod" + acl = "bucket-owner-full-control" + } +} + +module "s3-buckets" { + source = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/aws-s3-bucket.git" + s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":null},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}]}] + environment = "dev" +} diff --git a/testdata/m1_basic/expected_output/rds-tf/deploy.sh b/testdata/m1_basic/expected_output/rds-tf/deploy.sh new file mode 100755 index 0000000..729b7aa --- /dev/null +++ b/testdata/m1_basic/expected_output/rds-tf/deploy.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +additional_terraform_options="" +terraform_action=${1:-apply} + +if [ $terraform_action == "apply" ];then + additional_terraform_options="${additional_terraform_options} -auto-approve" +fi + +terraform init +terraform workspace select nonprod.np.navi-tech.in || terraform workspace new nonprod.np.navi-tech.in +terraform $terraform_action -target=module.rds.data.aws_subnet_ids.command_private_subnets $additional_terraform_options +terraform $terraform_action -target=module.rds.module.rds_instance $additional_terraform_options +terraform $terraform_action -target=module.rds.module.rds_instance_replica $additional_terraform_options +terraform $terraform_action -target=module.rds.module.postgres_db $additional_terraform_options diff --git a/testdata/m1_basic/expected_output/rds-tf/main.tf b/testdata/m1_basic/expected_output/rds-tf/main.tf new file mode 100755 index 0000000..0324e67 --- /dev/null +++ b/testdata/m1_basic/expected_output/rds-tf/main.tf @@ -0,0 +1,57 @@ +terraform { + backend "s3" { + bucket = "navi-bank-terraform-command-state" + region = "ap-south-1" + key = "rds" + workspace_key_prefix = "rds-states/foo-service" + profile = "cmd" + acl = "bucket-owner-full-control" + } +} + +module "rds" { + source = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/rds.git" + environment = "dev" + instance_name = "foo-service" + password = "foo_service_password" + user = "foo_service_user" + name = "foo" + + monitoring_password = "" + monitoring_user = "" + + databases = ["foo_service"] + database_tags = { + Team = "Infra" + medici-app = "foo-navi-service" + medici-owner = "Infra" + medici-environment = "dev" + } + + team = "Infra" + size = 7 + db_extensions = ["pgcrypto"] + readonly_user = "foo_readonly_user" + readonly_password = "foo_readonly_password" + backup_retention_period = 7 + multi_az = true + read_replica_instance_class = "db.t3.micro" + read_replica_performance_insights_enabled = true + parameters = [ + { + name = "rds.logical_replication" + value = "1" + apply_method = "pending-reboot" + } + ] + cpu_utilization_alarm_threshold = "70" + cpucredit_balance_alarm_threshold = "120" + burst_balance_alarm_threshold = "85" + db_connections_alarm_threshold = "200" + queue_depth_alarm_threshold = "20" + free_storage_space_percent = 90 + freeable_memory_threshold = 150 + read_latency_alarm_threshold = "0.5" + write_latency_alarm_threshold = "0.5" + +} diff --git a/testdata/m1_basic/sample_infra_manifest.json b/testdata/m1_basic/sample_infra_manifest.json new file mode 100644 index 0000000..7a0be53 --- /dev/null +++ b/testdata/m1_basic/sample_infra_manifest.json @@ -0,0 +1,74 @@ +{ + "extraResources": { + "environment": "dev", + "database": { + "instanceName": "foo-service", + "user": "foo_service_user", + "password": "foo_service_password", + "sizeInGb": 7, + "dbNames": ["foo_service"], + "dbExtensions": ["pgcrypto"], + "readonlyUser": "foo_readonly_user", + "readonlyPassword": "foo_readonly_password", + "applyImmediately": false, + "performanceInsightsEnabled": false, + "readReplica": { + "awsInstanceClass": "db.t3.micro", + "performanceInsightsEnabled": true + }, + "parameters": { + "rds.logical_replication": "1" + }, + "rdsAlertThresholds": { + "cpuUtilization": 70, + "cpuCreditBalance": 120, + "burstBalance": 85, + "dbConnections": 200, + "queueDepth": 20, + "freeStorageSpacePercent": 90, + "freeMemoryTooLowInMB": 150, + "readLatency": 0.5, + "writeLatency": 0.5 + } + }, + "aws_access": { + "policies": [ + { + "actions": ["s3:GetObject", "s3:PutObject"], + "resource": "arn:aws:s3:::navi-e3e2a9bfd88566b05001b02a3f51d286/*" + }, + { + "actions": ["s3:GetObject", "s3:PutObject"], + "resource": "*" + }, + { + "resource": "arn:aws:s3:::arn:aws:s3:::test-bucket-to-be-deleted/*", + "actions": ["sns:Publish", "sns:SetSMSAttributes"] + } + ] + }, + "s3_buckets": [ + { + "anonymizedBucketName": "navi-bucket-test-1", + "bucketTag": "customer-uploads" + }, + { + "anonymizedBucketName": "navi-bucket-test-2", + "bucketTag": "document-uploads", + "lifecycleRules": [ + { + "expiration": { + "days": 1 + } + } + ] + } + ] + }, + "team": { + "name": "Infra" + }, + "deployment": { + "name": "foo" + } +}