INFRA-1591 | Sandeep Sogani | Refactor test cases to add multiple manifests testing

main_test.go

@@ -16,8 +16,6 @@ import (
 const ActualOutputDir = "actual_output"
 const TestDataDir = "testdata"
 const ExpectedOutputDir = "expected_output"
-const ManifestFile = "sample_infra_manifest.json"
-const ManifestFile2 = "sample_infra_manifest_2.json"
 const S3TfPath = "aws-s3-bucket-tf"
 const RdsTfPath = "rds-tf"
 const AwsRoleTfPath = "aws-roles-tf"
@@ -25,6 +23,8 @@ const ElasticCacheTfPath = "elastic-cache-tf"
 const DynamoDbTfPath = "dynamo-db-tf"
 const DocumentDbTfPath = "document-db-tf"
 
+var ManifestFiles = [2]string{"sample_infra_manifest", "sample_infra_manifest_2"}
+
 var testActions Actions
 
 func textDiff(text1, text2 string) string {
@@ -45,16 +45,16 @@ func CompareResourceWithOutput(resouceDir string, resource string, manifestFileN
 	}
 	for _, dir := range dirs {
 		fmt.Print(dir.Name())
-		manifest, err := parseManifest(filepath.Join(TestDataDir, dir.Name(), manifestFileName), testActions)
+		manifest, err := parseManifest(filepath.Join(TestDataDir, manifestFileName+".json"), testActions)
 		if err != nil {
 			return err
 		}
 
-		err = templateResourceTf(resource, resouceDir, manifest, filepath.Join(TestDataDir, dir.Name(), ActualOutputDir, expectedOutPath))
+		err = templateResourceTf(resource, resouceDir, manifest, filepath.Join(TestDataDir, ActualOutputDir, manifestFileName, expectedOutPath))
 		if err != nil {
 			return err
 		}
-		err = filepath.Walk(filepath.Join(TestDataDir, dir.Name(), ExpectedOutputDir, expectedOutPath),
+		err = filepath.Walk(filepath.Join(TestDataDir, ExpectedOutputDir, manifestFileName, expectedOutPath),
 			func(path string, fileInfo os.FileInfo, err error) error {
 				if err != nil {
 					return err
@@ -66,7 +66,7 @@ func CompareResourceWithOutput(resouceDir string, resource string, manifestFileN
 				if err != nil {
 					return err
 				}
-				actualOutput, err := ioutil.ReadFile(filepath.Join(TestDataDir, dir.Name(), ActualOutputDir, expectedOutPath, filepath.Base(path)))
+				actualOutput, err := ioutil.ReadFile(filepath.Join(TestDataDir, ActualOutputDir, manifestFileName, expectedOutPath, filepath.Base(path)))
 				if err != nil {
 					return err
 				}
@@ -112,50 +112,55 @@ func TestBinData_CompareWithTemplates(t *testing.T) {
 }
 
 func TestTemplates_Rds_CompareWithOutput(t *testing.T) {
-	err := CompareResourceWithOutput(RdsTfPath, "rds", ManifestFile, RdsTfPath)
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestTemplates_Rds_CompareWithOutputV2(t *testing.T) {
-	err := CompareResourceWithOutput(RdsTfPath, "rds", ManifestFile2, "rds2-tf")
-	if err != nil {
-		t.Error(err)
+	for _, ManifestFile := range ManifestFiles {
+		err := CompareResourceWithOutput(RdsTfPath, "rds", ManifestFile, RdsTfPath)
+		if err != nil {
+			t.Error(err)
+		}
 	}
 }
 
 func TestTemplates_S3_CompareWithOutput(t *testing.T) {
-	err := CompareResourceWithOutput(S3TfPath, "s3-bucket", ManifestFile, S3TfPath)
-	if err != nil {
-		t.Error(err)
+	for _, ManifestFile := range ManifestFiles {
+		err := CompareResourceWithOutput(S3TfPath, "s3-bucket", ManifestFile, S3TfPath)
+		if err != nil {
+			t.Error(err)
+		}
 	}
 }
 
 func TestTemplates_AwsRole_CompareWithOutput(t *testing.T) {
-	err := CompareResourceWithOutput(AwsRoleTfPath, "iam-role", ManifestFile, AwsRoleTfPath)
-	if err != nil {
-		t.Error(err)
+	for _, ManifestFile := range ManifestFiles {
+		err := CompareResourceWithOutput(AwsRoleTfPath, "iam-role", ManifestFile, AwsRoleTfPath)
+		if err != nil {
+			t.Error(err)
+		}
 	}
 }
 
 func TestTemplates_ElasticCache_CompareWithOutput(t *testing.T) {
-	err := CompareResourceWithOutput(ElasticCacheTfPath, "elasticCache", ManifestFile, ElasticCacheTfPath)
-	if err != nil {
-		t.Error(err)
+	for _, ManifestFile := range ManifestFiles {
+		err := CompareResourceWithOutput(ElasticCacheTfPath, "elasticCache", ManifestFile, ElasticCacheTfPath)
+		if err != nil {
+			t.Error(err)
+		}
 	}
 }
 
 func TestTemplates_DocumentDb_CompareWithOutput(t *testing.T) {
-	err := CompareResourceWithOutput(DocumentDbTfPath, "docdb", ManifestFile, DocumentDbTfPath)
-	if err != nil {
-		t.Error(err)
+	for _, ManifestFile := range ManifestFiles {
+		err := CompareResourceWithOutput(DocumentDbTfPath, "docdb", ManifestFile, DocumentDbTfPath)
+		if err != nil {
+			t.Error(err)
+		}
 	}
 }
 
 func TestTemplates_DynamoDb_CompareWithOutput(t *testing.T) {
-	err := CompareResourceWithOutput(DynamoDbTfPath, "dynamodb", ManifestFile, DynamoDbTfPath)
-	if err != nil {
-		t.Error(err)
+	for _, ManifestFile := range ManifestFiles {
+		err := CompareResourceWithOutput(DynamoDbTfPath, "dynamodb", ManifestFile, DynamoDbTfPath)
+		if err != nil {
+			t.Error(err)
+		}
 	}
 }

testdata/expected_output/sample_infra_manifest_2/aws-roles-tf/deploy.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# exit when any command fails
+set -e
+
+additional_terraform_options=""
+additional_kube_options=""
+
+ terraform_action=apply  
+additional_terraform_options="${additional_terraform_options} -auto-approve"
+
+
+terraform init
+terraform workspace select aps1.np.navi-gi.in || terraform workspace new aps1.np.navi-gi.in
+
+providers=$(terraform providers | grep 'registry.terraform.io/-' | awk -F "[" '{print $2}' | sed 's/.$//')
+printf '\n'
+for provider in $providers; do
+  correct_provider="${provider//-/hashicorp}"
+  echo "executing"
+  echo "terraform state replace-provider -auto-approve $provider $correct_provider"
+  terraform state replace-provider -auto-approve "$provider" "$correct_provider"
+done
+printf '\n'
+
+terraform $terraform_action $additional_terraform_options
+  kubectl config use-context ${CLUSTER}
+  kubectl apply -f foo-navi-service-dev.yaml -n dev-3p $additional_kube_options
+  

testdata/expected_output/sample_infra_manifest_2/aws-roles-tf/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  backend "s3" {
+    bucket               = "navi-bank-terraform-command-state"
+    region               = "ap-south-1"
+    key                  = "service-iam-roles"
+    workspace_key_prefix = "iamroles/insurance/dev/foo-navi-service"
+    profile              = "cmd"
+    acl                  = "bucket-owner-full-control"
+  }
+}
+
+module "iam-role" {
+  source        = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/iam-roles.git"
+  environment   = "dev"
+  service_role  = {"policies":[{"actions":["s3:GetObject","s3:PutObject"],"resource":"arn:aws:s3:::navi-e3e2a9bfd88566b05001b02a3f51d286/*"},{"actions":["s3:GetObject","s3:PutObject"],"resource":"*"},{"actions":["sns:Publish","sns:SetSMSAttributes"],"resource":"arn:aws:s3:::arn:aws:s3:::test-bucket-to-be-deleted/*"}]}
+  namespace     = "dev-3p"
+  role_name     = "foo-navi-service"
+  infra_vertical = "insurance"
+  tags = {
+    Team = "Infra"
+    Owner = "gi"
+    Name = "foo"
+    Environment   = "dev"
+  }
+}

testdata/expected_output/sample_infra_manifest_2/aws-s3-bucket-tf/deploy.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# exit when any command fails
+set -e
+
+additional_terraform_options=""
+
+ terraform_action=apply  
+additional_terraform_options="${additional_terraform_options} -auto-approve"
+
+
+terraform init
+terraform workspace select aps1.np.navi-gi.in || terraform workspace new aps1.np.navi-gi.in
+
+providers=$(terraform providers | grep 'registry.terraform.io/-' | awk -F "[" '{print $2}' | sed 's/.$//')
+printf '\n'
+for provider in $providers; do
+  correct_provider="${provider//-/hashicorp}"
+  echo "executing"
+  echo "terraform state replace-provider -auto-approve $provider $correct_provider"
+  terraform state replace-provider -auto-approve "$provider" "$correct_provider"
+done
+printf '\n'
+
+terraform $terraform_action $additional_terraform_options

testdata/expected_output/sample_infra_manifest_2/aws-s3-bucket-tf/main.tf

@@ -0,0 +1,24 @@
+terraform {
+  backend "s3" {
+    bucket               = "navi-bank-terraform-command-state"
+    region               = "ap-south-1"
+    key                  = "s3-buckets"
+    workspace_key_prefix = "s3-buckets/insurance/dev/foo-navi-service"
+    profile              = "cmd"
+    acl                  = "bucket-owner-full-control"
+  }
+}
+
+module "s3-buckets" {
+  source        = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/aws-s3-bucket.git"
+  s3_buckets  = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["*"]}],"bucketPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}"},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["https://go-nlc.com"]}]}]
+  environment   = "dev"
+  infra_vertical = "insurance"
+  bucket_tags = {
+    Team          = "Infra"
+    Owner = "gi"
+    Product = "gi"
+    Name = "foo"
+    Environment   = "dev"
+  }
+}

testdata/expected_output/sample_infra_manifest_2/document-db-tf/deploy.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+
+additional_terraform_options=""
+
+ terraform_action=apply  
+additional_terraform_options="${additional_terraform_options} -auto-approve"
+
+
+terraform init
+terraform workspace select aps1.np.navi-gi.in || terraform workspace new aps1.np.navi-gi.in
+
+providers=$(terraform providers | grep 'registry.terraform.io/-' | awk -F "[" '{print $2}' | sed 's/.$//')
+printf '\n'
+for provider in $providers; do
+  correct_provider="${provider//-/hashicorp}"
+  echo "executing"
+  echo "terraform state replace-provider -auto-approve $provider $correct_provider"
+  terraform state replace-provider -auto-approve "$provider" "$correct_provider"
+done
+printf '\n'
+
+terraform $terraform_action $additional_terraform_options

testdata/expected_output/sample_infra_manifest_2/document-db-tf/main.tf

@@ -0,0 +1,47 @@
+terraform {
+  backend "s3" {
+    bucket               = "navi-bank-terraform-command-state"
+    region               = "ap-south-1"
+    key                  = "document-db"
+    workspace_key_prefix = "docdb-states/insurance/dev/docdb"
+    profile              = "cmd"
+    acl                  = "bucket-owner-full-control"
+  }
+}
+
+module "docdb" {
+  source           = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/doc-db-module.git"
+  environment      = "dev"
+  name_prefix      = "docdb"
+  infra_vertical   = "insurance"
+  team             = "Infra"
+  master_username  = "foo_service_user"
+  master_password  = "foo_service_pass"
+  cluster_size     = 1
+
+  tags = {
+    Team        = "Infra"
+    Owner       = "gi"
+    Product     = "gi"
+    Environment = "dev"
+    Name        = "foo"
+  }
+
+  instance_class    = "db.t4g.medium"
+  apply_immediately = false
+  cpu_utilization_alarm_threshold = "75"
+  cpu_utilization_alarm_evaluation_period = "15"
+  cpucredit_balance_alarm_threshold = "120"
+  cpucredit_balance_alarm_evaluation_period = "5"
+  volume_usage_alarm_threshold = "107374"
+  volume_usage_alarm_evaluation_period = "10"
+  freeable_memory_alarm_threshold = "200"
+  freeable_memory_alarm_evaluation_period = "10"
+  db_connections_alarm_threshold = "200"
+  db_connections_evaluation_period = "10"
+  read_latency_alarm_threshold = "0.5"
+  read_latency_alarm_evaluation_period = "5"
+  write_latency_alarm_threshold = "0.5"
+  write_latency_alarm_evaluation_period = "5"
+
+}

testdata/expected_output/sample_infra_manifest_2/dynamo-db-tf/deploy.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+additional_terraform_options=""
+
+ terraform_action=apply  
+additional_terraform_options="${additional_terraform_options} -auto-approve"
+
+
+terraform init
+terraform workspace select aps1.np.navi-gi.in || terraform workspace new aps1.np.navi-gi.in
+terraform $terraform_action $additional_terraform_options

testdata/expected_output/sample_infra_manifest_2/dynamo-db-tf/main.tf

@@ -0,0 +1,67 @@
+terraform {
+  backend "s3" {
+    bucket               = "navi-bank-terraform-command-state"
+    region               = "ap-south-1"
+    key                  = "dynamodb"
+    workspace_key_prefix = "dynamodb-states/insurance/dev/foo"
+    profile              = "cmd"
+    acl                  = "bucket-owner-full-control"
+  }
+}
+
+module "dynamodb" {
+  source         = "git@github.cmd.navi-tech.in:navi-infra/dynamo-db-module.git"
+  environment    = "dev"
+  infra_vertical = "insurance"
+  tags           = {
+    Team        = "Infra"
+    Owner       = "gi"
+    Product     = "gi"
+    Environment = "dev"
+    Name        = "foo"
+  }
+tables = [
+  {
+    table_name = "foo_dev_abc"
+    min_read_capacity = "5"
+    min_write_capacity = "5"
+    read_target_value = "85"
+    write_target_value = "85"
+    max_read_capacity = "20"
+    max_write_capacity = "20"
+    billing_mode = "PROVISIONED"
+    hash_key = "id"
+    range_key = "s"
+    ttl = {
+      attribute_name = "ttl"
+      enabled = "true"
+    }
+    attributes = [
+      {
+        name = "id"
+        type = "N"
+        },
+      {
+        name = "s"
+        type = "S"
+        },
+      ]
+      lsi = [
+        {
+          name = "test1"
+          range_key = "s"
+          projection_type = "KEYS_ONLY"
+        },
+      ]
+      gsi = [
+        {
+          name = "test"
+          hash_key = "id"
+          read_capacity = 5
+          write_capacity = 5
+          projection_type = "KEYS_ONLY"
+        },
+      ]
+    },
+  ]
+}

testdata/expected_output/sample_infra_manifest_2/elastic-cache-tf/deploy.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -e
+
+additional_terraform_options=""
+
+ terraform_action=apply  
+additional_terraform_options="${additional_terraform_options} -auto-approve"
+
+
+terraform init
+terraform workspace select aps1.np.navi-gi.in || terraform workspace new aps1.np.navi-gi.in
+
+providers=$(terraform providers | grep 'registry.terraform.io/-' | awk -F "[" '{print $2}' | sed 's/.$//')
+printf '\n'
+for provider in $providers; do
+  correct_provider="${provider//-/hashicorp}"
+  echo "executing"
+  echo "terraform state replace-provider -auto-approve $provider $correct_provider"
+  terraform state replace-provider -auto-approve "$provider" "$correct_provider"
+done
+printf '\n'
+
+
+terraform $terraform_action  $additional_terraform_options

testdata/expected_output/sample_infra_manifest_2/elastic-cache-tf/main.tf

@@ -0,0 +1,41 @@
+terraform {
+  backend "s3" {
+    bucket               = "navi-bank-terraform-command-state"
+    region               = "ap-south-1"
+    key                  = "elastic-cache"
+    workspace_key_prefix = "elastic-cache-states/insurance/dev/smaple-redis-cache"
+    profile              = "cmd"
+    acl                  = "bucket-owner-full-control"
+  }
+}
+
+module "elastic_cache" {
+  source        = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/elastic-cache-module.git"
+  environment   = "dev"
+  name_prefix = "smaple-redis-cache"
+  infra_vertical = "insurance"
+  team = "Infra"
+
+  tags = {
+    Team = "Infra"
+    Owner = "gi"
+    Product = "gi"
+    Environment   = "dev"
+    Name         = "foo"
+  }
+
+  node_type = "cache.t3.small"
+  apply_immediately = false
+  automatic_failover_enabled = true
+  multi_az_enabled = true
+  number_cache_clusters = 2
+}
+
+output "elasticache_replication_group_primary_endpoint_address" {
+  value       = module.elastic_cache.elasticache_replication_group_primary_endpoint_address
+  description = "The address of the endpoint for the primary node in the replication group."
+}
+output "elasticache_replication_group_reader_endpoint_address" {
+  value       = module.elastic_cache.elasticache_replication_group_reader_endpoint_address
+  description = "The address of the endpoint for the reader node in the replication group."
+}