varnitgoyal95/infra-provisioner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

INFRA-43| Deepak Jain| adding test

Browse Source
This commit is contained in:
Deepak Jain
2020-09-17 12:58:18 +05:30
parent e5223008f5
commit 07ec4c8cd6
12 changed files with 350 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -1,2 +1,3 @@
.idea
vendor
vendor
actual_output/

1
go.mod
View File

@@ -9,6 +9,7 @@ require (
github.com/caarlos0/env/v6 v6.2.1
github.com/huandu/xstrings v1.2.1 // indirect
github.com/imdario/mergo v0.3.8 // indirect
github.com/sergi/go-diff v1.1.0
github.com/urfave/cli/v2 v2.1.1
golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 // indirect
)

7
go.sum
View File

@@ -25,6 +25,9 @@ github.com/huandu/xstrings v1.2.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq
github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
@@ -33,6 +36,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
@@ -55,5 +60,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

118
main_test.go Normal file
View File

@@ -0,0 +1,118 @@
package main
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.cmd.navi-tech.in/navi-infra/infra-provisioner/bindata"
"github.com/sergi/go-diff/diffmatchpatch"
)
const ActualOutputDir = "actual_output"
const TestDataDir = "testdata"
const ExpectedOutputDir = "expected_output"
const ManifestFile = "sample_infra_manifest.json"
func textDiff(text1, text2 string) string {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(text1, text2, false)
return dmp.DiffPrettyText(diffs)
}
func CompareResourceWithOutput(resouceDir string, resource string) error {
dirs, err := ioutil.ReadDir(TestDataDir)
if err != nil {
return err
}
for _, dir := range dirs {
fmt.Print(dir.Name())
manifest, err := parseManifest(filepath.Join(TestDataDir, dir.Name(), ManifestFile))
if err != nil {
return err
}
err = templateResourceTf(resource, resouceDir, manifest, filepath.Join(TestDataDir, dir.Name(), ActualOutputDir, resouceDir))
if err != nil {
return err
}
err = filepath.Walk(filepath.Join(TestDataDir, dir.Name(), ExpectedOutputDir, resouceDir),
func(path string, fileInfo os.FileInfo, err error) error {
if err != nil {
return err
}
if fileInfo.IsDir() {
return nil
}
expectedOutput, err := ioutil.ReadFile(path)
if err != nil {
return err
}
actualOutput, err := ioutil.ReadFile(filepath.Join(TestDataDir, dir.Name(), ActualOutputDir, resouceDir, filepath.Base(path)))
if err != nil {
return err
}
if bytes.Compare(expectedOutput, actualOutput) != 0 {
return fmt.Errorf("Mismatch for %s, diff: %s\n", path, textDiff(string(actualOutput), string(expectedOutput)))
}
return nil
})
if err != nil {
return err
}
}
return nil
}
func TestBinData_CompareWithTemplates(t *testing.T) {
err := filepath.Walk(TemplatesDir,
func(path string, fileInfo os.FileInfo, err error) error {
if err != nil {
t.Error(err)
}
if fileInfo.IsDir() {
return nil
}
actualTemplate, err := ioutil.ReadFile(path)
if err != nil {
t.Error(err)
}
binDataTemplate, err := bindata.Asset(path)
if err != nil {
t.Error(err)
}
if bytes.Compare(actualTemplate, binDataTemplate) != 0 {
t.Errorf("Found outdated bindata for %s", path)
}
return nil
})
if err != nil {
t.Error(err)
}
}
func TestTemplates_Rds_CompareWithOutput(t *testing.T) {
err := CompareResourceWithOutput("rds-tf", "rds")
if err != nil {
t.Error(err)
}
}
func TestTemplates_S3_CompareWithOutput(t *testing.T) {
err := CompareResourceWithOutput("aws-s3-bucket-tf", "s3-bucket")
if err != nil {
t.Error(err)
}
}
func TestTemplates_AwsRole_CompareWithOutput(t *testing.T) {
err := CompareResourceWithOutput("aws-roles-tf", "iam-role")
if err != nil {
t.Error(err)
}
}

9
resource.go
View File

@@ -1,13 +1,14 @@
package main
import (
"github.cmd.navi-tech.in/navi-infra/infra-provisioner/bindata"
"log"
"os"
"os/exec"
"strings"
"text/template"
"github.cmd.navi-tech.in/navi-infra/infra-provisioner/bindata"
"github.com/Masterminds/sprig/v3"
)
@@ -15,7 +16,7 @@ const TemplatesDir = "templates"
const InitScript = "./deploy.sh"
func provisionResource(resourceName, resourceDir string, manifest *Manifest, templateOnly, plan bool) error {
err := templateResourceTf(resourceName, resourceDir, manifest)
err := templateResourceTf(resourceName, resourceDir, manifest, resourceDir)
if err != nil {
log.Fatalf("\nErr: %v", err)
return err
@@ -32,7 +33,7 @@ func provisionResource(resourceName, resourceDir string, manifest *Manifest, tem
return nil
}
func templateResourceTf(templateName, resourceDir string, manifest *Manifest) error {
func templateResourceTf(templateName, resourceDir string, manifest *Manifest, destinationDir string) error {
log.Printf("Creating templates for %s in %s", templateName, resourceDir)
tfFiles, err := bindata.AssetDir(strings.Join([]string{TemplatesDir, resourceDir}, "/"))
@@ -44,7 +45,7 @@ func templateResourceTf(templateName, resourceDir string, manifest *Manifest) er
tfBytes := bindata.MustAsset(strings.Join([]string{TemplatesDir, resourceDir, tfFile}, "/"))
t := template.Must(template.New(templateName).Funcs(sprig.TxtFuncMap()).Parse(string(tfBytes)))
tfOut, err := createFile(resourceDir, tfFile)
tfOut, err := createFile(destinationDir, tfFile)
if err != nil {
log.Fatalf("\nErr: %v", err)
return err

21
testdata/m1_basic/expected_output/aws-roles-tf/deploy.sh vendored Executable file
View File

@@ -0,0 +1,21 @@
#!/bin/sh
# exit when any command fails
set -e
additional_terraform_options=""
additional_kube_options=""
terraform_action=${1:-apply}
if [ $terraform_action == "apply" ];then
additional_terraform_options="${additional_terraform_options} -auto-approve"
else
additional_kube_options="${additional_kube_options} --dry-run"
fi
terraform init
terraform workspace select nonprod.np.navi-tech.in || terraform workspace new nonprod.np.navi-tech.in
terraform $terraform_action $additional_terraform_options
kubectl config use-context ${CLUSTER}
kubectl apply -f foo-navi-service-dev.yaml -n ${NAMESPACE} $additional_kube_options

17
testdata/m1_basic/expected_output/aws-roles-tf/main.tf vendored Executable file
View File

@@ -0,0 +1,17 @@
terraform {
backend "s3" {
bucket = "navi-bank-terraform-nonprod-state"
region = "ap-south-1"
key = "service-iam-roles"
workspace_key_prefix = "iamroles/dev/foo-navi-service"
profile = "nonprod"
acl = "bucket-owner-full-control"
}
}
module "iam-role" {
source = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/iam-roles.git"
environment = "dev"
service_role = {"policies":[{"actions":["s3:GetObject","s3:PutObject"],"resource":"arn:aws:s3:::navi-e3e2a9bfd88566b05001b02a3f51d286/*"},{"actions":["s3:GetObject","s3:PutObject"],"resource":"*"},{"actions":["sns:Publish","sns:SetSMSAttributes"],"resource":"arn:aws:s3:::arn:aws:s3:::test-bucket-to-be-deleted/*"}]}
role_name = "foo-navi-service"
}

15
testdata/m1_basic/expected_output/aws-s3-bucket-tf/deploy.sh vendored Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/sh
# exit when any command fails
set -e
additional_terraform_options=""
terraform_action=${1:-apply}
if [ $terraform_action == "apply" ];then
additional_terraform_options="${additional_terraform_options} -auto-approve"
fi
terraform init
terraform workspace select nonprod.np.navi-tech.in || terraform workspace new nonprod.np.navi-tech.in
terraform $terraform_action $additional_terraform_options

16
testdata/m1_basic/expected_output/aws-s3-bucket-tf/main.tf vendored Executable file
View File

@@ -0,0 +1,16 @@
terraform {
backend "s3" {
bucket = "navi-bank-terraform-nonprod-state"
region = "ap-south-1"
key = "s3-buckets"
workspace_key_prefix = "s3-buckets/dev/foo-navi-service"
profile = "nonprod"
acl = "bucket-owner-full-control"
}
}
module "s3-buckets" {
source = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/aws-s3-bucket.git"
s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":null},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}]}]
environment = "dev"
}

17
testdata/m1_basic/expected_output/rds-tf/deploy.sh vendored Executable file
View File

@@ -0,0 +1,17 @@
#!/bin/sh
set -e
additional_terraform_options=""
terraform_action=${1:-apply}
if [ $terraform_action == "apply" ];then
additional_terraform_options="${additional_terraform_options} -auto-approve"
fi
terraform init
terraform workspace select nonprod.np.navi-tech.in || terraform workspace new nonprod.np.navi-tech.in
terraform $terraform_action -target=module.rds.data.aws_subnet_ids.command_private_subnets $additional_terraform_options
terraform $terraform_action -target=module.rds.module.rds_instance $additional_terraform_options
terraform $terraform_action -target=module.rds.module.rds_instance_replica $additional_terraform_options
terraform $terraform_action -target=module.rds.module.postgres_db $additional_terraform_options

57
testdata/m1_basic/expected_output/rds-tf/main.tf vendored Executable file
View File

@@ -0,0 +1,57 @@
terraform {
backend "s3" {
bucket = "navi-bank-terraform-command-state"
region = "ap-south-1"
key = "rds"
workspace_key_prefix = "rds-states/foo-service"
profile = "cmd"
acl = "bucket-owner-full-control"
}
}
module "rds" {
source = "git::ssh://git@github.cmd.navi-tech.in/navi-infra/rds.git"
environment = "dev"
instance_name = "foo-service"
password = "foo_service_password"
user = "foo_service_user"
name = "foo"
monitoring_password = ""
monitoring_user = ""
databases = ["foo_service"]
database_tags = {
Team = "Infra"
medici-app = "foo-navi-service"
medici-owner = "Infra"
medici-environment = "dev"
}
team = "Infra"
size = 7
db_extensions = ["pgcrypto"]
readonly_user = "foo_readonly_user"
readonly_password = "foo_readonly_password"
backup_retention_period = 7
multi_az = true
read_replica_instance_class = "db.t3.micro"
read_replica_performance_insights_enabled = true
parameters = [
{
name = "rds.logical_replication"
value = "1"
apply_method = "pending-reboot"
}
]
cpu_utilization_alarm_threshold = "70"
cpucredit_balance_alarm_threshold = "120"
burst_balance_alarm_threshold = "85"
db_connections_alarm_threshold = "200"
queue_depth_alarm_threshold = "20"
free_storage_space_percent = 90
freeable_memory_threshold = 150
read_latency_alarm_threshold = "0.5"
write_latency_alarm_threshold = "0.5"
}

74
testdata/m1_basic/sample_infra_manifest.json vendored Normal file
View File

@@ -0,0 +1,74 @@
{
"extraResources": {
"environment": "dev",
"database": {
"instanceName": "foo-service",
"user": "foo_service_user",
"password": "foo_service_password",
"sizeInGb": 7,
"dbNames": ["foo_service"],
"dbExtensions": ["pgcrypto"],
"readonlyUser": "foo_readonly_user",
"readonlyPassword": "foo_readonly_password",
"applyImmediately": false,
"performanceInsightsEnabled": false,
"readReplica": {
"awsInstanceClass": "db.t3.micro",
"performanceInsightsEnabled": true
},
"parameters": {
"rds.logical_replication": "1"
},
"rdsAlertThresholds": {
"cpuUtilization": 70,
"cpuCreditBalance": 120,
"burstBalance": 85,
"dbConnections": 200,
"queueDepth": 20,
"freeStorageSpacePercent": 90,
"freeMemoryTooLowInMB": 150,
"readLatency": 0.5,
"writeLatency": 0.5
}
},
"aws_access": {
"policies": [
{
"actions": ["s3:GetObject", "s3:PutObject"],
"resource": "arn:aws:s3:::navi-e3e2a9bfd88566b05001b02a3f51d286/*"
},
{
"actions": ["s3:GetObject", "s3:PutObject"],
"resource": "*"
},
{
"resource": "arn:aws:s3:::arn:aws:s3:::test-bucket-to-be-deleted/*",
"actions": ["sns:Publish", "sns:SetSMSAttributes"]
}
]
},
"s3_buckets": [
{
"anonymizedBucketName": "navi-bucket-test-1",
"bucketTag": "customer-uploads"
},
{
"anonymizedBucketName": "navi-bucket-test-2",
"bucketTag": "document-uploads",
"lifecycleRules": [
{
"expiration": {
"days": 1
}
}
]
}
]
},
"team": {
"name": "Infra"
},
"deployment": {
"name": "foo"
}
}