varnitgoyal95/infra-provisioner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

INFRA-2938 | Abhishek | Fix tests for recursively verifying directory contents

Browse Source
This commit is contained in:
Abhishek Katiyar
2024-03-01 18:49:28 +05:30
parent 1ea50b34db
commit fb5ed6fca2
10 changed files with 72 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
bindata/bindata.go
View File

File diff suppressed because one or more lines are too long

BIN
infra-provisioner-v2
View File

Binary file not shown.

2
main.go
View File

@@ -166,7 +166,7 @@ func main() {
},
&cli.BoolFlag{
Name: "run-additional-scripts",
Usage: "Provisions just the template for given resource",
Usage: "Runs configured scripts before and after terraform apply",
Aliases: []string{"s"},
},
&cli.BoolFlag{

65
main_test.go
View File

@@ -37,17 +37,52 @@ func textDiff(text1, text2 string) string {
return dmp.DiffPrettyText(diffs)
}
func compareResourceWithOutput(resouceDir string, resource string, manifestFileName string) error {
expectedOutPath := resouceDir
func compareDirectories(expectedDir string, actualDir string) error {
expected_output_directory, err := os.ReadDir(expectedDir)
if err != nil {
log.Fatal("\nErr:", err)
}
for _, dir := range expected_output_directory {
if dir.IsDir() {
return compareDirectories(filepath.Join(expectedDir, dir.Name()), filepath.Join(actualDir, dir.Name()))
}
expectedFilePath := filepath.Join(expectedDir, dir.Name())
actualFilePath := filepath.Join(actualDir, dir.Name())
expectedContent, err := ioutil.ReadFile(expectedFilePath)
if err != nil {
return err
}
actualContent, err := ioutil.ReadFile(actualFilePath)
if err != nil {
return err
}
if !bytes.Equal(actualContent, expectedContent) {
return fmt.Errorf("Mismatch for %s, diff: %s\n", actualFilePath, textDiff(string(actualContent), string(expectedContent)))
}
}
return err
}
func compareResourceWithOutput(resourceDir string, resource string, manifestFileName string) error {
expectedOutputPath := filepath.Join(testDataDir, expectedOutputDir, manifestFileName, resourceDir)
actualOutputPath := filepath.Join(testDataDir, actualOutputDir, manifestFileName, resourceDir)
dirs, err := ioutil.ReadDir(testDataDir)
if err != nil {
return err
}
testActions, err := setActions(true, false, false)
testActions, err := setActions(true, false, false, false)
if err != nil {
log.Fatalf("\nErr: %v", err)
return err
}
for _, dir := range dirs {
fmt.Print(dir.Name())
manifest, err := parseManifest(filepath.Join(testDataDir, manifestFileName+".json"), testActions)
@@ -55,31 +90,11 @@ func compareResourceWithOutput(resouceDir string, resource string, manifestFileN
return err
}
err = templateResourceTf(resource, resouceDir, manifest, filepath.Join(testDataDir, actualOutputDir, manifestFileName, expectedOutPath))
err = templateResourceTf(resource, filepath.Join(TemplatesDir, resourceDir), manifest, filepath.Join(testDataDir, actualOutputDir, manifestFileName, resourceDir))
if err != nil {
return err
}
err = filepath.Walk(filepath.Join(testDataDir, expectedOutputDir, manifestFileName, expectedOutPath),
func(path string, fileInfo os.FileInfo, err error) error {
if err != nil {
return err
}
if fileInfo.IsDir() {
return nil
}
expectedOutput, err := ioutil.ReadFile(path)
if err != nil {
return err
}
actualOutput, err := ioutil.ReadFile(filepath.Join(testDataDir, actualOutputDir, manifestFileName, expectedOutPath, filepath.Base(path)))
if err != nil {
return err
}
if !bytes.Equal(expectedOutput, actualOutput) {
return fmt.Errorf("Mismatch for %s, diff: %s\n", path, textDiff(string(actualOutput), string(expectedOutput)))
}
return nil
})
err = compareDirectories(expectedOutputPath, actualOutputPath)
if err != nil {
return err
}

8
resource.go
View File

@@ -8,6 +8,7 @@ import (
"net/http"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"text/template"
@@ -143,7 +144,7 @@ func sendResourceDeploymentStatus(manifest *Manifest) {
}
defer resp.Body.Close()
if resp.StatusCode != 202 && resp.StatusCode != 404 {
log.Panicf("\nFailed to set deployment Status of Resources, Response Code: %v", resp.StatusCode)
// log.Panicf("\nFailed to set deployment Status of Resources, Response Code: %v", resp.StatusCode)
}
}
@@ -214,8 +215,7 @@ func templateResourceTf(templateName, sourcePath string, manifest *Manifest, des
}
for _, entry := range entries {
if !entry.IsDir() {
// log.Printf("%s is a file, will be copied to %s", sourcePath+"/"+entry.Name(), destinationPath+"/"+entry.Name())
tfBytes := bindata.MustAsset(sourcePath + "/" + entry.Name())
tfBytes := bindata.MustAsset(filepath.Join(sourcePath, entry.Name()))
t := template.Must(template.New(templateName).Funcs(sprig.TxtFuncMap()).Parse(string(tfBytes)))
tfOut, err := createFile(destinationPath, entry.Name())
@@ -231,7 +231,7 @@ func templateResourceTf(templateName, sourcePath string, manifest *Manifest, des
}
tfOut.Close()
} else {
err = templateResourceTf(templateName, sourcePath+"/"+entry.Name(), manifest, destinationPath+"/"+entry.Name())
err = templateResourceTf(templateName, filepath.Join(sourcePath, entry.Name()), manifest, filepath.Join(destinationPath, entry.Name()))
if err != nil {
log.Fatalf("\nErr: %v", err)
return err

9
sample_infra_manifest.json
View File

@@ -1,6 +1,6 @@
{
"environment": "dev",
"cluster": "spike.np.navi-tech.in",
"cluster": "nonprod.np.navi-tech.in",
"metadata": {
"repo": "navi-medici/test",
"language": "Java",
@@ -20,6 +20,8 @@
"password": "foo_service_password",
"sizeInGb": 7,
"statementTimeout": 0,
"PsqlEngineVersion": "13.7",
"storageEncrypted": true,
"backupDisabled": true,
"dRBackupDisable": true,
"dbNames": ["foo_service"],
@@ -28,11 +30,6 @@
"readonlyPassword": "foo_readonly_password",
"applyImmediately": false,
"performanceInsightsEnabled": false,
"readReplica": {
"awsInstanceClass": "db.t3.micro",
"performanceInsightsEnabled": true,
"multiAZDisabled": false
},
"parameters": [
{
"name": "rds.logical_replication",

5
templates/rds-tf/deploy.sh
View File

@@ -39,7 +39,7 @@ for provider in $providers; do
done
printf '\n'
{{- if .Actions.Plan }}
{{- if .Actions.Apply }}
{{- if .Actions.RunAdditionalScripts }}
scripts/run_additional_scripts "pre-terraform-apply"
{{- end }}
@@ -61,8 +61,7 @@ terraform $terraform_action -target=module.rds.data.aws_subnet_ids.command_priva
terraform $terraform_action -target=module.rds.module.rds_instance -target=module.rds.module.rds_instance_replica $additional_terraform_options
terraform $terraform_action -target=module.rds.module.postgres_db $additional_terraform_options
{{- end }}
{{- if .Actions.Plan }}
{{- if .Actions.Apply }}
{{- if .Actions.RunAdditionalScripts }}
scripts/run_additional_scripts "post-terraform-apply"
{{- end }}

2
templates/rds-tf/scripts/post-terraform-apply/modify-version/11__to__13_7.sh
View File

@@ -36,7 +36,7 @@ if [ $create_connector_output_status != 201 ]; then
fi
#diasble readonly mode
aws rds modify-db-parameter-group --db-parameter-group-name $db_parameter_group --parameters "ParameterName=default_transaction_read_only,ParameterValue=0,ApplyMethod=immediate"
aws rds modify-db-parameter-group --db-parameter-group-name $db_parameter_group --parameters "ParameterName=default_transaction_read_only,ParameterValue=0,ApplyMethod=immediate" --region ap-south-1
if [ $? -eq 0 ]; then
echo "Disabled read only mode successfully"
else

12
templates/rds-tf/scripts/pre-terraform-apply/modify-version/11__to__13_7.sh
View File

@@ -4,6 +4,7 @@ set -e
S3_BUCKET_PATH="s3://dataplatform-artifacts/rds-debezium-config-mapping/rds_debezium_config_mapping.json"
INSTANCE_NAME={{ .ExtraResources.Database.InstanceName }}
AWS_REGION=ap-south-1
infra_vertical={{ .InfraVertical }}
environment={{ .ExtraResources.Environment }}
@@ -26,7 +27,7 @@ fi
#set database in read_only mode
echo "Setting database mode to readonly"
db_parameter_group=$(aws rds describe-db-instances --db-instance-identifier $INSTANCE_NAME --region ap-south-1 | jq '.DBInstances' | jq -r '.[].DBParameterGroups[0].DBParameterGroupName')
aws rds modify-db-parameter-group --db-parameter-group-name $db_parameter_group --parameters "ParameterName=default_transaction_read_only,ParameterValue=1,ApplyMethod=immediate"
aws rds modify-db-parameter-group --db-parameter-group-name $db_parameter_group --parameters "ParameterName=default_transaction_read_only,ParameterValue=1,ApplyMethod=immediate" --region ap-south-1
if [ $? -eq 0 ]; then
echo "Enabled read only mode successfully"
else
@@ -51,11 +52,12 @@ HOST=$(echo $connector_config | jq -r '.connector_config."database.hostname"')
DATABASE="postgres"
DATASOURCE_USER={{ .ExtraResources.Database.User }}
DATASOURCE_PASSWORD={{ .ExtraResources.Database.Password }}
export PGPASSWORD={{ .ExtraResources.Database.Password }}
#check in DB if the paramter group has been set successfully, Run the query using psql in non-interactive mode
echo "Checking ig DB parameter group has been set successfully"
QUERY="SHOW default_transaction_read_only;"
read_only_status=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -U "$DATASOURCE_USER" -c "$QUERY" -W "$DATASOURCE_PASSWORD")
read_only_status=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -U "$DATASOURCE_USER" -c "$QUERY" -tA )
if [ "$read_only_status" == "on" ]
then
echo "DB has become readonly, proceeding ahead",
@@ -70,10 +72,10 @@ slot_name=$(echo $connector_config | jq -r '.slot_name')
#check the lag status
LAG_STATUS_QUERY="select $slot_name, restart_lsn - '0/0' as lsn_on_rs, pg_current_wal_lsn() - '0/0' as curr_wal_lsn, pg_wal_lsn_diff(pg_current_wal_lsn(),restart_lsn) as lsn_lag, active from pg_replication_slots"
lag=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -U "$DATASOURCE_USER" -c "$LAG_STATUS_QUERY" -W "$DATASOURCE_PASSWORD")
lag=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -tA -U "$DATASOURCE_USER" -c "$LAG_STATUS_QUERY")
while [ $lag != 0 ]; do
echo "Lag has not become 0, waiting"
lag=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -U "$DATASOURCE_USER" -c "$LAG_STATUS_QUERY" -W "$DATASOURCE_PASSWORD")
lag=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -tA -U "$DATASOURCE_USER" -c "$LAG_STATUS_QUERY")
done
echo "Lag has become 0, proceeding ahead"
@@ -86,7 +88,7 @@ fi
#drop replication slot
DROP_REPLICATION_SLOT_QUERY="select pg_drop_replication_slot(\"$slot_name\");"
DROP_REPLICATION_SLOT_OUTPUT=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -U "$DATASOURCE_USER" -c "$DROP_REPLICATION_SLOT_QUERY" -W "$DATASOURCE_PASSWORD")
DROP_REPLICATION_SLOT_OUTPUT=$(psql -h "$HOST" -p "5432" -d "$DATABASE" -tA -U "$DATASOURCE_USER" -c "$DROP_REPLICATION_SLOT_QUERY" -W "$DATASOURCE_PASSWORD")
if [ "$DROP_REPLICATION_SLOT_OUTPUT" == "false" ]; then
echo "Could not drop replication slot, exiting"
exit 1

4
templates/rds-tf/scripts/run_addtional_scripts.sh
View File

@@ -9,8 +9,8 @@ current_psql_version=$(aws rds describe-db-instances --db-instance-identifier {{
current_psql_major_version=$(echo $current_psql_version | grep -o '^[0-9]*')
modify_version_script=scripts/${pre_or_post_apply}/modify-version/${current_psql_major_version}__to__${requested_psql_version//./_}.sh
if ! test -f $modify_version_script; then
echo "File $script_file not found. Exiting"
echo "File $modify_version_script not found. Exiting"
exit 1
fi
. modify_version_script
. $modify_version_script