varnitgoyal95/infra-provisioner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

INFRA-2938 | Abhishek | Add a feature to set lifecycle rules based on prefix path in S3 buckets

Browse Source
This commit is contained in:
Abhishek Katiyar
2024-04-10 18:33:13 +05:30
parent 96d0539399
commit 0a2b897165
6 changed files with 508 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

685
bindata/bindata.go
View File

File diff suppressed because one or more lines are too long

75
sample_infra_manifest.json
View File

@@ -22,8 +22,12 @@
"statementTimeout": 0,
"psqlEngineVersion": "13.9",
"storageEncrypted": true,
"dbNames": ["foo_service"],
"dbExtensions": ["pgcrypto"],
"dbNames": [
"foo_service"
],
"dbExtensions": [
"pgcrypto"
],
"readonlyUser": "foo_readonly_user",
"readonlyPassword": "foo_readonly_password",
"applyImmediately": true,
@@ -91,16 +95,25 @@
"aws_access": {
"policies": [
{
"actions": ["s3:GetObject", "s3:PutObject"],
"actions": [
"s3:GetObject",
"s3:PutObject"
],
"resource": "arn:aws:s3:::navi-e3e2a9bfd88566b05001b02a3f51d286/*"
},
{
"actions": ["s3:GetObject", "s3:PutObject"],
"actions": [
"s3:GetObject",
"s3:PutObject"
],
"resource": "*"
},
{
"resource": "arn:aws:s3:::arn:aws:s3:::test-bucket-to-be-deleted/*",
"actions": ["sns:Publish", "sns:SetSMSAttributes"]
"actions": [
"sns:Publish",
"sns:SetSMSAttributes"
]
}
]
},
@@ -111,16 +124,51 @@
"lifecycleRules": [
{
"expiration": {
"days": 1
"days": 3
},
"pathPrefix": "testing/test",
"transition": {
"days": 4,
"storageClass": "GLACIER"
},
"noncurrent_version_expiration": {
"days": 2
},
"noncurrent_version_transition": {
"days": 2,
"storageClass": "GLACIER"
}
},
{
"expiration": {
"days": 2
},
"transition": {
"days": 3,
"storageClass": "GLACIER"
},
"noncurrent_version_expiration": {
"days": 2
},
"noncurrent_version_transition": {
"days": 2,
"storageClass": "GLACIER"
}
}
],
"bucketPolicy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}",
"corsPolicy": [
{
"AllowedHeaders": ["*"],
"AllowedMethods": ["GET", "POST", "PUT"],
"AllowedOrigins": ["https://google.com"]
"AllowedHeaders": [
"*"
],
"AllowedMethods": [
"GET",
"POST",
"PUT"
],
"AllowedOrigins": [
"https://google.com"
]
}
]
},
@@ -131,6 +179,13 @@
{
"expiration": {
"days": 1
},
"noncurrent_version_expiration": {
"days": 2
},
"noncurrent_version_transition": {
"days": 2,
"storageClass": "GLACIER"
}
}
]

2
testdata/expected_output/sample_infra_manifest/aws-s3-bucket-tf/main.tf vendored
View File

@@ -11,7 +11,7 @@ terraform {
module "s3-buckets" {
source = "git::ssh://git@github.com/navi-infra/aws-s3-bucket.git"
s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["*"]}],"bucketPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}"},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["https://go-nlc.com"]}]}]
s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":[{"expiration":{"days":1}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["*"]}],"bucketPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}"},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["https://go-nlc.com"]}]}]
environment = "dev"
infra_vertical = "lending"
bucket_tags = {

2
testdata/expected_output/sample_infra_manifest_2/aws-s3-bucket-tf/main.tf vendored
View File

@@ -11,7 +11,7 @@ terraform {
module "s3-buckets" {
source = "git::ssh://git@github.com/navi-infra/aws-s3-bucket.git"
s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["*"]}],"bucketPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}"},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["https://go-nlc.com"]}]}]
s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":[{"expiration":{"days":1}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["*"]}],"bucketPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}"},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["https://go-nlc.com"]}]}]
environment = "dev"
infra_vertical = "insurance"
bucket_tags = {

2
testdata/expected_output/sample_infra_manifest_flink/aws-s3-bucket-tf/main.tf vendored
View File

@@ -11,7 +11,7 @@ terraform {
module "s3-buckets" {
source = "git::ssh://git@github.com/navi-infra/aws-s3-bucket.git"
s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["*"]}],"bucketPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}"},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1,"storageClass":""}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["https://go-nlc.com"]}]}]
s3_buckets = [{"anonymizedBucketName":"navi-bucket-test-1","bucketTag":"customer-uploads","lifecycleRules":[{"expiration":{"days":1}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["*"]}],"bucketPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::594542361424:role/databricks-ec2-role\"},\"Action\":[\"s3:GetObject\",\"s3:PutObject\",\"s3:PutObjectAcl\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1\",\"arn:aws:s3:::navi-dd80e5524820496dbacd4e84d05d95f1/*\"]}]}"},{"anonymizedBucketName":"navi-bucket-test-2","bucketTag":"document-uploads","lifecycleRules":[{"expiration":{"days":1}}],"enableAccessLog":false,"enablePublicBucket":false,"corsPolicy":[{"AllowedHeaders":["*"],"AllowedMethods":["PUT","GET"],"AllowedOrigins":["https://go-nlc.com"]}]}]
environment = "dev"
infra_vertical = "insurance"
bucket_tags = {

28
types.go
View File

@@ -243,14 +243,14 @@ type StateStoreBackend struct {
}
type S3Bucket struct {
BucketName string `json:"anonymizedBucketName" valid:"required"`
BucketTag string `json:"bucketTag" valid:"required"`
LifecycleRules []map[string]LifecycleRule `json:"lifecycleRules"`
EnableAccessLog bool `json:"enableAccessLog"`
EnablePublicBucket bool `json:"enablePublicBucket"`
CorsPolicy []CorsRule `json:"corsPolicy,omitempty"`
BucketPolicy string `json:"bucketPolicy,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
BucketName string `json:"anonymizedBucketName" valid:"required"`
BucketTag string `json:"bucketTag" valid:"required"`
LifecycleRules []LifecycleRuleWithPathPrefix `json:"lifecycleRules"`
EnableAccessLog bool `json:"enableAccessLog"`
EnablePublicBucket bool `json:"enablePublicBucket"`
CorsPolicy []CorsRule `json:"corsPolicy,omitempty"`
BucketPolicy string `json:"bucketPolicy,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
}
type RdsAlertThresholds struct {
@@ -307,9 +307,17 @@ type AuroraAlertDurations struct {
EBSIOBalance int `json:"ebsIOBalance"`
}
type LifecycleRuleWithPathPrefix struct {
PathPrefix string `json:"pathPrefix,omitempty"`
Expiration *LifecycleRule `json:"expiration,omitempty"`
Transition *LifecycleRule `json:"transition,omitempty"`
NoncurrentVersionExpiration *LifecycleRule `json:"noncurrent_version_expiration,omitempty"`
NoncurrentVersionTransition *LifecycleRule `json:"noncurrent_version_transition,omitempty"`
}
type LifecycleRule struct {
Days int `json:"days"`
StorageClass string `json:"storageClass"`
Days int `json:"days,omitempty"`
StorageClass string `json:"storageClass,omitempty"`
}
type CorsRule struct {