diff --git a/kutegen b/kutegen index 79b03196..4d98606c 160000 --- a/kutegen +++ b/kutegen @@ -1 +1 @@ -Subproject commit 79b031962444ddd18dc1724a010a43782805511c +Subproject commit 4d98606cf00c49d443759b53640c19b2f487d10c diff --git a/scripts/add_flink_autoscaler_field.py b/scripts/add_flink_autoscaler_field.py new file mode 100644 index 00000000..b11d0857 --- /dev/null +++ b/scripts/add_flink_autoscaler_field.py @@ -0,0 +1,32 @@ +import asyncio + +from _migration_framework.abstract import AbstractMigration + + +class CustomMigration(AbstractMigration): + + def migrate(self, manifest: dict) -> dict: + if 'type' in manifest and manifest['type'] == 'deployment': + return manifest + + (manifest.setdefault('flink', {}) + .setdefault('flinkDeployment', {}) + .setdefault('flinkConfiguration', {}) + .setdefault('autoscaler', {}) + .setdefault('enabled', False)) + + return manifest + + +async def main(): + migration = CustomMigration( + "https://deployment-portal-backend.np.navi-tech.in", + "", + # Dry run is enabled by default. Uncomment this line to disable dry run + # dry_run=False + ) + await migration.run() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/main/resources/jsonschema/flink/flinkDeployment.json b/src/main/resources/jsonschema/flink/flinkDeployment.json index 3305f7d3..626ff6a0 100644 --- a/src/main/resources/jsonschema/flink/flinkDeployment.json +++ b/src/main/resources/jsonschema/flink/flinkDeployment.json @@ -12,7 +12,8 @@ "type": "object", "required": [ "taskManagerSlots", - "flinkVersion" + "flinkVersion", + "autoscaler" ], "properties": { "taskManagerSlots": { @@ -24,6 +25,43 @@ "v1_17", "v1_19" ] + }, + "autoscaler": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "stabilizationInterval": { + "type": "string", + "pattern": "^\\d+m$" + }, + "metricsWindow": { + "type": "string", + "pattern": "^\\d+m$" + }, + "targetUtilizationBoundary": { + "type": "string", + "pattern": "^(0(\\.\\d+)?|1(\\.0)?)$" + } + }, + "required": [ + "enabled" + ], + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": [ + "stabilizationInterval", + "metricsWindow", + "targetUtilizationBoundary" + ] + } } } }, diff --git a/src/test/resources/fixtures/kube_objects/kube_objects_flink.json b/src/test/resources/fixtures/kube_objects/kube_objects_flink.json index 526b37d3..c53a81cf 100644 --- a/src/test/resources/fixtures/kube_objects/kube_objects_flink.json +++ b/src/test/resources/fixtures/kube_objects/kube_objects_flink.json @@ -20,8 +20,6 @@ "kubernetes.operator.pod-template.merge-arrays-by-name": "true", "kubernetes.operator.savepoint.history.max.count": "24", "kubernetes.taskmanager.cpu.limit-factor": "1.5", - "metrics.reporter.promgateway.groupingKey": "tag_team=Infra", - "metrics.reporter.promgateway.jobName": "test-job", "restart-strategy.exponential-delay.backoff-multiplier": "2.0", "restart-strategy.exponential-delay.initial-backoff": "10s", "restart-strategy.exponential-delay.jitter-factor": "0.1", @@ -31,7 +29,8 @@ "state.backend.type": "filesystem", "state.checkpoints.dir": "s3://navi-flink-nonprod/jobs/dev/test-job/checkpoints", "state.savepoints.dir": "s3://navi-flink-nonprod/jobs/dev/test-job/savepoints", - "taskmanager.numberOfTaskSlots": "2" + "taskmanager.numberOfTaskSlots": "2", + "pipeline.max-parallelism": "100" }, "flinkVersion": "v1_17", "image": "193044292705.dkr.ecr.ap-south-1.amazonaws.com/common/flink:1.17.2-s3-hadoop", @@ -636,7 +635,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "test-job", @@ -668,7 +667,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "test-job", diff --git a/src/test/resources/fixtures/kube_objects/kube_objects_flink_custom_alerts.json b/src/test/resources/fixtures/kube_objects/kube_objects_flink_custom_alerts.json index 44809c57..f26a4a93 100644 --- a/src/test/resources/fixtures/kube_objects/kube_objects_flink_custom_alerts.json +++ b/src/test/resources/fixtures/kube_objects/kube_objects_flink_custom_alerts.json @@ -183,8 +183,6 @@ "kubernetes.operator.pod-template.merge-arrays-by-name": "true", "kubernetes.operator.savepoint.history.max.count": "24", "kubernetes.taskmanager.cpu.limit-factor": "1.5", - "metrics.reporter.promgateway.groupingKey": "tag_team=Cloud-Platform", - "metrics.reporter.promgateway.jobName": "delete-flink-test", "restart-strategy.exponential-delay.backoff-multiplier": "2.0", "restart-strategy.exponential-delay.initial-backoff": "10s", "restart-strategy.exponential-delay.jitter-factor": "0.1", @@ -194,7 +192,8 @@ "state.backend.type": "filesystem", "state.checkpoints.dir": "s3://navi-flink-nonprod/jobs/dev/delete-flink-test/checkpoints", "state.savepoints.dir": "s3://navi-flink-nonprod/jobs/dev/delete-flink-test/savepoints", - "taskmanager.numberOfTaskSlots": "2" + "taskmanager.numberOfTaskSlots": "2", + "pipeline.max-parallelism": "100" }, "flinkVersion": "v1_17", "image": "193044292705.dkr.ecr.ap-south-1.amazonaws.com/common/flink:1.17.2-s3-hadoop", @@ -652,7 +651,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "delete-flink-test", @@ -684,7 +683,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "delete-flink-test", diff --git a/src/test/resources/fixtures/kube_objects/kube_objects_flink_eks.json b/src/test/resources/fixtures/kube_objects/kube_objects_flink_eks.json index e14da768..1d8420ea 100644 --- a/src/test/resources/fixtures/kube_objects/kube_objects_flink_eks.json +++ b/src/test/resources/fixtures/kube_objects/kube_objects_flink_eks.json @@ -208,8 +208,6 @@ "kubernetes.operator.pod-template.merge-arrays-by-name": "true", "kubernetes.operator.savepoint.history.max.count": "24", "kubernetes.taskmanager.cpu.limit-factor": "1.5", - "metrics.reporter.promgateway.groupingKey": "tag_team=Infra", - "metrics.reporter.promgateway.jobName": "sample-flink-job-1", "restart-strategy.exponential-delay.backoff-multiplier": "2.0", "restart-strategy.exponential-delay.initial-backoff": "10s", "restart-strategy.exponential-delay.jitter-factor": "0.1", @@ -219,7 +217,8 @@ "state.backend.type": "filesystem", "state.checkpoints.dir": "s3://navi-flink-navi-pay-nonprod/jobs/dev/sample-flink-job-1/checkpoints", "state.savepoints.dir": "s3://navi-flink-navi-pay-nonprod/jobs/dev/sample-flink-job-1/savepoints", - "taskmanager.numberOfTaskSlots": "2" + "taskmanager.numberOfTaskSlots": "2", + "pipeline.max-parallelism": "100" }, "flinkVersion": "v1_17", "image": "193044292705.dkr.ecr.ap-south-1.amazonaws.com/common/flink:1.17.2-s3-hadoop", @@ -266,11 +265,11 @@ "name": "metrics" } ], - "volumeMounts": [ ] + "volumeMounts": [] } ], "serviceAccountName": "sample-flink-job-1-navi-service-dev", - "volumes": [ ] + "volumes": [] } }, "restartNonce": 0, @@ -617,7 +616,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "sample-flink-job-1", @@ -649,7 +648,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "sample-flink-job-1", diff --git a/src/test/resources/fixtures/kube_objects/kube_objects_flink_with_rocksdb_support.json b/src/test/resources/fixtures/kube_objects/kube_objects_flink_with_rocksdb_support.json index 5bae8391..75298b31 100644 --- a/src/test/resources/fixtures/kube_objects/kube_objects_flink_with_rocksdb_support.json +++ b/src/test/resources/fixtures/kube_objects/kube_objects_flink_with_rocksdb_support.json @@ -207,8 +207,6 @@ "kubernetes.operator.pod-template.merge-arrays-by-name": "true", "kubernetes.operator.savepoint.history.max.count": "24", "kubernetes.taskmanager.cpu.limit-factor": "1.5", - "metrics.reporter.promgateway.groupingKey": "tag_team=Infra", - "metrics.reporter.promgateway.jobName": "test-job", "restart-strategy.exponential-delay.backoff-multiplier": "2.0", "restart-strategy.exponential-delay.initial-backoff": "10s", "restart-strategy.exponential-delay.jitter-factor": "0.1", @@ -219,7 +217,8 @@ "state.backend.type": "rocksdb", "state.checkpoints.dir": "s3://navi-flink-nonprod/jobs/dev/test-job/checkpoints", "state.savepoints.dir": "s3://navi-flink-nonprod/jobs/dev/test-job/savepoints", - "taskmanager.numberOfTaskSlots": "2" + "taskmanager.numberOfTaskSlots": "2", + "pipeline.max-parallelism": "100" }, "flinkVersion": "v1_17", "image": "193044292705.dkr.ecr.ap-south-1.amazonaws.com/common/flink:1.17.2-s3-hadoop", @@ -690,7 +689,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "test-job", @@ -722,7 +721,7 @@ "apiVersion": "policy/v1", "kind": "PodDisruptionBudget", "metadata": { - "annotations": { }, + "annotations": {}, "labels": { "Environment": "dev", "Name": "test-job", @@ -750,8 +749,6 @@ "maxUnavailable": 0 } } - - ], "kind": "List", "apiVersion": "v1" diff --git a/src/test/resources/fixtures/kube_objects/manifest_flink.json b/src/test/resources/fixtures/kube_objects/manifest_flink.json index cd2fa5ab..acede4e4 100644 --- a/src/test/resources/fixtures/kube_objects/manifest_flink.json +++ b/src/test/resources/fixtures/kube_objects/manifest_flink.json @@ -93,7 +93,10 @@ "flinkConfiguration": { "flinkVersion": "v1_17", "taskManagerSlots": 2, - "savepointFrequency": "1h" + "savepointFrequency": "1h", + "autoscaler": { + "enabled": false + } } }, "flinkJob": { diff --git a/src/test/resources/fixtures/kube_objects/manifest_flink_custom_alerts.json b/src/test/resources/fixtures/kube_objects/manifest_flink_custom_alerts.json index e80a6d80..cb9c55e0 100644 --- a/src/test/resources/fixtures/kube_objects/manifest_flink_custom_alerts.json +++ b/src/test/resources/fixtures/kube_objects/manifest_flink_custom_alerts.json @@ -85,7 +85,10 @@ "flinkConfiguration": { "taskManagerSlots": 2, "savepointFrequency": "1d", - "flinkVersion": "v1_17" + "flinkVersion": "v1_17", + "autoscaler": { + "enabled": false + } } }, "flinkJob": { diff --git a/src/test/resources/fixtures/kube_objects/manifest_flink_eks.json b/src/test/resources/fixtures/kube_objects/manifest_flink_eks.json index 5d6cfe69..fbbe3182 100644 --- a/src/test/resources/fixtures/kube_objects/manifest_flink_eks.json +++ b/src/test/resources/fixtures/kube_objects/manifest_flink_eks.json @@ -71,7 +71,10 @@ "flinkConfiguration": { "taskManagerSlots": 2, "savepointFrequency": "1h", - "flinkVersion": "v1_17" + "flinkVersion": "v1_17", + "autoscaler": { + "enabled": false + } } }, "flinkJob": { diff --git a/src/test/resources/fixtures/kube_objects/manifest_flink_with_rocksdb_support.json b/src/test/resources/fixtures/kube_objects/manifest_flink_with_rocksdb_support.json index 03fc8503..b9ebe0db 100644 --- a/src/test/resources/fixtures/kube_objects/manifest_flink_with_rocksdb_support.json +++ b/src/test/resources/fixtures/kube_objects/manifest_flink_with_rocksdb_support.json @@ -59,7 +59,10 @@ "flinkConfiguration": { "taskManagerSlots": 2, "savepointFrequency": "1h", - "flinkVersion": "v1_17" + "flinkVersion": "v1_17", + "autoscaler": { + "enabled": false + } } }, "flinkJob": {