From cb548ab8eef3580589002a8cbb59c9343c8eaf8a Mon Sep 17 00:00:00 2001 From: Abhishek Katiyar Date: Mon, 23 Dec 2024 05:54:04 +0530 Subject: [PATCH 1/5] INFRA-3746 | Abhishek | Update kutegen --- kutegen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kutegen b/kutegen index 0bb5b971..48a74e64 160000 --- a/kutegen +++ b/kutegen @@ -1 +1 @@ -Subproject commit 0bb5b971c990abea93686ad64b990aab74a4f079 +Subproject commit 48a74e64a876b329590d41301857a4d8529dcb4c From 2d209df3091dd12dd61adcffbbb82eb71bb6d56b Mon Sep 17 00:00:00 2001 From: Abhishek Katiyar Date: Mon, 23 Dec 2024 22:07:40 +0530 Subject: [PATCH 2/5] INFRA-3746 | Abhishek | Add migration scripts to add port and language column in diagnostic table --- .../migration/V1.100__Adds_port_column_to_diagnostic_table.sql | 2 ++ .../V1.99__Adds_language_column_to_diagnostic_table.sql | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql create mode 100644 src/main/resources/db/migration/V1.99__Adds_language_column_to_diagnostic_table.sql diff --git a/src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql b/src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql new file mode 100644 index 00000000..5b360da4 --- /dev/null +++ b/src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql @@ -0,0 +1,2 @@ +ALTER TABLE diagnostic + ADD COLUMN port integer; diff --git a/src/main/resources/db/migration/V1.99__Adds_language_column_to_diagnostic_table.sql b/src/main/resources/db/migration/V1.99__Adds_language_column_to_diagnostic_table.sql new file mode 100644 index 00000000..d5cd2306 --- /dev/null +++ b/src/main/resources/db/migration/V1.99__Adds_language_column_to_diagnostic_table.sql @@ -0,0 +1,2 @@ +ALTER TABLE diagnostic + ADD COLUMN language VARCHAR(255) NOT NULL DEFAULT 'Java'; From 678321c744fdbb57f4d912b418f3a6f1a10ab70f Mon Sep 17 00:00:00 2001 From: Abhishek Katiyar Date: Tue, 24 Dec 2024 13:10:24 +0530 Subject: [PATCH 3/5] INFRA-3746 | Abhishek | Ensure usage of language and port for generating diagnostic reports --- .../portal/domain/manifest/Deployment.java | 20 ++++++++++++++++- .../v2/client/airflow/AirflowClient.java | 2 ++ .../portal/v2/diagnostic/Diagnostic.java | 5 +++++ .../v2/diagnostic/DiagnosticServiceImpl.java | 22 +++++++++++++++---- .../DiagnosticServiceUtilities.java | 10 +++++++++ .../diagnostic/DiagnosticTriggerRequest.java | 2 ++ 6 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java b/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java index 579b1aa9..a2d24e71 100644 --- a/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java +++ b/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java @@ -8,8 +8,10 @@ import com.navi.infra.portal.domain.JsonEntity; import com.navi.infra.portal.dto.manifest.CloneManifestRequest; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import javax.persistence.CascadeType; @@ -85,7 +87,7 @@ public class Deployment extends JsonEntity implements StatusMarker { if (hasField("allowEgress") && !cloneRequest.isCopyOutbound()) { getData().put("allowEgress", new ArrayList<>()); } - if (!Objects.equals(this.getSecurityGroups(),Collections.emptyList())){ + if (!Objects.equals(this.getSecurityGroups(), Collections.emptyList())) { removeSecurityGroupIds(); } @@ -143,4 +145,20 @@ public class Deployment extends JsonEntity implements StatusMarker { return (String) getData().get("controller"); } + @JsonIgnore + public List getExposedPorts() { + return (List) this.getData().get("exposedPorts"); + } + + @JsonIgnore + public Integer getServicePort() { + List exposedPorts = getExposedPorts(); + var servicePort = exposedPorts.stream() + .filter(exposedPort -> exposedPort.get("name").equals("serviceport")) + .collect(Collectors.toList()); + if (!servicePort.isEmpty()) { + return (Integer) servicePort.stream().findFirst().get().get("port"); + } + throw new RuntimeException("Could not find serviceport in manifest"); + } } diff --git a/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java b/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java index c0f2dee6..0478bd61 100644 --- a/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java +++ b/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java @@ -58,6 +58,8 @@ public class AirflowClient { Map.entry("pod_name", diagnosticTriggerRequest.getPodName()), Map.entry("target_container_name", diagnosticTriggerRequest.getContainerName()), Map.entry("image", diagnosticTriggerRequest.getRunnerImage()), + Map.entry("language", diagnosticTriggerRequest.getLanguage()), + Map.entry("port", diagnosticTriggerRequest.getPort()), Map.entry("pre_signed_url", Optional.ofNullable(diagnosticTriggerRequest.getPreSignedUrl()).orElse("")), Map.entry("sample_pre_signed_url", diff --git a/src/main/java/com/navi/infra/portal/v2/diagnostic/Diagnostic.java b/src/main/java/com/navi/infra/portal/v2/diagnostic/Diagnostic.java index 75b43514..ded7187e 100644 --- a/src/main/java/com/navi/infra/portal/v2/diagnostic/Diagnostic.java +++ b/src/main/java/com/navi/infra/portal/v2/diagnostic/Diagnostic.java @@ -1,6 +1,7 @@ package com.navi.infra.portal.v2.diagnostic; import com.navi.infra.portal.domain.BaseEntity; +import javax.annotation.Nullable; import javax.persistence.Convert; import javax.persistence.Entity; import javax.persistence.EnumType; @@ -27,6 +28,10 @@ public class Diagnostic extends BaseEntity { private String cluster; private String namespace; private String podName; + private String language; + + @Nullable + private Integer port; @Enumerated(EnumType.STRING) private DiagnosticStatus status; diff --git a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java index ea2dc88f..17d50879 100644 --- a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java +++ b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java @@ -9,7 +9,9 @@ import static com.navi.infra.portal.v2.diagnostic.DiagnosticServiceUtilities.get import static com.navi.infra.portal.v2.diagnostic.DiagnosticServiceUtilities.getDiagnosticPath; import static com.navi.infra.portal.v2.diagnostic.DiagnosticServiceUtilities.getDiagnosticTaskLogUrl; import static com.navi.infra.portal.v2.diagnostic.DiagnosticServiceUtilities.getEnvironment; +import static com.navi.infra.portal.v2.diagnostic.DiagnosticServiceUtilities.getLanguage; import static com.navi.infra.portal.v2.diagnostic.DiagnosticServiceUtilities.getNamespace; +import static com.navi.infra.portal.v2.diagnostic.DiagnosticServiceUtilities.getPort; import static java.util.stream.Collectors.toList; import com.navi.infra.portal.client.AwsS3Client; @@ -69,8 +71,10 @@ class DiagnosticServiceImpl implements DiagnosticService { final var namespace = getNamespace(manifest); final var environment = getEnvironment(manifest); final var manifestName = manifest.getName(); + final var language = getLanguage(manifest); + final var port = getPort(manifest); return createDiagnostic( - triggerDiagnosticRequest, cluster, namespace, environment, manifestName + triggerDiagnosticRequest, cluster, namespace, environment, manifestName, language, port ) .map(DiagnosticDto::new) .collect(toList()); @@ -81,7 +85,9 @@ class DiagnosticServiceImpl implements DiagnosticService { String cluster, String namespace, String env, - String manifestName + String manifestName, + String language, + Integer port ) { Function toDiagnosticEntity = response -> { var diagnostic = new Diagnostic(); @@ -90,6 +96,8 @@ class DiagnosticServiceImpl implements DiagnosticService { diagnostic.setManifestId(triggerDiagnosticRequest.getManifestId()); diagnostic.setCluster(response.getCluster()); diagnostic.setNamespace(response.getNamespace()); + diagnostic.setLanguage(language); + diagnostic.setPort(port); diagnostic.setPodName(response.getPodName()); diagnostic.setStatus(DiagnosticStatus.STARTED); diagnostic.setDiagnosticResultName(response.getDiagnosticResultName()); @@ -106,7 +114,9 @@ class DiagnosticServiceImpl implements DiagnosticService { cluster, namespace, env, - manifestName + manifestName, + language, + port ) ) .map(airflowClient::triggerDag) @@ -119,7 +129,9 @@ class DiagnosticServiceImpl implements DiagnosticService { String cluster, String namespace, String environment, - String manifestName + String manifestName, + String language, + Integer port ) { return podName -> { final var instant = LocalDateTime.now(ZoneId.of(ASIA_KOLKATA)); @@ -136,6 +148,8 @@ class DiagnosticServiceImpl implements DiagnosticService { .cluster(cluster) .namespace(namespace) .podName(podName) + .language(language) + .port(port.toString()) .dagId(dagId) .runId(createRunId(podName, instant)) .containerName(getContainerFromPod(podName)) diff --git a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java index 906e3a07..9cffec85 100644 --- a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java +++ b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java @@ -58,6 +58,16 @@ public class DiagnosticServiceUtilities { "Environment is not present in manifest " + manifest.fullName()); } + static String getLanguage(Manifest manifest) { + return requireNonNull(manifest.getMetadata().getLanguage(), + "Language is not present in manifest " + manifest.fullName()); + } + + static Integer getPort(Manifest manifest) { + return requireNonNull(manifest.getDeployment().getServicePort(), + "Port for diagnostics is not present in manifest " + manifest.fullName()); + } + static String getNamespace(Manifest manifest) { return requireNonNull(manifest.getDeployment().getNamespace(), "Namespace is not present in manifest " + manifest.fullName()); diff --git a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java index d37448a1..5be7a357 100644 --- a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java +++ b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java @@ -14,6 +14,8 @@ public class DiagnosticTriggerRequest { private String namespace; private String runId; private String podName; + private String language; + private String port; private String containerName; private String runnerImage; private String preSignedUrl; From acddd1d434327ea7507b2cfa423ce2658c860879 Mon Sep 17 00:00:00 2001 From: Abhishek Katiyar Date: Tue, 24 Dec 2024 14:57:56 +0530 Subject: [PATCH 4/5] INFRA-3746 | Abhishek | Update kutegen --- kutegen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kutegen b/kutegen index 48a74e64..fc15f165 160000 --- a/kutegen +++ b/kutegen @@ -1 +1 @@ -Subproject commit 48a74e64a876b329590d41301857a4d8529dcb4c +Subproject commit fc15f1657b9c754bd55ab0461a8b42a03e25eb6f From e308b7bc091061a870e56998979e2d5b58aeafab Mon Sep 17 00:00:00 2001 From: Abhishek Katiyar Date: Tue, 24 Dec 2024 14:59:12 +0530 Subject: [PATCH 5/5] INFRA-3746 | Abhishek | Use metrics port for diagnostics in Golang --- .../com/navi/infra/portal/domain/manifest/Deployment.java | 7 +++---- .../navi/infra/portal/v2/client/airflow/AirflowClient.java | 4 +++- .../infra/portal/v2/diagnostic/DiagnosticServiceImpl.java | 2 +- .../portal/v2/diagnostic/DiagnosticServiceUtilities.java | 4 ++-- .../portal/v2/diagnostic/DiagnosticTriggerRequest.java | 3 ++- ...__Adds_port_and_language_column_to_diagnostic_table.sql | 2 ++ .../V1.100__Adds_port_column_to_diagnostic_table.sql | 2 -- .../V1.101__Adds_language_column_to_diagnostic_table.sql | 2 -- 8 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 src/main/resources/db/migration/V1.100__Adds_port_and_language_column_to_diagnostic_table.sql delete mode 100644 src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql delete mode 100644 src/main/resources/db/migration/V1.101__Adds_language_column_to_diagnostic_table.sql diff --git a/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java b/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java index a2d24e71..8b27ea23 100644 --- a/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java +++ b/src/main/java/com/navi/infra/portal/domain/manifest/Deployment.java @@ -3,7 +3,6 @@ package com.navi.infra.portal.domain.manifest; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.navi.infra.portal.domain.JsonEntity; import com.navi.infra.portal.dto.manifest.CloneManifestRequest; import java.util.ArrayList; @@ -151,14 +150,14 @@ public class Deployment extends JsonEntity implements StatusMarker { } @JsonIgnore - public Integer getServicePort() { + public Integer getMetricsPort() { List exposedPorts = getExposedPorts(); var servicePort = exposedPorts.stream() - .filter(exposedPort -> exposedPort.get("name").equals("serviceport")) + .filter(exposedPort -> exposedPort.get("name").equals("metrics")) .collect(Collectors.toList()); if (!servicePort.isEmpty()) { return (Integer) servicePort.stream().findFirst().get().get("port"); } - throw new RuntimeException("Could not find serviceport in manifest"); + return null; } } diff --git a/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java b/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java index 0478bd61..b1af93ad 100644 --- a/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java +++ b/src/main/java/com/navi/infra/portal/v2/client/airflow/AirflowClient.java @@ -59,7 +59,9 @@ public class AirflowClient { Map.entry("target_container_name", diagnosticTriggerRequest.getContainerName()), Map.entry("image", diagnosticTriggerRequest.getRunnerImage()), Map.entry("language", diagnosticTriggerRequest.getLanguage()), - Map.entry("port", diagnosticTriggerRequest.getPort()), + Map.entry("port", + Optional.ofNullable(diagnosticTriggerRequest.getPort()).map(Object::toString) + .orElse("")), Map.entry("pre_signed_url", Optional.ofNullable(diagnosticTriggerRequest.getPreSignedUrl()).orElse("")), Map.entry("sample_pre_signed_url", diff --git a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java index 17d50879..70ea0bce 100644 --- a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java +++ b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceImpl.java @@ -149,7 +149,7 @@ class DiagnosticServiceImpl implements DiagnosticService { .namespace(namespace) .podName(podName) .language(language) - .port(port.toString()) + .port(port) .dagId(dagId) .runId(createRunId(podName, instant)) .containerName(getContainerFromPod(podName)) diff --git a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java index 9cffec85..fd8e830e 100644 --- a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java +++ b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticServiceUtilities.java @@ -64,8 +64,8 @@ public class DiagnosticServiceUtilities { } static Integer getPort(Manifest manifest) { - return requireNonNull(manifest.getDeployment().getServicePort(), - "Port for diagnostics is not present in manifest " + manifest.fullName()); + return requireNonNull(manifest.getDeployment().getMetricsPort(), + "Metrics port is not present in manifest " + manifest.fullName()); } static String getNamespace(Manifest manifest) { diff --git a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java index 5be7a357..0a7c5cfd 100644 --- a/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java +++ b/src/main/java/com/navi/infra/portal/v2/diagnostic/DiagnosticTriggerRequest.java @@ -2,6 +2,7 @@ package com.navi.infra.portal.v2.diagnostic; import java.util.List; +import javax.annotation.Nullable; import lombok.Builder; import lombok.Data; @@ -15,7 +16,7 @@ public class DiagnosticTriggerRequest { private String runId; private String podName; private String language; - private String port; + private Integer port; private String containerName; private String runnerImage; private String preSignedUrl; diff --git a/src/main/resources/db/migration/V1.100__Adds_port_and_language_column_to_diagnostic_table.sql b/src/main/resources/db/migration/V1.100__Adds_port_and_language_column_to_diagnostic_table.sql new file mode 100644 index 00000000..e8e592a1 --- /dev/null +++ b/src/main/resources/db/migration/V1.100__Adds_port_and_language_column_to_diagnostic_table.sql @@ -0,0 +1,2 @@ +ALTER TABLE diagnostic ADD COLUMN port integer; +ALTER TABLE diagnostic ADD COLUMN language VARCHAR(255) NOT NULL DEFAULT 'Java'; \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql b/src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql deleted file mode 100644 index 5b360da4..00000000 --- a/src/main/resources/db/migration/V1.100__Adds_port_column_to_diagnostic_table.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE diagnostic - ADD COLUMN port integer; diff --git a/src/main/resources/db/migration/V1.101__Adds_language_column_to_diagnostic_table.sql b/src/main/resources/db/migration/V1.101__Adds_language_column_to_diagnostic_table.sql deleted file mode 100644 index d5cd2306..00000000 --- a/src/main/resources/db/migration/V1.101__Adds_language_column_to_diagnostic_table.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE diagnostic - ADD COLUMN language VARCHAR(255) NOT NULL DEFAULT 'Java';