diff --git a/pom.xml b/pom.xml index 1fd8bcf1..54f9d538 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,6 @@ httpclient 4.5.6 - io.micrometer @@ -160,17 +159,6 @@ org.springframework.boot test - - com.h2database - h2 - 1.4.200 - test - - - net.joshka - junit-json-params - 5.6.2-r0 - org.glassfish javax.json diff --git a/src/main/java/com/navi/infra/portal/domain/manifest/Manifest.java b/src/main/java/com/navi/infra/portal/domain/manifest/Manifest.java index c020c196..fcc8debe 100644 --- a/src/main/java/com/navi/infra/portal/domain/manifest/Manifest.java +++ b/src/main/java/com/navi/infra/portal/domain/manifest/Manifest.java @@ -2,10 +2,12 @@ package com.navi.infra.portal.domain.manifest; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.hash.Hashing; import com.navi.infra.portal.domain.JsonEntity; import com.navi.infra.portal.domain.notification.Notification; +import com.navi.infra.portal.util.Mapper; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.function.Predicate; @@ -47,6 +49,7 @@ public class Manifest extends JsonEntity { @JsonManagedReference private Deployment deployment; + @Transient private String infraVertical; @@ -83,7 +86,7 @@ public class Manifest extends JsonEntity { public String getSecretVaultPath(String portalVertical) { if (portalVertical.equals("insurance")) { return String.format("config/infra/gi/prod/portal-asset/secret/%s/%s", environment, - name); + name); } return String.format("config/infra/prod/portal-asset/secret/%s/%s", environment, name); } @@ -92,10 +95,10 @@ public class Manifest extends JsonEntity { public String getSuperSecretVaultPath(String portalVertical) { if (portalVertical.equals("insurance")) { return String - .format("config/infra/gi/prod/portal-asset/super-secret/%s/%s", environment, name); + .format("config/infra/gi/prod/portal-asset/super-secret/%s/%s", environment, name); } return String - .format("config/infra/prod/portal-asset/super-secret/%s/%s", environment, name); + .format("config/infra/prod/portal-asset/super-secret/%s/%s", environment, name); } @JsonIgnore @@ -105,8 +108,8 @@ public class Manifest extends JsonEntity { List environmentVariables = deployment.getEnvironmentVariables(); if (environmentVariables != null) { environmentVariables - .forEach(secretConfig -> environmentMap - .put(secretConfig.getName(), secretConfig.getValue())); + .forEach(secretConfig -> environmentMap + .put(secretConfig.getName(), secretConfig.getValue())); } } return environmentMap; @@ -126,9 +129,9 @@ public class Manifest extends JsonEntity { List environmentVariables = deployment.getEnvironmentVariables(); if (environmentVariables != null) { return environmentVariables.stream() - .filter(SecretConfig::isSuperSecret) - .map(SecretConfig::getName) - .collect(Collectors.toSet()); + .filter(SecretConfig::isSuperSecret) + .map(SecretConfig::getName) + .collect(Collectors.toSet()); } } return new HashSet<>(); @@ -140,15 +143,15 @@ public class Manifest extends JsonEntity { List environmentVariables = deployment.getEnvironmentVariables(); if (environmentVariables != null) { environmentVariables.stream() - .filter(predicate) - .forEach(secretConfig -> { - String name = secretConfig.getName(); - String value = secretConfig.getValue(); - if (!value.equals(redactedValueString)) { - secrets.put(name, value); - } - secretConfig.setDefaultValue(); - }); + .filter(predicate) + .forEach(secretConfig -> { + String name = secretConfig.getName(); + String value = secretConfig.getValue(); + if (!value.equals(redactedValueString)) { + secrets.put(name, value); + } + secretConfig.setDefaultValue(); + }); } } return secrets; @@ -171,9 +174,9 @@ public class Manifest extends JsonEntity { List environmentVariables = deployment.getEnvironmentVariables(); if (environmentVariables != null) { Optional anySecrets = environmentVariables - .stream() - .filter(predicate) - .findAny(); + .stream() + .filter(predicate) + .findAny(); return anySecrets.isPresent(); } } @@ -189,16 +192,16 @@ public class Manifest extends JsonEntity { List environmentVariables = deployment.getEnvironmentVariables(); if (environmentVariables != null) { environmentVariables - .stream() - .filter(predicate) - .forEach(secretConfig -> { - String name = secretConfig.getName(); - String value = secretConfig.getValue(); - if (value == null) { - value = redactedValueString; - } - secretConfig.setValue(secrets.getOrDefault(name, value)); - }); + .stream() + .filter(predicate) + .forEach(secretConfig -> { + String name = secretConfig.getName(); + String value = secretConfig.getValue(); + if (value == null) { + value = redactedValueString; + } + secretConfig.setValue(secrets.getOrDefault(name, value)); + }); } } } @@ -215,7 +218,7 @@ public class Manifest extends JsonEntity { public boolean hasEmptySecurityGroup() { Deployment deployment = this.getDeployment(); List> securityGroupObject = - (List>) deployment.getData().get("securityGroup"); + (List>) deployment.getData().get("securityGroup"); if (securityGroupObject.isEmpty()) { return true; } @@ -223,33 +226,30 @@ public class Manifest extends JsonEntity { } public void updateShaInSecretConfig(Map secrets, - Map superSecrets) { + Map superSecrets) { if (deployment != null) { List environmentVariables = deployment.getEnvironmentVariables(); if (environmentVariables != null) { environmentVariables - .stream() - .forEach(secretConfig -> { - if (secretConfig.getType().toString() - .equals(SecretConfig.ConfigType.SECRET.toString())) { - secretConfig.setSha256(Hashing.sha256() - .hashString(secrets.get(secretConfig.getName()), - StandardCharsets.UTF_8).toString()); - } else if (secretConfig.getType().toString() - .equals(SecretConfig.ConfigType.SUPER_SECRET.toString())) { - secretConfig.setSha256(Hashing.sha256() - .hashString(superSecrets.get(secretConfig.getName()), - StandardCharsets.UTF_8).toString()); - } else { - secretConfig.setSha256(Hashing.sha256() - .hashString(secretConfig.getValue(), StandardCharsets.UTF_8) - .toString()); - } - }); + .stream() + .forEach(secretConfig -> { + if (secretConfig.isSecret() && secrets.containsKey(secretConfig.getName())) { + secretConfig.setSha256(getSha256(secrets.get(secretConfig.getName()))); + } else if (secretConfig.isSuperSecret() && superSecrets.containsKey(secretConfig.getName())) { + secretConfig.setSha256(getSha256(superSecrets.get(secretConfig.getName()))); + } else if (secretConfig.isConfig()) { + secretConfig.setSha256(getSha256(secretConfig.getValue())); + } + }); } } } + private String getSha256(String value) { + return Hashing.sha256() + .hashString(value, StandardCharsets.UTF_8).toString(); + } + public void addRedactedValuesToSuperSecrets() { addRedactedValuesToSecrets(SecretConfig::isSuperSecret); } @@ -263,18 +263,26 @@ public class Manifest extends JsonEntity { List environmentVariables = deployment.getEnvironmentVariables(); if (environmentVariables != null) { environmentVariables.stream() - .filter(predicate) - .forEach(secretConfig -> { - if (secretConfig.getValue() == null) { - secretConfig.setValue(redactedValueString); - } - }); + .filter(predicate) + .forEach(secretConfig -> { + if (secretConfig.getValue() == null) { + secretConfig.setValue(redactedValueString); + } + }); } } } - public static Map convertToMap(Manifest manifest) { - ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.convertValue(manifest, Map.class); + public Map convertToMap() { + return Mapper.mapper.convertValue(this, Map.class); } + + public JsonNode convertToJson() { + return Mapper.mapper.valueToTree(this); + } + + public String convertToString() throws JsonProcessingException { + return Mapper.mapper.writeValueAsString(this); + } + } diff --git a/src/main/java/com/navi/infra/portal/service/manifest/ManifestService.java b/src/main/java/com/navi/infra/portal/service/manifest/ManifestService.java index ac1e2d61..5c0e1217 100644 --- a/src/main/java/com/navi/infra/portal/service/manifest/ManifestService.java +++ b/src/main/java/com/navi/infra/portal/service/manifest/ManifestService.java @@ -22,7 +22,6 @@ import java.util.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.commons.text.StringSubstitutor; -import org.json.simple.JSONObject; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.security.access.AccessDeniedException; @@ -53,10 +52,10 @@ public class ManifestService { private String portalVertical; ManifestService(ObjectMapper objectMapper, - ManifestRepository manifestRepository, - VaultService vaultService, - KubernetesManifestService kubernetesManifestService, - LitmusChaosService litmusChaosService, PortalEventPublisher portalEventPublisher) { + ManifestRepository manifestRepository, + VaultService vaultService, + KubernetesManifestService kubernetesManifestService, + LitmusChaosService litmusChaosService, PortalEventPublisher portalEventPublisher) { this.objectMapper = objectMapper; this.manifestRepository = manifestRepository; this.vaultService = vaultService; @@ -66,6 +65,19 @@ public class ManifestService { this.portalEventPublisher = portalEventPublisher; } + private void LogManifestDifference(Manifest newManifest, Manifest oldManifest) { + String username = AuthorizationContext.getUsername(); + Map newManifestMap = newManifest.convertToMap(); + if (oldManifest == null) { + log.info(String.format("MANIFEST CREATED by %s, createdManifest = %s", username, newManifestMap)); + } else { + Map oldManifestMap = oldManifest.convertToMap(); + log.info(String.format("MANIFEST UPDATED by %s, from = %s \t to = %s", username, + oldManifestMap, newManifestMap)); + portalEventPublisher.publishManifestUpdatedEvent(newManifest, username, oldManifestMap, newManifestMap); + } + } + //TODO: Refactor and simplify this function @PreAuthorize("hasAuthority('manifest.write')") public ManifestResponse createOrUpdate(JsonNode manifestRequest) { @@ -75,34 +87,16 @@ public class ManifestService { if (validationReport.size() != 0) { return manifestResponse; } - Manifest manifest = objectMapper.convertValue(manifestRequest, Manifest.class); - Manifest newManifest; + Manifest oldManifest = null; if (manifest.getId() != null) { - Optional oldManifest = manifestRepository.findById(manifest.getId()); - Manifest oldManifestFromDB = objectMapper.convertValue(oldManifest, Manifest.class); - newManifest = saveManifestWithoutSecrets(manifest); - Manifest newManifestFromDB = objectMapper - .convertValue(manifestRepository.findById(manifest.getId()), Manifest.class); - - Map oldManifestMap = Manifest.convertToMap(oldManifestFromDB); - Map newManifestMap = Manifest.convertToMap(newManifestFromDB); - log.info(String.format("MANIFEST UPDATED by %s, from = %s , to = %s", - AuthorizationContext.getUsername(), - oldManifestMap, - newManifestMap) - ); - portalEventPublisher - .publishManifestUpdatedEvent(newManifest, AuthorizationContext.getUsername(), - oldManifestMap, newManifestMap); - } else { - newManifest = saveManifestWithoutSecrets(manifest); - Manifest newManifestFromDB = manifestRepository.findById(manifest.getId()).get(); - log.info(String.format("MANIFEST CREATED by %s, createdManifest = %s", - AuthorizationContext.getUsername(), - new JSONObject(Manifest.convertToMap(newManifestFromDB))) - ); + oldManifest = objectMapper.convertValue(manifestRepository.findById(manifest.getId()) + , Manifest.class); } + Manifest newManifest = saveManifestWithoutSecrets(manifest); + Manifest newManifestCopy = objectMapper.convertValue(newManifest, Manifest.class); + newManifestCopy.removeSecrets(); + LogManifestDifference(newManifestCopy, oldManifest); newManifest.addRedactedValuesToSecrets(); newManifest.addRedactedValuesToSuperSecrets(); manifestResponse.setManifest(newManifest); @@ -113,7 +107,7 @@ public class ManifestService { @PreAuthorize("hasAuthority('manifest.read')") public Manifest fetchByNameAndEnvironment(String name, String environment) { Optional optionalManifest = manifestRepository - .findByNameAndEnvironment(name, environment); + .findByNameAndEnvironment(name, environment); return map(optionalManifest); } @@ -139,7 +133,7 @@ public class ManifestService { @PreAuthorize("hasAuthority('manifest.read')") public String fetchKubeObjectByName(String name, String environment, String image) { Optional optionalManifest = manifestRepository - .findByNameAndEnvironment(name, environment); + .findByNameAndEnvironment(name, environment); Manifest manifest = map(optionalManifest); return kubernetesManifestService.getKubeObjects(manifest, image).toString(); } @@ -148,7 +142,7 @@ public class ManifestService { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); try { return IOUtils.toString(classLoader.getResourceAsStream("defaults/manifest.json"), - Charset.defaultCharset()); + Charset.defaultCharset()); } catch (IOException e) { throw new RuntimeException(e); } @@ -158,36 +152,35 @@ public class ManifestService { Optional manifest = manifestRepository.findById(id); Manifest manifestCopy; try { - manifestCopy = objectMapper - .readValue(objectMapper.writeValueAsString(manifest), Manifest.class); + manifestCopy = objectMapper.convertValue(manifest, Manifest.class); manifestCopy.nullifyIds(); manifestCopy.removeFieldsBeforeCopying(); } catch (Exception e) { throw new RuntimeException( - String.format("Could not create copy of manifest because of %s", e)); + String.format("Could not create copy of manifest because of %s", e)); } return manifestCopy; } @PreAuthorize("hasAuthority('manifest.import')") public void importManifest(String deploymentManifestJson, String vaultPath, - String environment) { + String environment) { log.info( - "Importing manifest with json: {}, vaultPath: {}, environment: {} triggered by: {}", - deploymentManifestJson, - vaultPath, environment, AuthorizationContext.getUsername()); + "Importing manifest with json: {}, vaultPath: {}, environment: {} triggered by: {}", + deploymentManifestJson, + vaultPath, environment, AuthorizationContext.getUsername()); VaultResponse vaultResponse = vaultService.fetchConfig(vaultPath); final String[] manifestWrapper = new String[]{deploymentManifestJson}; if (!vaultResponse.isOk()) { throw new RuntimeException(String - .format("Unable to read secrets from vault because of: %s", vaultResponse)); + .format("Unable to read secrets from vault because of: %s", vaultResponse)); } vaultResponse.getData(). - forEach((key, value) -> manifestWrapper[0] = manifestWrapper[0] - .replaceAll(String.format("\\$%s", key), value)); + forEach((key, value) -> manifestWrapper[0] = manifestWrapper[0] + .replaceAll(String.format("\\$%s", key), value)); deploymentManifestJson = manifestWrapper[0]; try { @@ -196,8 +189,8 @@ public class ManifestService { String deploymentName = deployment.get("name").asText(); Optional byNameAndEnvironment = manifestRepository.findByNameAndEnvironment(deploymentName, environment); if (byNameAndEnvironment.isPresent()) { - throw new RuntimeException(String.format("Manifest: %s/%s already exists. Please delete and reimport", - environment, deploymentName)); + throw new RuntimeException(String.format("Manifest: %s/%s already exists. Please delete and reimport", + environment, deploymentName)); } deploymentManifest.remove("version"); @@ -222,16 +215,16 @@ public class ManifestService { database.put("readonlyUser", "${DATASOURCE_READONLY_USER}"); database.put("readonlyPassword", "${DATASOURCE_READONLY_PASSWORD}"); database.set("rdsAlertThresholds", objectMapper.readTree("{\n" + - " \"cpuUtilization\": 70,\n" + - " \"cpuCreditBalance\": 120,\n" + - " \"burstBalance\": 85,\n" + - " \"dbConnections\": 200,\n" + - " \"queueDepth\": 20,\n" + - " \"freeStorageSpacePercent\": 90,\n" + - " \"freeMemoryTooLowInMB\": 150,\n" + - " \"readLatency\": 0.5,\n" + - " \"writeLatency\": 0.5\n" + - " }")); + " \"cpuUtilization\": 70,\n" + + " \"cpuCreditBalance\": 120,\n" + + " \"burstBalance\": 85,\n" + + " \"dbConnections\": 200,\n" + + " \"queueDepth\": 20,\n" + + " \"freeStorageSpacePercent\": 90,\n" + + " \"freeMemoryTooLowInMB\": 150,\n" + + " \"readLatency\": 0.5,\n" + + " \"writeLatency\": 0.5\n" + + " }")); } @@ -261,58 +254,58 @@ public class ManifestService { } newAlerts.set("loadBalancer", objectMapper.readTree("[\n" + - " {\n" + - " \"type\": \"http4xx\",\n" + - " \"threshold\": \"15\",\n" + - " \"duration\": \"3m\",\n" + - " \"severity\": \"critical\"\n" + - " },\n" + - " {\n" + - " \"type\": \"http5xx\",\n" + - " \"threshold\": \"2\",\n" + - " \"duration\": \"3m\",\n" + - " \"severity\": \"critical\"\n" + - " },\n" + - " {\n" + - " \"type\": \"elb4xx\",\n" + - " \"threshold\": \"1\",\n" + - " \"duration\": \"3m\",\n" + - " \"severity\": \"critical\"\n" + - " },\n" + - " {\n" + - " \"type\": \"elb5xx\",\n" + - " \"threshold\": \"1\",\n" + - " \"duration\": \"3m\",\n" + - " \"severity\": \"critical\"\n" + - " },\n" + - " {\n" + - " \"type\": \"latency\",\n" + - " \"threshold\": \"800\",\n" + - " \"duration\": \"3m\",\n" + - " \"severity\": \"warning\"\n" + - " }\n" + - " ]")); + " {\n" + + " \"type\": \"http4xx\",\n" + + " \"threshold\": \"15\",\n" + + " \"duration\": \"3m\",\n" + + " \"severity\": \"critical\"\n" + + " },\n" + + " {\n" + + " \"type\": \"http5xx\",\n" + + " \"threshold\": \"2\",\n" + + " \"duration\": \"3m\",\n" + + " \"severity\": \"critical\"\n" + + " },\n" + + " {\n" + + " \"type\": \"elb4xx\",\n" + + " \"threshold\": \"1\",\n" + + " \"duration\": \"3m\",\n" + + " \"severity\": \"critical\"\n" + + " },\n" + + " {\n" + + " \"type\": \"elb5xx\",\n" + + " \"threshold\": \"1\",\n" + + " \"duration\": \"3m\",\n" + + " \"severity\": \"critical\"\n" + + " },\n" + + " {\n" + + " \"type\": \"latency\",\n" + + " \"threshold\": \"800\",\n" + + " \"duration\": \"3m\",\n" + + " \"severity\": \"warning\"\n" + + " }\n" + + " ]")); newAlerts.set("database", objectMapper.readTree("[\n" + - " {\n" + - " \"type\": \"highActiveConnection\",\n" + - " \"threshold\": \"85\",\n" + - " \"duration\": \"3m\",\n" + - " \"severity\": \"warning\"\n" + - " },\n" + - " {\n" + - " \"type\": \"connectionAcquireTimeIsHigh\",\n" + - " \"threshold\": \"5\",\n" + - " \"duration\": \"3m\",\n" + - " \"severity\": \"warning\"\n" + - " },\n" + - " {\n" + - " \"type\": \"maxConnectionPoolReached\",\n" + - " \"threshold\": \"100\",\n" + - " \"duration\": \"5m\",\n" + - " \"severity\": \"warning\"\n" + - " }\n" + - " ]")); + " {\n" + + " \"type\": \"highActiveConnection\",\n" + + " \"threshold\": \"85\",\n" + + " \"duration\": \"3m\",\n" + + " \"severity\": \"warning\"\n" + + " },\n" + + " {\n" + + " \"type\": \"connectionAcquireTimeIsHigh\",\n" + + " \"threshold\": \"5\",\n" + + " \"duration\": \"3m\",\n" + + " \"severity\": \"warning\"\n" + + " },\n" + + " {\n" + + " \"type\": \"maxConnectionPoolReached\",\n" + + " \"threshold\": \"100\",\n" + + " \"duration\": \"5m\",\n" + + " \"severity\": \"warning\"\n" + + " }\n" + + " ]")); deployment.replace("alerts", newAlerts); deployment.remove("newAlerts"); @@ -336,15 +329,15 @@ public class ManifestService { @PreAuthorize("hasAuthority('manifest.delete')") public List delete(Long id, Boolean deleteManifest) { Manifest manifest = manifestRepository.findById(id) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); List result = new ArrayList<>(); if (deleteManifest) { log.info("Deleting manifest Object - {}/{}", manifest.getEnvironment(), - manifest.getName()); + manifest.getName()); manifestRepository.deleteById(id); } else { log.info("Deleting manifest Resources for {}/{}", manifest.getEnvironment(), - manifest.getName()); + manifest.getName()); result = kubernetesManifestService.deleteResources(manifest); } return result; @@ -352,10 +345,10 @@ public class ManifestService { private Manifest map(Optional optionalManifest) { return optionalManifest - .map(this::addAllSecrets) - .map(this::makeEnvironmentSubstitution) - .map(this::setInfraVertical) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); + .map(this::addAllSecrets) + .map(this::makeEnvironmentSubstitution) + .map(this::setInfraVertical) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); } /** @@ -369,10 +362,10 @@ public class ManifestService { throw new AccessDeniedException("Unable to write secrets"); } VaultResponse vaultResponse = vaultService - .writeConfig(manifest.getSecretVaultPath(portalVertical), secrets); + .writeConfig(manifest.getSecretVaultPath(portalVertical), secrets); if (!vaultResponse.isOk()) { throw new RuntimeException(String - .format("Unable to write secrets in vault because of: %s", vaultResponse)); + .format("Unable to write secrets in vault because of: %s", vaultResponse)); } } @@ -382,18 +375,16 @@ public class ManifestService { throw new AccessDeniedException("Unable to write super secrets"); } VaultResponse vaultResponse = vaultService - .updateConfig(manifest.getSuperSecretVaultPath(portalVertical), superSecrets, - manifest.getSuperSecretKeys()); + .updateConfig(manifest.getSuperSecretVaultPath(portalVertical), superSecrets, + manifest.getSuperSecretKeys()); if (!vaultResponse.isOk()) { throw new RuntimeException(String - .format("Unable to write super secrets in vault because of: %s", - vaultResponse)); + .format("Unable to write super secrets in vault because of: %s", + vaultResponse)); } } - Map updatedSecrets = vaultService.fetchConfig(manifest.getSecretVaultPath(portalVertical)).getData(); - Map updatedSuperSecrets = vaultService.fetchConfig(manifest.getSuperSecretVaultPath(portalVertical)).getData(); - manifest.updateShaInSecretConfig(updatedSecrets,updatedSuperSecrets); + manifest.updateShaInSecretConfig((Map) (Map) secrets, (Map) (Map) superSecrets); Manifest savedManifest = manifestRepository.save(manifest); return addGivenSecrets(savedManifest, (Map) (Map) (secrets)); @@ -402,19 +393,18 @@ public class ManifestService { private Manifest addAllSecrets(Manifest manifest) { Manifest manifestDeepCopy; try { - manifestDeepCopy = objectMapper - .readValue(objectMapper.writeValueAsBytes(manifest), Manifest.class); + manifestDeepCopy = objectMapper.convertValue(manifest, Manifest.class); } catch (Exception e) { throw new RuntimeException(e); } if (manifest.hasSecrets() && AuthorizationContext.hasAuthority("secret.read")) { VaultResponse vaultResponse = - vaultService.fetchConfig(manifest.getSecretVaultPath(portalVertical)); + vaultService.fetchConfig(manifest.getSecretVaultPath(portalVertical)); if (!vaultResponse.isOk()) { throw new RuntimeException(String - .format("Unable to fetch secrets from vault, HTTP STATUS: %s", - vaultResponse.getHttpStatus())); + .format("Unable to fetch secrets from vault, HTTP STATUS: %s", + vaultResponse.getHttpStatus())); } manifestDeepCopy.addSecrets(vaultResponse.getData()); } else { @@ -423,11 +413,11 @@ public class ManifestService { if (manifest.hasSuperSecrets() && AuthorizationContext.hasAuthority("supersecret.read")) { VaultResponse vaultResponse = vaultService - .fetchConfig(manifest.getSuperSecretVaultPath(portalVertical)); + .fetchConfig(manifest.getSuperSecretVaultPath(portalVertical)); if (!vaultResponse.isOk()) { throw new RuntimeException(String - .format("Unable to fetch super secrets from vault, HTTP STATUS: %s", - vaultResponse.getHttpStatus())); + .format("Unable to fetch super secrets from vault, HTTP STATUS: %s", + vaultResponse.getHttpStatus())); } manifestDeepCopy.addSuperSecrets(vaultResponse.getData()); } else { @@ -443,8 +433,7 @@ public class ManifestService { Manifest manifestDeepCopy; try { - manifestDeepCopy = objectMapper - .readValue(objectMapper.writeValueAsBytes(manifest), Manifest.class); + manifestDeepCopy = objectMapper.convertValue(manifest, Manifest.class); } catch (Exception e) { throw new RuntimeException(e); } @@ -458,9 +447,9 @@ public class ManifestService { } try { - String manifestJson = objectMapper.writeValueAsString(manifest); + String manifestJson = manifest.convertToString(); StringSubstitutor stringSubstitutor = new StringSubstitutor( - manifest.getEnvironmentAsMap()); + manifest.getEnvironmentAsMap()); return objectMapper.readValue(stringSubstitutor.replace(manifestJson), Manifest.class); } catch (Exception e) { throw new RuntimeException(e); diff --git a/src/main/java/com/navi/infra/portal/util/Mapper.java b/src/main/java/com/navi/infra/portal/util/Mapper.java new file mode 100644 index 00000000..a906ad64 --- /dev/null +++ b/src/main/java/com/navi/infra/portal/util/Mapper.java @@ -0,0 +1,7 @@ +package com.navi.infra.portal.util; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class Mapper { + public static ObjectMapper mapper = new ObjectMapper(); +} diff --git a/src/test/java/com/navi/infra/portal/service/ManifestServiceTest.java b/src/test/java/com/navi/infra/portal/service/ManifestServiceTest.java index 3c5fac7c..15ba753c 100644 --- a/src/test/java/com/navi/infra/portal/service/ManifestServiceTest.java +++ b/src/test/java/com/navi/infra/portal/service/ManifestServiceTest.java @@ -7,11 +7,8 @@ import com.navi.infra.portal.dto.manifest.ManifestResponse; import com.navi.infra.portal.provider.ExternalIntegrationProvider; import com.navi.infra.portal.service.manifest.ManifestService; import java.io.IOException; -import javax.json.JsonObject; -import net.joshka.junit.json.params.JsonFileSource; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; import org.skyscreamer.jsonassert.JSONAssert; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.AutoConfigureTestEntityManager; @@ -35,14 +32,14 @@ public class ManifestServiceTest extends ExternalIntegrationProvider { private ObjectMapper objectMapper = OBJECT_MAPPER; - @ParameterizedTest + @Test @WithMockUser(value = "admin_user", username = "admin@navi.com", authorities = {"secret.write", "secret.read", "manifest.write", "supersecret.write", "supersecret.read", "manifest.read", "manifest.write"}, password = "admin") - @JsonFileSource(resources = "/fixtures/manifest/dev-testapp.json") @Transactional @DisplayName("Test Manifest Create and Update") - void ManifestCreateTest(JsonObject manifestData) throws IOException { + void ManifestCreateTest() throws IOException { + JsonNode manifestData = readFileToJsonNode("fixtures/manifest/dev-testapp.json"); JsonNode manifestNode = objectMapper.readTree(manifestData.toString()); ManifestResponse manifestResponse = manifestService.createOrUpdate(manifestNode); String ActualManifestResponseJson = @@ -80,7 +77,7 @@ public class ManifestServiceTest extends ExternalIntegrationProvider { Manifest manifest = objectMapper.convertValue(manifestRequest, Manifest.class); testEntityManager.persist(manifest); Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("testapp", "dev"); - String actualManifestJson = objectMapper.writeValueAsString(fetchManifest); + String actualManifestJson = fetchManifest.convertToString(); assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false)); } @@ -96,8 +93,42 @@ public class ManifestServiceTest extends ExternalIntegrationProvider { "dev-testapp-get-with-no-super-secret-access.json"); manifestService.createOrUpdate(manifestRequest); Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("testapp", "dev"); - String actualManifestJson = objectMapper.writeValueAsString(fetchManifest); - System.out.println("fetchManifest is " + actualManifestJson); + String actualManifestJson = fetchManifest.convertToString(); + assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false)); + } + + @Test + @WithMockUser(value = "read_user", username = "admin@navi.com", authorities = {"secret.write", "secret.read", + "manifest.write", "supersecret.write", "manifest.read", "manifest.write", + "supersecret.read"}, password + = "read") + @Transactional + @DisplayName("create copy manifest") + void CreateCopyManifestTest() throws IOException { + JsonNode manifestRequest = readFileToJsonNode("fixtures/manifest/dev" + + "-test-clone.json"); + String expectedManifestGetOutputJson = readFile("fixtures/manifest/expected_output/" + + "dev-test-clone-get.json"); + manifestService.createOrUpdate(manifestRequest); + Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("test-clone", "dev"); + String actualManifestJson = fetchManifest.convertToString(); + assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false)); + } + + @Test + @WithMockUser(value = "read_user", username = "admin@navi.com", authorities = {"secret.write", "secret.read", + "manifest.write", "manifest.read", "manifest.write"}, password + = "read") + @Transactional + @DisplayName("Request with ***** super secret") + void SaveManifestWithoutSuperSecretModificationTest() throws IOException { + JsonNode manifestRequest = readFileToJsonNode("fixtures/manifest/dev" + + "-test-with-reducted-secret.json"); + String expectedManifestGetOutputJson = readFile("fixtures/manifest/expected_output/" + + "dev-test-reducted-secret-get.json"); + manifestService.createOrUpdate(manifestRequest); + Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("test-clone", "dev"); + String actualManifestJson = fetchManifest.convertToString(); assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false)); } } diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index f5974132..f355f548 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -19,6 +19,11 @@ vault.kubernetes.provider="" vault.kubernetes.role="" vault.kv.mount=secret vault.kubernetes.tokenRenewCron=${VAULT_KUBE_TOKEN_CRON:0 0 0 */15 * ?} +#spring session +spring.session.store-type=jdbc +spring.session.jdbc.initialize-schema=always +# http session +server.servlet.session.timeout=${SESSION_TIMEOUT:24h} ##okta spring.security.oauth2.client.registration.okta.client-id=${OKTA_CLIENT_ID} spring.security.oauth2.client.registration.okta.client-secret=${OKTA_CLIENT_SECRET} @@ -28,4 +33,9 @@ spring.security.oauth2.client.provider.okta.token-uri=${OKTA_URL}/oauth2/v1/toke spring.security.oauth2.client.provider.okta.user-info-uri=${OKTA_URL}/oauth2/v1/userinfo spring.security.oauth2.client.provider.okta.user-name-attribute=sub spring.security.oauth2.client.provider.okta.jwk-set-uri=${OKTA_URL}/oauth2/v1/keys -portal.vertical=lending \ No newline at end of file +portal.vertical=lending +#jackson +spring.jackson.default-property-inclusion=non_null +#flyway +spring.flyway.baseline-on-migrate=true + diff --git a/src/test/resources/fixtures/manifest/dev-test-clone.json b/src/test/resources/fixtures/manifest/dev-test-clone.json new file mode 100644 index 00000000..e4e91c1a --- /dev/null +++ b/src/test/resources/fixtures/manifest/dev-test-clone.json @@ -0,0 +1,103 @@ +{ + "version": 46, + "deployment": { + "version": 46, + "loadBalancers": [], + "environmentVariables": [ + { + "name": "APP_NAME", + "type": "CONFIG", + "sha256": "4ff6303b0c4a88c0beb95a1ac1f3ab1bfefa326f74992cf42f7b9a04a5d8703d", + "value": "test1" + }, + { + "name": "test_Config", + "type": "SECRET", + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "value": "test2" + }, + { + "name": "test_super_secret", + "type": "SUPER_SECRET", + "sha256": "ddfaa92ae32b9ff82c40ce5e3350f16de528f021727f13468d9b26201905f59a", + "value": "test3" + } + ], + "alerts": { + "elb4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "elb5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "http4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "15" + }, + "http5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "2" + }, + "latency": { + "duration": "3m", + "severity": "warning", + "thresholdPercent": "800" + } + }, + "cluster": "spike.np.navi-tech.in", + "instance": { + "cpu": 0.24, + "memory": "300Mi" + }, + "exposedPorts": [ + { + "name": "serviceport", + "port": 8080 + }, + { + "name": "metrics", + "port": 4001 + } + ], + "allowEgress": [], + "healthCheck": { + "livenessCheck": { + "path": "/actuator/health", + "port": "metrics", + "type": "http", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + }, + "readinessCheck": { + "port": "serviceport", + "type": "tcp", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + } + }, + "hpa": { + "maxReplicas": 2, + "minReplicas": 2 + }, + "timeout": 1500, + "namespace": "dev" + }, + "team": { + "name": "Infra" + }, + "labels": { + "micrometer-prometheus": "enabled" + }, + "environment": "dev", + "name": "test-clone" +} \ No newline at end of file diff --git a/src/test/resources/fixtures/manifest/dev-test-with-reducted-secret.json b/src/test/resources/fixtures/manifest/dev-test-with-reducted-secret.json new file mode 100644 index 00000000..9a1c0fd9 --- /dev/null +++ b/src/test/resources/fixtures/manifest/dev-test-with-reducted-secret.json @@ -0,0 +1,103 @@ +{ + "version": 46, + "deployment": { + "version": 46, + "loadBalancers": [], + "environmentVariables": [ + { + "name": "APP_NAME", + "type": "CONFIG", + "sha256": "4ff6303b0c4a88c0beb95a1ac1f3ab1bfefa326f74992cf42f7b9a04a5d8703d", + "value": "test1" + }, + { + "name": "test_Config", + "type": "SECRET", + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "value": "test2" + }, + { + "name": "test_super_secret", + "type": "SUPER_SECRET", + "sha256": "ddfaa92ae32b9ff82c40ce5e3350f16de528f021727f13468d9b26201905f59a", + "value": "*****" + } + ], + "alerts": { + "elb4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "elb5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "http4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "15" + }, + "http5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "2" + }, + "latency": { + "duration": "3m", + "severity": "warning", + "thresholdPercent": "800" + } + }, + "cluster": "spike.np.navi-tech.in", + "instance": { + "cpu": 0.24, + "memory": "300Mi" + }, + "exposedPorts": [ + { + "name": "serviceport", + "port": 8080 + }, + { + "name": "metrics", + "port": 4001 + } + ], + "allowEgress": [], + "healthCheck": { + "livenessCheck": { + "path": "/actuator/health", + "port": "metrics", + "type": "http", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + }, + "readinessCheck": { + "port": "serviceport", + "type": "tcp", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + } + }, + "hpa": { + "maxReplicas": 2, + "minReplicas": 2 + }, + "timeout": 1500, + "namespace": "dev" + }, + "team": { + "name": "Infra" + }, + "labels": { + "micrometer-prometheus": "enabled" + }, + "environment": "dev", + "name": "test-clone" +} \ No newline at end of file diff --git a/src/test/resources/fixtures/manifest/dev-testapp.json b/src/test/resources/fixtures/manifest/dev-testapp.json index 5129971e..6746bdce 100644 --- a/src/test/resources/fixtures/manifest/dev-testapp.json +++ b/src/test/resources/fixtures/manifest/dev-testapp.json @@ -2,7 +2,6 @@ "name": "testapp", "environment": "dev", "extraResources": { - "id": 3, "aws_access": { "policies": [ { diff --git a/src/test/resources/fixtures/manifest/expected_output/dev-test-clone-get.json b/src/test/resources/fixtures/manifest/expected_output/dev-test-clone-get.json new file mode 100644 index 00000000..5d68bee8 --- /dev/null +++ b/src/test/resources/fixtures/manifest/expected_output/dev-test-clone-get.json @@ -0,0 +1,109 @@ +{ + "version": 46, + "name": "test-clone", + "environment": "dev", + "extraResources": null, + "notification": null, + "deployment": { + "version": 46, + "loadBalancers": [], + "environmentVariables": [ + { + "name": "APP_NAME", + "value": "test1", + "type": "CONFIG", + "allowEgress": null, + "sha256": "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014" + }, + { + "name": "test_Config", + "value": "test2", + "type": "SECRET", + "allowEgress": null, + "sha256": "60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752" + }, + { + "name": "test_super_secret", + "value": "test3", + "type": "SUPER_SECRET", + "allowEgress": null, + "sha256": "fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13" + } + ], + "alerts": { + "elb4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "elb5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "http4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "15" + }, + "http5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "2" + }, + "latency": { + "duration": "3m", + "severity": "warning", + "thresholdPercent": "800" + } + }, + "cluster": "spike.np.navi-tech.in", + "instance": { + "cpu": 0.24, + "memory": "300Mi" + }, + "exposedPorts": [ + { + "name": "serviceport", + "port": 8080 + }, + { + "name": "metrics", + "port": 4001 + } + ], + "allowEgress": [], + "healthCheck": { + "livenessCheck": { + "path": "/actuator/health", + "port": "metrics", + "type": "http", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + }, + "readinessCheck": { + "port": "serviceport", + "type": "tcp", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + } + }, + "hpa": { + "maxReplicas": 2, + "minReplicas": 2 + }, + "namespace": "dev", + "timeout": 1500 + }, + "infraVertical": "lending", + "team": { + "name": "Infra" + }, + "labels": { + "micrometer-prometheus": "enabled" + } +} \ No newline at end of file diff --git a/src/test/resources/fixtures/manifest/expected_output/dev-test-reducted-secret-get.json b/src/test/resources/fixtures/manifest/expected_output/dev-test-reducted-secret-get.json new file mode 100644 index 00000000..04ee5cba --- /dev/null +++ b/src/test/resources/fixtures/manifest/expected_output/dev-test-reducted-secret-get.json @@ -0,0 +1,109 @@ +{ + "version": 46, + "name": "test-clone", + "environment": "dev", + "extraResources": null, + "notification": null, + "deployment": { + "version": 46, + "loadBalancers": [], + "environmentVariables": [ + { + "name": "APP_NAME", + "value": "test1", + "type": "CONFIG", + "allowEgress": null, + "sha256": "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014" + }, + { + "name": "test_Config", + "value": "test2", + "type": "SECRET", + "allowEgress": null, + "sha256": "60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752" + }, + { + "name": "test_super_secret", + "value": "*****", + "type": "SUPER_SECRET", + "allowEgress": null, + "sha256": "ddfaa92ae32b9ff82c40ce5e3350f16de528f021727f13468d9b26201905f59a" + } + ], + "alerts": { + "elb4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "elb5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "1" + }, + "http4xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "15" + }, + "http5xx": { + "duration": "3m", + "severity": "critical", + "thresholdPercent": "2" + }, + "latency": { + "duration": "3m", + "severity": "warning", + "thresholdPercent": "800" + } + }, + "cluster": "spike.np.navi-tech.in", + "instance": { + "cpu": 0.24, + "memory": "300Mi" + }, + "exposedPorts": [ + { + "name": "serviceport", + "port": 8080 + }, + { + "name": "metrics", + "port": 4001 + } + ], + "allowEgress": [], + "healthCheck": { + "livenessCheck": { + "path": "/actuator/health", + "port": "metrics", + "type": "http", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + }, + "readinessCheck": { + "port": "serviceport", + "type": "tcp", + "periodSeconds": 30, + "failureThreshold": 5, + "successThreshold": 1, + "initialDelaySeconds": 60 + } + }, + "hpa": { + "maxReplicas": 2, + "minReplicas": 2 + }, + "namespace": "dev", + "timeout": 1500 + }, + "infraVertical": "lending", + "team": { + "name": "Infra" + }, + "labels": { + "micrometer-prometheus": "enabled" + } +} \ No newline at end of file