varnitgoyal95/deployment-portal-be
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #110 from navi-infra/INFRA-566

Browse Source 
INFRA-566 | Deepak Jain| refactoring manifest service
This commit is contained in:
Deepak Jain
2020-10-21 10:48:54 +05:30
committed by GitHub Enterprise
parent f731f2252a 404fbcc13e
commit 624ca3c9e7
11 changed files with 672 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@@ -82,7 +82,6 @@
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
</dependency>
<!-- &#45;&#45; Extra dependencies-->
<dependency>
<groupId>io.micrometer</groupId>
@@ -160,17 +159,6 @@
<groupId>org.springframework.boot</groupId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.200</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.joshka</groupId>
<artifactId>junit-json-params</artifactId>
<version>5.6.2-r0</version>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.json</artifactId>

128
src/main/java/com/navi/infra/portal/domain/manifest/Manifest.java
View File

@@ -2,10 +2,12 @@ package com.navi.infra.portal.domain.manifest;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonManagedReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.hash.Hashing;
import com.navi.infra.portal.domain.JsonEntity;
import com.navi.infra.portal.domain.notification.Notification;
import com.navi.infra.portal.util.Mapper;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.function.Predicate;
@@ -47,6 +49,7 @@ public class Manifest extends JsonEntity {
@JsonManagedReference
private Deployment deployment;
@Transient
private String infraVertical;
@@ -83,7 +86,7 @@ public class Manifest extends JsonEntity {
public String getSecretVaultPath(String portalVertical) {
if (portalVertical.equals("insurance")) {
return String.format("config/infra/gi/prod/portal-asset/secret/%s/%s", environment,
name);
name);
}
return String.format("config/infra/prod/portal-asset/secret/%s/%s", environment, name);
}
@@ -92,10 +95,10 @@ public class Manifest extends JsonEntity {
public String getSuperSecretVaultPath(String portalVertical) {
if (portalVertical.equals("insurance")) {
return String
.format("config/infra/gi/prod/portal-asset/super-secret/%s/%s", environment, name);
.format("config/infra/gi/prod/portal-asset/super-secret/%s/%s", environment, name);
}
return String
.format("config/infra/prod/portal-asset/super-secret/%s/%s", environment, name);
.format("config/infra/prod/portal-asset/super-secret/%s/%s", environment, name);
}
@JsonIgnore
@@ -105,8 +108,8 @@ public class Manifest extends JsonEntity {
List<SecretConfig> environmentVariables = deployment.getEnvironmentVariables();
if (environmentVariables != null) {
environmentVariables
.forEach(secretConfig -> environmentMap
.put(secretConfig.getName(), secretConfig.getValue()));
.forEach(secretConfig -> environmentMap
.put(secretConfig.getName(), secretConfig.getValue()));
}
}
return environmentMap;
@@ -126,9 +129,9 @@ public class Manifest extends JsonEntity {
List<SecretConfig> environmentVariables = deployment.getEnvironmentVariables();
if (environmentVariables != null) {
return environmentVariables.stream()
.filter(SecretConfig::isSuperSecret)
.map(SecretConfig::getName)
.collect(Collectors.toSet());
.filter(SecretConfig::isSuperSecret)
.map(SecretConfig::getName)
.collect(Collectors.toSet());
}
}
return new HashSet<>();
@@ -140,15 +143,15 @@ public class Manifest extends JsonEntity {
List<SecretConfig> environmentVariables = deployment.getEnvironmentVariables();
if (environmentVariables != null) {
environmentVariables.stream()
.filter(predicate)
.forEach(secretConfig -> {
String name = secretConfig.getName();
String value = secretConfig.getValue();
if (!value.equals(redactedValueString)) {
secrets.put(name, value);
}
secretConfig.setDefaultValue();
});
.filter(predicate)
.forEach(secretConfig -> {
String name = secretConfig.getName();
String value = secretConfig.getValue();
if (!value.equals(redactedValueString)) {
secrets.put(name, value);
}
secretConfig.setDefaultValue();
});
}
}
return secrets;
@@ -171,9 +174,9 @@ public class Manifest extends JsonEntity {
List<SecretConfig> environmentVariables = deployment.getEnvironmentVariables();
if (environmentVariables != null) {
Optional<SecretConfig> anySecrets = environmentVariables
.stream()
.filter(predicate)
.findAny();
.stream()
.filter(predicate)
.findAny();
return anySecrets.isPresent();
}
}
@@ -189,16 +192,16 @@ public class Manifest extends JsonEntity {
List<SecretConfig> environmentVariables = deployment.getEnvironmentVariables();
if (environmentVariables != null) {
environmentVariables
.stream()
.filter(predicate)
.forEach(secretConfig -> {
String name = secretConfig.getName();
String value = secretConfig.getValue();
if (value == null) {
value = redactedValueString;
}
secretConfig.setValue(secrets.getOrDefault(name, value));
});
.stream()
.filter(predicate)
.forEach(secretConfig -> {
String name = secretConfig.getName();
String value = secretConfig.getValue();
if (value == null) {
value = redactedValueString;
}
secretConfig.setValue(secrets.getOrDefault(name, value));
});
}
}
}
@@ -215,7 +218,7 @@ public class Manifest extends JsonEntity {
public boolean hasEmptySecurityGroup() {
Deployment deployment = this.getDeployment();
List<LinkedHashMap<String, Object>> securityGroupObject =
(List<LinkedHashMap<String, Object>>) deployment.getData().get("securityGroup");
(List<LinkedHashMap<String, Object>>) deployment.getData().get("securityGroup");
if (securityGroupObject.isEmpty()) {
return true;
}
@@ -223,33 +226,30 @@ public class Manifest extends JsonEntity {
}
public void updateShaInSecretConfig(Map<String, String> secrets,
Map<String, String> superSecrets) {
Map<String, String> superSecrets) {
if (deployment != null) {
List<SecretConfig> environmentVariables = deployment.getEnvironmentVariables();
if (environmentVariables != null) {
environmentVariables
.stream()
.forEach(secretConfig -> {
if (secretConfig.getType().toString()
.equals(SecretConfig.ConfigType.SECRET.toString())) {
secretConfig.setSha256(Hashing.sha256()
.hashString(secrets.get(secretConfig.getName()),
StandardCharsets.UTF_8).toString());
} else if (secretConfig.getType().toString()
.equals(SecretConfig.ConfigType.SUPER_SECRET.toString())) {
secretConfig.setSha256(Hashing.sha256()
.hashString(superSecrets.get(secretConfig.getName()),
StandardCharsets.UTF_8).toString());
} else {
secretConfig.setSha256(Hashing.sha256()
.hashString(secretConfig.getValue(), StandardCharsets.UTF_8)
.toString());
}
});
.stream()
.forEach(secretConfig -> {
if (secretConfig.isSecret() && secrets.containsKey(secretConfig.getName())) {
secretConfig.setSha256(getSha256(secrets.get(secretConfig.getName())));
} else if (secretConfig.isSuperSecret() && superSecrets.containsKey(secretConfig.getName())) {
secretConfig.setSha256(getSha256(superSecrets.get(secretConfig.getName())));
} else if (secretConfig.isConfig()) {
secretConfig.setSha256(getSha256(secretConfig.getValue()));
}
});
}
}
}
private String getSha256(String value) {
return Hashing.sha256()
.hashString(value, StandardCharsets.UTF_8).toString();
}
public void addRedactedValuesToSuperSecrets() {
addRedactedValuesToSecrets(SecretConfig::isSuperSecret);
}
@@ -263,18 +263,26 @@ public class Manifest extends JsonEntity {
List<SecretConfig> environmentVariables = deployment.getEnvironmentVariables();
if (environmentVariables != null) {
environmentVariables.stream()
.filter(predicate)
.forEach(secretConfig -> {
if (secretConfig.getValue() == null) {
secretConfig.setValue(redactedValueString);
}
});
.filter(predicate)
.forEach(secretConfig -> {
if (secretConfig.getValue() == null) {
secretConfig.setValue(redactedValueString);
}
});
}
}
}
public static Map<String, Object> convertToMap(Manifest manifest) {
ObjectMapper objectMapper = new ObjectMapper();
return objectMapper.convertValue(manifest, Map.class);
public Map<String, Object> convertToMap() {
return Mapper.mapper.convertValue(this, Map.class);
}
public JsonNode convertToJson() {
return Mapper.mapper.valueToTree(this);
}
public String convertToString() throws JsonProcessingException {
return Mapper.mapper.writeValueAsString(this);
}
}

255
src/main/java/com/navi/infra/portal/service/manifest/ManifestService.java
View File

@@ -22,7 +22,6 @@ import java.util.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.apache.commons.text.StringSubstitutor;
import org.json.simple.JSONObject;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.security.access.AccessDeniedException;
@@ -53,10 +52,10 @@ public class ManifestService {
private String portalVertical;
ManifestService(ObjectMapper objectMapper,
ManifestRepository manifestRepository,
VaultService vaultService,
KubernetesManifestService kubernetesManifestService,
LitmusChaosService litmusChaosService, PortalEventPublisher portalEventPublisher) {
ManifestRepository manifestRepository,
VaultService vaultService,
KubernetesManifestService kubernetesManifestService,
LitmusChaosService litmusChaosService, PortalEventPublisher portalEventPublisher) {
this.objectMapper = objectMapper;
this.manifestRepository = manifestRepository;
this.vaultService = vaultService;
@@ -66,6 +65,19 @@ public class ManifestService {
this.portalEventPublisher = portalEventPublisher;
}
private void LogManifestDifference(Manifest newManifest, Manifest oldManifest) {
String username = AuthorizationContext.getUsername();
Map<String, Object> newManifestMap = newManifest.convertToMap();
if (oldManifest == null) {
log.info(String.format("MANIFEST CREATED by %s, createdManifest = %s", username, newManifestMap));
} else {
Map<String, Object> oldManifestMap = oldManifest.convertToMap();
log.info(String.format("MANIFEST UPDATED by %s, from = %s \t to = %s", username,
oldManifestMap, newManifestMap));
portalEventPublisher.publishManifestUpdatedEvent(newManifest, username, oldManifestMap, newManifestMap);
}
}
//TODO: Refactor and simplify this function
@PreAuthorize("hasAuthority('manifest.write')")
public ManifestResponse createOrUpdate(JsonNode manifestRequest) {
@@ -75,34 +87,16 @@ public class ManifestService {
if (validationReport.size() != 0) {
return manifestResponse;
}
Manifest manifest = objectMapper.convertValue(manifestRequest, Manifest.class);
Manifest newManifest;
Manifest oldManifest = null;
if (manifest.getId() != null) {
Optional<Manifest> oldManifest = manifestRepository.findById(manifest.getId());
Manifest oldManifestFromDB = objectMapper.convertValue(oldManifest, Manifest.class);
newManifest = saveManifestWithoutSecrets(manifest);
Manifest newManifestFromDB = objectMapper
.convertValue(manifestRepository.findById(manifest.getId()), Manifest.class);
Map<String, Object> oldManifestMap = Manifest.convertToMap(oldManifestFromDB);
Map<String, Object> newManifestMap = Manifest.convertToMap(newManifestFromDB);
log.info(String.format("MANIFEST UPDATED by %s, from = %s , to = %s",
AuthorizationContext.getUsername(),
oldManifestMap,
newManifestMap)
);
portalEventPublisher
.publishManifestUpdatedEvent(newManifest, AuthorizationContext.getUsername(),
oldManifestMap, newManifestMap);
} else {
newManifest = saveManifestWithoutSecrets(manifest);
Manifest newManifestFromDB = manifestRepository.findById(manifest.getId()).get();
log.info(String.format("MANIFEST CREATED by %s, createdManifest = %s",
AuthorizationContext.getUsername(),
new JSONObject(Manifest.convertToMap(newManifestFromDB)))
);
oldManifest = objectMapper.convertValue(manifestRepository.findById(manifest.getId())
, Manifest.class);
}
Manifest newManifest = saveManifestWithoutSecrets(manifest);
Manifest newManifestCopy = objectMapper.convertValue(newManifest, Manifest.class);
newManifestCopy.removeSecrets();
LogManifestDifference(newManifestCopy, oldManifest);
newManifest.addRedactedValuesToSecrets();
newManifest.addRedactedValuesToSuperSecrets();
manifestResponse.setManifest(newManifest);
@@ -113,7 +107,7 @@ public class ManifestService {
@PreAuthorize("hasAuthority('manifest.read')")
public Manifest fetchByNameAndEnvironment(String name, String environment) {
Optional<Manifest> optionalManifest = manifestRepository
.findByNameAndEnvironment(name, environment);
.findByNameAndEnvironment(name, environment);
return map(optionalManifest);
}
@@ -139,7 +133,7 @@ public class ManifestService {
@PreAuthorize("hasAuthority('manifest.read')")
public String fetchKubeObjectByName(String name, String environment, String image) {
Optional<Manifest> optionalManifest = manifestRepository
.findByNameAndEnvironment(name, environment);
.findByNameAndEnvironment(name, environment);
Manifest manifest = map(optionalManifest);
return kubernetesManifestService.getKubeObjects(manifest, image).toString();
}
@@ -148,7 +142,7 @@ public class ManifestService {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
try {
return IOUtils.toString(classLoader.getResourceAsStream("defaults/manifest.json"),
Charset.defaultCharset());
Charset.defaultCharset());
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -158,36 +152,35 @@ public class ManifestService {
Optional<Manifest> manifest = manifestRepository.findById(id);
Manifest manifestCopy;
try {
manifestCopy = objectMapper
.readValue(objectMapper.writeValueAsString(manifest), Manifest.class);
manifestCopy = objectMapper.convertValue(manifest, Manifest.class);
manifestCopy.nullifyIds();
manifestCopy.removeFieldsBeforeCopying();
} catch (Exception e) {
throw new RuntimeException(
String.format("Could not create copy of manifest because of %s", e));
String.format("Could not create copy of manifest because of %s", e));
}
return manifestCopy;
}
@PreAuthorize("hasAuthority('manifest.import')")
public void importManifest(String deploymentManifestJson, String vaultPath,
String environment) {
String environment) {
log.info(
"Importing manifest with json: {}, vaultPath: {}, environment: {} triggered by: {}",
deploymentManifestJson,
vaultPath, environment, AuthorizationContext.getUsername());
"Importing manifest with json: {}, vaultPath: {}, environment: {} triggered by: {}",
deploymentManifestJson,
vaultPath, environment, AuthorizationContext.getUsername());
VaultResponse vaultResponse = vaultService.fetchConfig(vaultPath);
final String[] manifestWrapper = new String[]{deploymentManifestJson};
if (!vaultResponse.isOk()) {
throw new RuntimeException(String
.format("Unable to read secrets from vault because of: %s", vaultResponse));
.format("Unable to read secrets from vault because of: %s", vaultResponse));
}
vaultResponse.getData().
forEach((key, value) -> manifestWrapper[0] = manifestWrapper[0]
.replaceAll(String.format("\\$%s", key), value));
forEach((key, value) -> manifestWrapper[0] = manifestWrapper[0]
.replaceAll(String.format("\\$%s", key), value));
deploymentManifestJson = manifestWrapper[0];
try {
@@ -196,8 +189,8 @@ public class ManifestService {
String deploymentName = deployment.get("name").asText();
Optional<Manifest> byNameAndEnvironment = manifestRepository.findByNameAndEnvironment(deploymentName, environment);
if (byNameAndEnvironment.isPresent()) {
throw new RuntimeException(String.format("Manifest: %s/%s already exists. Please delete and reimport",
environment, deploymentName));
throw new RuntimeException(String.format("Manifest: %s/%s already exists. Please delete and reimport",
environment, deploymentName));
}
deploymentManifest.remove("version");
@@ -222,16 +215,16 @@ public class ManifestService {
database.put("readonlyUser", "${DATASOURCE_READONLY_USER}");
database.put("readonlyPassword", "${DATASOURCE_READONLY_PASSWORD}");
database.set("rdsAlertThresholds", objectMapper.readTree("{\n" +
" \"cpuUtilization\": 70,\n" +
" \"cpuCreditBalance\": 120,\n" +
" \"burstBalance\": 85,\n" +
" \"dbConnections\": 200,\n" +
" \"queueDepth\": 20,\n" +
" \"freeStorageSpacePercent\": 90,\n" +
" \"freeMemoryTooLowInMB\": 150,\n" +
" \"readLatency\": 0.5,\n" +
" \"writeLatency\": 0.5\n" +
" }"));
" \"cpuUtilization\": 70,\n" +
" \"cpuCreditBalance\": 120,\n" +
" \"burstBalance\": 85,\n" +
" \"dbConnections\": 200,\n" +
" \"queueDepth\": 20,\n" +
" \"freeStorageSpacePercent\": 90,\n" +
" \"freeMemoryTooLowInMB\": 150,\n" +
" \"readLatency\": 0.5,\n" +
" \"writeLatency\": 0.5\n" +
" }"));
}
@@ -261,58 +254,58 @@ public class ManifestService {
}
newAlerts.set("loadBalancer", objectMapper.readTree("[\n" +
" {\n" +
" \"type\": \"http4xx\",\n" +
" \"threshold\": \"15\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"http5xx\",\n" +
" \"threshold\": \"2\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"elb4xx\",\n" +
" \"threshold\": \"1\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"elb5xx\",\n" +
" \"threshold\": \"1\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"latency\",\n" +
" \"threshold\": \"800\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"warning\"\n" +
" }\n" +
" ]"));
" {\n" +
" \"type\": \"http4xx\",\n" +
" \"threshold\": \"15\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"http5xx\",\n" +
" \"threshold\": \"2\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"elb4xx\",\n" +
" \"threshold\": \"1\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"elb5xx\",\n" +
" \"threshold\": \"1\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"critical\"\n" +
" },\n" +
" {\n" +
" \"type\": \"latency\",\n" +
" \"threshold\": \"800\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"warning\"\n" +
" }\n" +
" ]"));
newAlerts.set("database", objectMapper.readTree("[\n" +
" {\n" +
" \"type\": \"highActiveConnection\",\n" +
" \"threshold\": \"85\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"warning\"\n" +
" },\n" +
" {\n" +
" \"type\": \"connectionAcquireTimeIsHigh\",\n" +
" \"threshold\": \"5\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"warning\"\n" +
" },\n" +
" {\n" +
" \"type\": \"maxConnectionPoolReached\",\n" +
" \"threshold\": \"100\",\n" +
" \"duration\": \"5m\",\n" +
" \"severity\": \"warning\"\n" +
" }\n" +
" ]"));
" {\n" +
" \"type\": \"highActiveConnection\",\n" +
" \"threshold\": \"85\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"warning\"\n" +
" },\n" +
" {\n" +
" \"type\": \"connectionAcquireTimeIsHigh\",\n" +
" \"threshold\": \"5\",\n" +
" \"duration\": \"3m\",\n" +
" \"severity\": \"warning\"\n" +
" },\n" +
" {\n" +
" \"type\": \"maxConnectionPoolReached\",\n" +
" \"threshold\": \"100\",\n" +
" \"duration\": \"5m\",\n" +
" \"severity\": \"warning\"\n" +
" }\n" +
" ]"));
deployment.replace("alerts", newAlerts);
deployment.remove("newAlerts");
@@ -336,15 +329,15 @@ public class ManifestService {
@PreAuthorize("hasAuthority('manifest.delete')")
public List<String> delete(Long id, Boolean deleteManifest) {
Manifest manifest = manifestRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
List<String> result = new ArrayList<>();
if (deleteManifest) {
log.info("Deleting manifest Object - {}/{}", manifest.getEnvironment(),
manifest.getName());
manifest.getName());
manifestRepository.deleteById(id);
} else {
log.info("Deleting manifest Resources for {}/{}", manifest.getEnvironment(),
manifest.getName());
manifest.getName());
result = kubernetesManifestService.deleteResources(manifest);
}
return result;
@@ -352,10 +345,10 @@ public class ManifestService {
private Manifest map(Optional<Manifest> optionalManifest) {
return optionalManifest
.map(this::addAllSecrets)
.map(this::makeEnvironmentSubstitution)
.map(this::setInfraVertical)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
.map(this::addAllSecrets)
.map(this::makeEnvironmentSubstitution)
.map(this::setInfraVertical)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
}
/**
@@ -369,10 +362,10 @@ public class ManifestService {
throw new AccessDeniedException("Unable to write secrets");
}
VaultResponse vaultResponse = vaultService
.writeConfig(manifest.getSecretVaultPath(portalVertical), secrets);
.writeConfig(manifest.getSecretVaultPath(portalVertical), secrets);
if (!vaultResponse.isOk()) {
throw new RuntimeException(String
.format("Unable to write secrets in vault because of: %s", vaultResponse));
.format("Unable to write secrets in vault because of: %s", vaultResponse));
}
}
@@ -382,18 +375,16 @@ public class ManifestService {
throw new AccessDeniedException("Unable to write super secrets");
}
VaultResponse vaultResponse = vaultService
.updateConfig(manifest.getSuperSecretVaultPath(portalVertical), superSecrets,
manifest.getSuperSecretKeys());
.updateConfig(manifest.getSuperSecretVaultPath(portalVertical), superSecrets,
manifest.getSuperSecretKeys());
if (!vaultResponse.isOk()) {
throw new RuntimeException(String
.format("Unable to write super secrets in vault because of: %s",
vaultResponse));
.format("Unable to write super secrets in vault because of: %s",
vaultResponse));
}
}
Map<String,String> updatedSecrets = vaultService.fetchConfig(manifest.getSecretVaultPath(portalVertical)).getData();
Map<String,String> updatedSuperSecrets = vaultService.fetchConfig(manifest.getSuperSecretVaultPath(portalVertical)).getData();
manifest.updateShaInSecretConfig(updatedSecrets,updatedSuperSecrets);
manifest.updateShaInSecretConfig((Map<String, String>) (Map) secrets, (Map<String, String>) (Map) superSecrets);
Manifest savedManifest = manifestRepository.save(manifest);
return addGivenSecrets(savedManifest, (Map<String, String>) (Map) (secrets));
@@ -402,19 +393,18 @@ public class ManifestService {
private Manifest addAllSecrets(Manifest manifest) {
Manifest manifestDeepCopy;
try {
manifestDeepCopy = objectMapper
.readValue(objectMapper.writeValueAsBytes(manifest), Manifest.class);
manifestDeepCopy = objectMapper.convertValue(manifest, Manifest.class);
} catch (Exception e) {
throw new RuntimeException(e);
}
if (manifest.hasSecrets() && AuthorizationContext.hasAuthority("secret.read")) {
VaultResponse vaultResponse =
vaultService.fetchConfig(manifest.getSecretVaultPath(portalVertical));
vaultService.fetchConfig(manifest.getSecretVaultPath(portalVertical));
if (!vaultResponse.isOk()) {
throw new RuntimeException(String
.format("Unable to fetch secrets from vault, HTTP STATUS: %s",
vaultResponse.getHttpStatus()));
.format("Unable to fetch secrets from vault, HTTP STATUS: %s",
vaultResponse.getHttpStatus()));
}
manifestDeepCopy.addSecrets(vaultResponse.getData());
} else {
@@ -423,11 +413,11 @@ public class ManifestService {
if (manifest.hasSuperSecrets() && AuthorizationContext.hasAuthority("supersecret.read")) {
VaultResponse vaultResponse = vaultService
.fetchConfig(manifest.getSuperSecretVaultPath(portalVertical));
.fetchConfig(manifest.getSuperSecretVaultPath(portalVertical));
if (!vaultResponse.isOk()) {
throw new RuntimeException(String
.format("Unable to fetch super secrets from vault, HTTP STATUS: %s",
vaultResponse.getHttpStatus()));
.format("Unable to fetch super secrets from vault, HTTP STATUS: %s",
vaultResponse.getHttpStatus()));
}
manifestDeepCopy.addSuperSecrets(vaultResponse.getData());
} else {
@@ -443,8 +433,7 @@ public class ManifestService {
Manifest manifestDeepCopy;
try {
manifestDeepCopy = objectMapper
.readValue(objectMapper.writeValueAsBytes(manifest), Manifest.class);
manifestDeepCopy = objectMapper.convertValue(manifest, Manifest.class);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -458,9 +447,9 @@ public class ManifestService {
}
try {
String manifestJson = objectMapper.writeValueAsString(manifest);
String manifestJson = manifest.convertToString();
StringSubstitutor stringSubstitutor = new StringSubstitutor(
manifest.getEnvironmentAsMap());
manifest.getEnvironmentAsMap());
return objectMapper.readValue(stringSubstitutor.replace(manifestJson), Manifest.class);
} catch (Exception e) {
throw new RuntimeException(e);

7
src/main/java/com/navi/infra/portal/util/Mapper.java Normal file
View File

@@ -0,0 +1,7 @@
package com.navi.infra.portal.util;
import com.fasterxml.jackson.databind.ObjectMapper;
public class Mapper {
public static ObjectMapper mapper = new ObjectMapper();
}

49
src/test/java/com/navi/infra/portal/service/ManifestServiceTest.java
View File

@@ -7,11 +7,8 @@ import com.navi.infra.portal.dto.manifest.ManifestResponse;
import com.navi.infra.portal.provider.ExternalIntegrationProvider;
import com.navi.infra.portal.service.manifest.ManifestService;
import java.io.IOException;
import javax.json.JsonObject;
import net.joshka.junit.json.params.JsonFileSource;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.skyscreamer.jsonassert.JSONAssert;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.orm.jpa.AutoConfigureTestEntityManager;
@@ -35,14 +32,14 @@ public class ManifestServiceTest extends ExternalIntegrationProvider {
private ObjectMapper objectMapper = OBJECT_MAPPER;
@ParameterizedTest
@Test
@WithMockUser(value = "admin_user", username = "admin@navi.com", authorities = {"secret.write", "secret.read",
"manifest.write", "supersecret.write", "supersecret.read", "manifest.read", "manifest.write"}, password =
"admin")
@JsonFileSource(resources = "/fixtures/manifest/dev-testapp.json")
@Transactional
@DisplayName("Test Manifest Create and Update")
void ManifestCreateTest(JsonObject manifestData) throws IOException {
void ManifestCreateTest() throws IOException {
JsonNode manifestData = readFileToJsonNode("fixtures/manifest/dev-testapp.json");
JsonNode manifestNode = objectMapper.readTree(manifestData.toString());
ManifestResponse manifestResponse = manifestService.createOrUpdate(manifestNode);
String ActualManifestResponseJson =
@@ -80,7 +77,7 @@ public class ManifestServiceTest extends ExternalIntegrationProvider {
Manifest manifest = objectMapper.convertValue(manifestRequest, Manifest.class);
testEntityManager.persist(manifest);
Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("testapp", "dev");
String actualManifestJson = objectMapper.writeValueAsString(fetchManifest);
String actualManifestJson = fetchManifest.convertToString();
assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false));
}
@@ -96,8 +93,42 @@ public class ManifestServiceTest extends ExternalIntegrationProvider {
"dev-testapp-get-with-no-super-secret-access.json");
manifestService.createOrUpdate(manifestRequest);
Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("testapp", "dev");
String actualManifestJson = objectMapper.writeValueAsString(fetchManifest);
System.out.println("fetchManifest is " + actualManifestJson);
String actualManifestJson = fetchManifest.convertToString();
assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false));
}
@Test
@WithMockUser(value = "read_user", username = "admin@navi.com", authorities = {"secret.write", "secret.read",
"manifest.write", "supersecret.write", "manifest.read", "manifest.write",
"supersecret.read"}, password
= "read")
@Transactional
@DisplayName("create copy manifest")
void CreateCopyManifestTest() throws IOException {
JsonNode manifestRequest = readFileToJsonNode("fixtures/manifest/dev" +
"-test-clone.json");
String expectedManifestGetOutputJson = readFile("fixtures/manifest/expected_output/" +
"dev-test-clone-get.json");
manifestService.createOrUpdate(manifestRequest);
Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("test-clone", "dev");
String actualManifestJson = fetchManifest.convertToString();
assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false));
}
@Test
@WithMockUser(value = "read_user", username = "admin@navi.com", authorities = {"secret.write", "secret.read",
"manifest.write", "manifest.read", "manifest.write"}, password
= "read")
@Transactional
@DisplayName("Request with ***** super secret")
void SaveManifestWithoutSuperSecretModificationTest() throws IOException {
JsonNode manifestRequest = readFileToJsonNode("fixtures/manifest/dev" +
"-test-with-reducted-secret.json");
String expectedManifestGetOutputJson = readFile("fixtures/manifest/expected_output/" +
"dev-test-reducted-secret-get.json");
manifestService.createOrUpdate(manifestRequest);
Manifest fetchManifest = manifestService.fetchByNameAndEnvironment("test-clone", "dev");
String actualManifestJson = fetchManifest.convertToString();
assertAll(() -> JSONAssert.assertEquals(expectedManifestGetOutputJson, actualManifestJson, false));
}
}

12
src/test/resources/application.properties
View File

@@ -19,6 +19,11 @@ vault.kubernetes.provider=""
vault.kubernetes.role=""
vault.kv.mount=secret
vault.kubernetes.tokenRenewCron=${VAULT_KUBE_TOKEN_CRON:0 0 0 */15 * ?}
#spring session
spring.session.store-type=jdbc
spring.session.jdbc.initialize-schema=always
# http session
server.servlet.session.timeout=${SESSION_TIMEOUT:24h}
##okta
spring.security.oauth2.client.registration.okta.client-id=${OKTA_CLIENT_ID}
spring.security.oauth2.client.registration.okta.client-secret=${OKTA_CLIENT_SECRET}
@@ -28,4 +33,9 @@ spring.security.oauth2.client.provider.okta.token-uri=${OKTA_URL}/oauth2/v1/toke
spring.security.oauth2.client.provider.okta.user-info-uri=${OKTA_URL}/oauth2/v1/userinfo
spring.security.oauth2.client.provider.okta.user-name-attribute=sub
spring.security.oauth2.client.provider.okta.jwk-set-uri=${OKTA_URL}/oauth2/v1/keys
portal.vertical=lending
portal.vertical=lending
#jackson
spring.jackson.default-property-inclusion=non_null
#flyway
spring.flyway.baseline-on-migrate=true

103
src/test/resources/fixtures/manifest/dev-test-clone.json Normal file
View File

@@ -0,0 +1,103 @@
{
"version": 46,
"deployment": {
"version": 46,
"loadBalancers": [],
"environmentVariables": [
{
"name": "APP_NAME",
"type": "CONFIG",
"sha256": "4ff6303b0c4a88c0beb95a1ac1f3ab1bfefa326f74992cf42f7b9a04a5d8703d",
"value": "test1"
},
{
"name": "test_Config",
"type": "SECRET",
"sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
"value": "test2"
},
{
"name": "test_super_secret",
"type": "SUPER_SECRET",
"sha256": "ddfaa92ae32b9ff82c40ce5e3350f16de528f021727f13468d9b26201905f59a",
"value": "test3"
}
],
"alerts": {
"elb4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"elb5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"http4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "15"
},
"http5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "2"
},
"latency": {
"duration": "3m",
"severity": "warning",
"thresholdPercent": "800"
}
},
"cluster": "spike.np.navi-tech.in",
"instance": {
"cpu": 0.24,
"memory": "300Mi"
},
"exposedPorts": [
{
"name": "serviceport",
"port": 8080
},
{
"name": "metrics",
"port": 4001
}
],
"allowEgress": [],
"healthCheck": {
"livenessCheck": {
"path": "/actuator/health",
"port": "metrics",
"type": "http",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
},
"readinessCheck": {
"port": "serviceport",
"type": "tcp",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
}
},
"hpa": {
"maxReplicas": 2,
"minReplicas": 2
},
"timeout": 1500,
"namespace": "dev"
},
"team": {
"name": "Infra"
},
"labels": {
"micrometer-prometheus": "enabled"
},
"environment": "dev",
"name": "test-clone"
}

103
src/test/resources/fixtures/manifest/dev-test-with-reducted-secret.json Normal file
View File

@@ -0,0 +1,103 @@
{
"version": 46,
"deployment": {
"version": 46,
"loadBalancers": [],
"environmentVariables": [
{
"name": "APP_NAME",
"type": "CONFIG",
"sha256": "4ff6303b0c4a88c0beb95a1ac1f3ab1bfefa326f74992cf42f7b9a04a5d8703d",
"value": "test1"
},
{
"name": "test_Config",
"type": "SECRET",
"sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
"value": "test2"
},
{
"name": "test_super_secret",
"type": "SUPER_SECRET",
"sha256": "ddfaa92ae32b9ff82c40ce5e3350f16de528f021727f13468d9b26201905f59a",
"value": "*****"
}
],
"alerts": {
"elb4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"elb5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"http4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "15"
},
"http5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "2"
},
"latency": {
"duration": "3m",
"severity": "warning",
"thresholdPercent": "800"
}
},
"cluster": "spike.np.navi-tech.in",
"instance": {
"cpu": 0.24,
"memory": "300Mi"
},
"exposedPorts": [
{
"name": "serviceport",
"port": 8080
},
{
"name": "metrics",
"port": 4001
}
],
"allowEgress": [],
"healthCheck": {
"livenessCheck": {
"path": "/actuator/health",
"port": "metrics",
"type": "http",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
},
"readinessCheck": {
"port": "serviceport",
"type": "tcp",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
}
},
"hpa": {
"maxReplicas": 2,
"minReplicas": 2
},
"timeout": 1500,
"namespace": "dev"
},
"team": {
"name": "Infra"
},
"labels": {
"micrometer-prometheus": "enabled"
},
"environment": "dev",
"name": "test-clone"
}

1
src/test/resources/fixtures/manifest/dev-testapp.json
View File

@@ -2,7 +2,6 @@
"name": "testapp",
"environment": "dev",
"extraResources": {
"id": 3,
"aws_access": {
"policies": [
{

109
src/test/resources/fixtures/manifest/expected_output/dev-test-clone-get.json Normal file
View File

@@ -0,0 +1,109 @@
{
"version": 46,
"name": "test-clone",
"environment": "dev",
"extraResources": null,
"notification": null,
"deployment": {
"version": 46,
"loadBalancers": [],
"environmentVariables": [
{
"name": "APP_NAME",
"value": "test1",
"type": "CONFIG",
"allowEgress": null,
"sha256": "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014"
},
{
"name": "test_Config",
"value": "test2",
"type": "SECRET",
"allowEgress": null,
"sha256": "60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752"
},
{
"name": "test_super_secret",
"value": "test3",
"type": "SUPER_SECRET",
"allowEgress": null,
"sha256": "fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13"
}
],
"alerts": {
"elb4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"elb5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"http4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "15"
},
"http5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "2"
},
"latency": {
"duration": "3m",
"severity": "warning",
"thresholdPercent": "800"
}
},
"cluster": "spike.np.navi-tech.in",
"instance": {
"cpu": 0.24,
"memory": "300Mi"
},
"exposedPorts": [
{
"name": "serviceport",
"port": 8080
},
{
"name": "metrics",
"port": 4001
}
],
"allowEgress": [],
"healthCheck": {
"livenessCheck": {
"path": "/actuator/health",
"port": "metrics",
"type": "http",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
},
"readinessCheck": {
"port": "serviceport",
"type": "tcp",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
}
},
"hpa": {
"maxReplicas": 2,
"minReplicas": 2
},
"namespace": "dev",
"timeout": 1500
},
"infraVertical": "lending",
"team": {
"name": "Infra"
},
"labels": {
"micrometer-prometheus": "enabled"
}
}

109
src/test/resources/fixtures/manifest/expected_output/dev-test-reducted-secret-get.json Normal file
View File

@@ -0,0 +1,109 @@
{
"version": 46,
"name": "test-clone",
"environment": "dev",
"extraResources": null,
"notification": null,
"deployment": {
"version": 46,
"loadBalancers": [],
"environmentVariables": [
{
"name": "APP_NAME",
"value": "test1",
"type": "CONFIG",
"allowEgress": null,
"sha256": "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014"
},
{
"name": "test_Config",
"value": "test2",
"type": "SECRET",
"allowEgress": null,
"sha256": "60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752"
},
{
"name": "test_super_secret",
"value": "*****",
"type": "SUPER_SECRET",
"allowEgress": null,
"sha256": "ddfaa92ae32b9ff82c40ce5e3350f16de528f021727f13468d9b26201905f59a"
}
],
"alerts": {
"elb4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"elb5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "1"
},
"http4xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "15"
},
"http5xx": {
"duration": "3m",
"severity": "critical",
"thresholdPercent": "2"
},
"latency": {
"duration": "3m",
"severity": "warning",
"thresholdPercent": "800"
}
},
"cluster": "spike.np.navi-tech.in",
"instance": {
"cpu": 0.24,
"memory": "300Mi"
},
"exposedPorts": [
{
"name": "serviceport",
"port": 8080
},
{
"name": "metrics",
"port": 4001
}
],
"allowEgress": [],
"healthCheck": {
"livenessCheck": {
"path": "/actuator/health",
"port": "metrics",
"type": "http",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
},
"readinessCheck": {
"port": "serviceport",
"type": "tcp",
"periodSeconds": 30,
"failureThreshold": 5,
"successThreshold": 1,
"initialDelaySeconds": 60
}
},
"hpa": {
"maxReplicas": 2,
"minReplicas": 2
},
"namespace": "dev",
"timeout": 1500
},
"infraVertical": "lending",
"team": {
"name": "Infra"
},
"labels": {
"micrometer-prometheus": "enabled"
}
}