diff --git a/pom.xml b/pom.xml index 07f1f4c7..03dbcd27 100644 --- a/pom.xml +++ b/pom.xml @@ -138,6 +138,16 @@ httpclient 4.5.13 + + kotlin-stdlib + org.jetbrains.kotlin + 2.0.0 + + + com.spotify + github-client + 0.2.17 + httpmime org.apache.httpcomponents diff --git a/src/main/java/com/navi/infra/portal/configuration/GitHubClientConfig.java b/src/main/java/com/navi/infra/portal/configuration/GitHubClientConfig.java new file mode 100644 index 00000000..b768640d --- /dev/null +++ b/src/main/java/com/navi/infra/portal/configuration/GitHubClientConfig.java @@ -0,0 +1,16 @@ +package com.navi.infra.portal.configuration; + +import com.spotify.github.v3.clients.GitHubClient; +import java.net.URI; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class GitHubClientConfig { + + @Bean + public GitHubClient githubClient(@Value("${github.token}") String authToken) { + return GitHubClient.create(URI.create("https://api.github.com/"), authToken); + } +} \ No newline at end of file diff --git a/src/main/java/com/navi/infra/portal/service/gocd/PipelineManifestService.java b/src/main/java/com/navi/infra/portal/service/gocd/PipelineManifestService.java index 24d6e944..d002e552 100644 --- a/src/main/java/com/navi/infra/portal/service/gocd/PipelineManifestService.java +++ b/src/main/java/com/navi/infra/portal/service/gocd/PipelineManifestService.java @@ -18,11 +18,13 @@ import com.navi.infra.portal.util.KubernetesManifestGenerator; import com.navi.infra.portal.util.MapDiffUtil; import com.navi.infra.portal.util.gocd.GocdConfigValidatorUtil; import com.navi.infra.portal.util.gocd.PipelineValidatorUtil; -import com.navi.infra.portal.v2.client.github.GitHubApiRequest; -import com.navi.infra.portal.v2.client.github.GitHubApiResponse; -import com.navi.infra.portal.v2.client.github.GitHubClient; import com.navi.infra.portal.v2.exception.GocdValidationException; import com.navi.infra.portal.v2.exception.PipelineDifferenceException; +import com.spotify.github.v3.clients.GitHubClient; +import com.spotify.github.v3.clients.RepositoryClient; +import com.spotify.github.v3.repos.Content; +import com.spotify.github.v3.repos.requests.FileCreate; +import com.spotify.github.v3.repos.requests.ImmutableFileCreate; import com.sun.jdi.request.InvalidRequestStateException; import java.io.ByteArrayOutputStream; import java.io.File; @@ -34,7 +36,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.time.LocalDateTime; import java.util.Map; -import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -49,9 +50,11 @@ public class PipelineManifestService { private final String PIPELINE_PATH = "pipelines"; private final String PIPELINE_MANIFEST_FILE_NAME = "pipeline_manifest.json"; private final String PIPELINE_FILE_NAME = "pipelines.gocd.yaml"; - private final String orgName; + + private final String repository = "test-github-action-deployment-portal-backend"; private final ObjectMapper objectMapper; private final ObjectMapper yamlMapper; + private final RepositoryClient repositoryClient; private final PipelineManifestRepository pipelineManifestRepository; private final KubernetesManifestGenerator kubernetesManifestGenerator; @@ -60,16 +63,15 @@ public class PipelineManifestService { private final GocdConfigValidatorUtil gocdConfigValidatorUtil; private final PipelineValidatorUtil pipelineValidatorUtil; private final MapDiffUtil mapDiffUtil; - private final GitHubClient gitHubClient; PipelineManifestService( ObjectMapper objectMapper, @Qualifier("yamlMapper") ObjectMapper yamlMapper, + GitHubClient githubClient, PipelineManifestRepository pipelineManifestRepository, KubernetesManifestGenerator kubernetesManifestGenerator, ManifestService manifestService, AuthorizationService authorizationFilter, - GitHubClient gitHubClient, PipelineValidatorUtil pipelineValidatorUtil, GocdConfigValidatorUtil gocdConfigValidatorUtil, @Value("${github.org.name}") String orgName, @@ -82,11 +84,10 @@ public class PipelineManifestService { this.kubernetesManifestGenerator = kubernetesManifestGenerator; this.manifestService = manifestService; this.authorizationFilter = authorizationFilter; - this.gitHubClient = gitHubClient; this.pipelineValidatorUtil = pipelineValidatorUtil; this.gocdConfigValidatorUtil = gocdConfigValidatorUtil; - this.orgName = orgName; this.mapDiffUtil = mapDiffUtil; + this.repositoryClient = githubClient.createRepositoryClient(orgName, repository); } @@ -147,7 +148,7 @@ public class PipelineManifestService { private void validatePipelineDifference( PipelineManifest pipelineManifestRequest, - GitHubApiResponse fetchedPipeline + Content fetchedPipeline ) { PipelineManifest existingPipelineManifest = getPipelineManifest( pipelineManifestRequest.getName(), Gocd.MERGED.name()); @@ -155,30 +156,30 @@ public class PipelineManifestService { return; } if ((existingPipelineManifest == null ^ fetchedPipeline == null) - || manualChangesInPipeline(existingPipelineManifest, fetchedPipeline.getContent())) { + || manualChangesInPipeline(existingPipelineManifest, fetchedPipeline.content())) { throw new PipelineDifferenceException(String.format( "Manual changes detected in pipeline %s. Please update the pipeline through github.", pipelineManifestRequest.getName())); } } - private String getPipeLineCreationMessage(GitHubApiResponse response) { + private String getPipeLineCreationMessage(Content response) { return String.format("Pipeline %s by %s at %s", response == null ? "created" : "updated", getUserEmail(), LocalDateTime.now()); } private void createOrUpdatePipelineInGitHub( - String filePath, GitHubApiResponse content, Map pipelineMap + String filePath, Content content, Map pipelineMap ) { try { String encodedContent = base64Encode(yamlMapper.writeValueAsString(pipelineMap)); String message = getPipeLineCreationMessage(content); - String sha = Optional.ofNullable(content) - .map(GitHubApiResponse::getSha) - .orElse(null); - GitHubApiRequest githubRequest = new GitHubApiRequest(sha, message, - encodedContent, "master"); - gitHubClient.createOrUpdateFileContent(orgName, filePath, githubRequest); + FileCreate fileCreate = ImmutableFileCreate.builder() + .message(message) + .content(encodedContent) + .branch("master") + .build(); + repositoryClient.createFileContent(filePath, fileCreate); } catch (Exception e) { throw new RuntimeException(e.getMessage()); } @@ -203,7 +204,12 @@ public class PipelineManifestService { private Map generateGocdPipeline(PipelineManifest pipelineManifest) { String filePath = String.format("%s.gocd.yaml", pipelineManifest.getName()); - GitHubApiResponse fetchedPipeline = gitHubClient.getFileContent(orgName, filePath); + Content fetchedPipeline; + try { + fetchedPipeline = repositoryClient.getFileContent(filePath).get(); + } catch (Exception e) { + fetchedPipeline = null; + } var createdPipelineManifestRequest = pipelineManifestRepository.save(pipelineManifest); validatePipelineDifference(pipelineManifest, fetchedPipeline); Map pipelineMap = generatePipelineTemplate(pipelineManifest); diff --git a/src/main/java/com/navi/infra/portal/util/gocd/GocdConfigValidatorUtil.java b/src/main/java/com/navi/infra/portal/util/gocd/GocdConfigValidatorUtil.java index da56267c..17bb5502 100644 --- a/src/main/java/com/navi/infra/portal/util/gocd/GocdConfigValidatorUtil.java +++ b/src/main/java/com/navi/infra/portal/util/gocd/GocdConfigValidatorUtil.java @@ -31,6 +31,7 @@ public class GocdConfigValidatorUtil { } public boolean validate(String filePath) throws Exception { + validateFilePath(filePath); Path path = Paths.get(filePath); if (!Files.exists(path)) { throw new FileNotFoundException("File not found: " + filePath); @@ -42,4 +43,11 @@ public class GocdConfigValidatorUtil { } return true; } + + private void validateFilePath(String filePath) { + Path path = Paths.get(filePath).normalize(); + if (path.isAbsolute() || path.toString().contains("..")) { + throw new IllegalArgumentException("Invalid file path: " + filePath); + } + } } diff --git a/src/main/java/com/navi/infra/portal/v2/client/github/GitHubApiRequest.java b/src/main/java/com/navi/infra/portal/v2/client/github/GitHubApiRequest.java deleted file mode 100644 index a40f21be..00000000 --- a/src/main/java/com/navi/infra/portal/v2/client/github/GitHubApiRequest.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.navi.infra.portal.v2.client.github; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -@Getter -@Setter -@ToString -@NoArgsConstructor -@AllArgsConstructor -public class GitHubApiRequest { - - private String sha; - private String message; - private String content; - private String branch; - -} diff --git a/src/main/java/com/navi/infra/portal/v2/client/github/GitHubApiResponse.java b/src/main/java/com/navi/infra/portal/v2/client/github/GitHubApiResponse.java deleted file mode 100644 index e707f2a8..00000000 --- a/src/main/java/com/navi/infra/portal/v2/client/github/GitHubApiResponse.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.navi.infra.portal.v2.client.github; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import java.io.Serializable; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -@Getter -@Setter -@ToString -@NoArgsConstructor -@AllArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class GitHubApiResponse implements Serializable { - - private static final long serialVersionUID = 8769281411503539181L; - private String name; - private String path; - private String sha; - private String content; - private String encoding; - -} diff --git a/src/main/java/com/navi/infra/portal/v2/client/github/GitHubClient.java b/src/main/java/com/navi/infra/portal/v2/client/github/GitHubClient.java deleted file mode 100644 index aa7fabb8..00000000 --- a/src/main/java/com/navi/infra/portal/v2/client/github/GitHubClient.java +++ /dev/null @@ -1,99 +0,0 @@ -package com.navi.infra.portal.v2.client.github; - -import static java.lang.String.format; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class GitHubClient { - - private final String authToken; - private final ObjectMapper objectMapper; - private final HttpClient httpClient; - // TODO change to gocd-pipelines - private final String repository = "test-github-action-deployment-portal-backend"; - - public GitHubClient( - @Value("${github.token}") String authToken, - @Qualifier("jsonMapper") ObjectMapper objectMapper - ) { - this.authToken = authToken; - this.objectMapper = objectMapper; - this.httpClient = HttpClient.newHttpClient(); - } - - private HttpRequest.Builder createRequestBuilder(String url) { - return HttpRequest.newBuilder() - .uri(URI.create(url)) - .header("Authorization", "token " + this.authToken) - .header("Accept", "application/vnd.github+json") - .header("X-GitHub-Api-Version", "2022-11-28") - .header("Content-Type", "application/json"); - } - - - String filePathUrl(String org, String repository, String filePath) { - return format("https://api.github.com/repos/%s/%s/contents/%s", org, repository, filePath); - } - - public GitHubApiResponse getFileContent(String org, String filePath) { - String url = filePathUrl(org, repository, filePath); - try { - HttpRequest request = createRequestBuilder(url).GET().build(); - HttpResponse response = httpClient.send(request, - HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() == HttpStatus.OK.value()) { - log.info("Successfully retrieved file content from GitHub for {}", filePath); - return objectMapper.readValue(response.body(), GitHubApiResponse.class); - } - if (response.statusCode() == HttpStatus.NOT_FOUND.value()) { - log.error("File not found in GitHub : {}", response.body()); - return null; - } - throw new RuntimeException("Failed to access content from GitHub."); - } catch (Exception e) { - log.error("Error occurred while getting file from filePath {} : {}", - filePath, e.getMessage()); - throw new RuntimeException("Failed to get file from GitHub", e); - } - } - - public void createOrUpdateFileContent( - String org, - String filePath, - GitHubApiRequest githubRequest - ) { - String url = filePathUrl(org, repository, filePath); - try { - String payload = objectMapper.writeValueAsString(githubRequest); - HttpRequest request = createRequestBuilder(url).PUT( - HttpRequest.BodyPublishers.ofString(payload)).build(); - HttpResponse response = httpClient.send(request, - HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() == HttpStatus.OK.value()) { - log.info("Successfully updated file in GitHub."); - return; - } - if (response.statusCode() == HttpStatus.CREATED.value()) { - log.info("Successfully created file in GitHub."); - return; - } - throw new RuntimeException(response.body()); - } catch (Exception e) { - log.error("Error occurred while creating/updating file in {} org with path {} : {}", - org, filePath, e.getMessage()); - throw new RuntimeException("Failed to commit changes to GitHub", e); - } - } - -} \ No newline at end of file diff --git a/src/test/java/com/navi/infra/portal/service/gocd/PipelineManifestServiceTest.java b/src/test/java/com/navi/infra/portal/service/gocd/PipelineManifestServiceTest.java index c32cf7db..1ec2bf9f 100644 --- a/src/test/java/com/navi/infra/portal/service/gocd/PipelineManifestServiceTest.java +++ b/src/test/java/com/navi/infra/portal/service/gocd/PipelineManifestServiceTest.java @@ -6,7 +6,7 @@ import static com.navi.infra.portal.v2.privilege.ResourceType.MANIFEST; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import com.fasterxml.jackson.core.JsonProcessingException; @@ -24,15 +24,17 @@ import com.navi.infra.portal.util.KutegenClient; import com.navi.infra.portal.util.MapDiffUtil; import com.navi.infra.portal.util.gocd.GocdConfigValidatorUtil; import com.navi.infra.portal.util.gocd.PipelineValidatorUtil; -import com.navi.infra.portal.v2.client.github.GitHubApiResponse; -import com.navi.infra.portal.v2.client.github.GitHubClient; import com.navi.infra.portal.v2.client.gocd.GocdClient; import com.navi.infra.portal.v2.exception.PipelineDifferenceException; +import com.spotify.github.v3.clients.GitHubClient; +import com.spotify.github.v3.clients.RepositoryClient; +import com.spotify.github.v3.repos.ImmutableContent; import com.sun.jdi.request.InvalidRequestStateException; import io.kubernetes.client.openapi.ApiException; import java.io.IOException; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -64,9 +66,12 @@ public class PipelineManifestServiceTest extends ExternalIntegrationProvider { PipelineManifest oldPipelineManifestWithId; PipelineManifest newPipelineManifestWithId; Manifest manifest; + @Mock GitHubClient gitHubClient; @Mock + RepositoryClient repositoryClient; + @Mock GocdClient gocdClient; private PipelineManifestService pipelineManifestService; @Mock @@ -92,14 +97,16 @@ public class PipelineManifestServiceTest extends ExternalIntegrationProvider { @BeforeEach public void init() throws IOException { + when(gitHubClient.createRepositoryClient(anyString(), anyString())).thenReturn( + repositoryClient); pipelineManifestService = new PipelineManifestService( objectMapper, yamlMapper, + gitHubClient, pipelineManifestRepository, kutegenClient, manifestService, authorizationService, - gitHubClient, pipelineValidatorUtil, gocdConfigValidatorUtil, "navi-infra", @@ -190,16 +197,15 @@ public class PipelineManifestServiceTest extends ExternalIntegrationProvider { Manifest manifest = new Manifest(); // base64 encoded string which isn't a valid yaml - var gitHubApiResponse = new GitHubApiResponse(null, null, - "4827ca65e48152eb304c4fe39d7baea3e13856c2", - "some-invalid-content-from-github-that-isn't-base-64", "base64"); + var content = ImmutableContent.builder().content("some-invalid-content-from-github") + .sha("4827ca65e48152eb304c4fe39d7baea3e13856c2").encoding("base64").build(); when(manifestService.fetchById(manifestId)).thenReturn(manifest); when(authorizationService.hasPermissions(MANIFEST, manifest, MANIFEST_WRITE)).thenReturn( true); when(pipelineValidatorUtil.isValidPipeline(any())).thenReturn(true); - - when(gitHubClient.getFileContent(any(), any())).thenReturn(gitHubApiResponse); + when(repositoryClient.getFileContent(any())).thenReturn( + CompletableFuture.completedFuture(content)); when(pipelineManifestRepository.save(newPipelineManifest)).thenReturn( newPipelineManifestWithId); when(pipelineManifestRepository.findLatestByNameAndStatus(any(), any())) @@ -215,16 +221,15 @@ public class PipelineManifestServiceTest extends ExternalIntegrationProvider { Manifest manifest = new Manifest(); // base64 encoded string which isn't same as the pipeline manifest yaml - var gitHubApiResponse = new GitHubApiResponse(null, null, - "4827ca65e48152eb304c4fe39d7baea3e13856c2", - inconsistentBase64EncodedYaml, "base64"); + var content = ImmutableContent.builder().content(inconsistentBase64EncodedYaml) + .sha("4827ca65e48152eb304c4fe39d7baea3e13856c2").encoding("base64").build(); when(manifestService.fetchById(manifestId)).thenReturn(manifest); when(authorizationService.hasPermissions(MANIFEST, manifest, MANIFEST_WRITE)).thenReturn( true); when(pipelineValidatorUtil.isValidPipeline(any())).thenReturn(true); - - when(gitHubClient.getFileContent(any(), any())).thenReturn(gitHubApiResponse); + when(repositoryClient.getFileContent(any())).thenReturn( + CompletableFuture.completedFuture(content)); when(pipelineManifestRepository.save(newPipelineManifest)).thenReturn( newPipelineManifestWithId); when(pipelineManifestRepository.findLatestByNameAndStatus(any(), any())) @@ -242,16 +247,17 @@ public class PipelineManifestServiceTest extends ExternalIntegrationProvider { Manifest manifest = new Manifest(); // base64 encoded string which is same as the old pipeline manifest yaml - var gitHubApiResponse = new GitHubApiResponse(null, null, - "4827ca65e48152eb304c4fe39d7baea3e13856c2", - oldBase64EncodedPipelineYaml, "base64"); + var content = ImmutableContent.builder().content(oldBase64EncodedPipelineYaml) + .sha("4827ca65e48152eb304c4fe39d7baea3e13856c2").encoding("base64").build(); when(manifestService.fetchById(manifestId)).thenReturn(manifest); when(authorizationService.hasPermissions(MANIFEST, manifest, MANIFEST_WRITE)).thenReturn( true); when(pipelineValidatorUtil.isValidPipeline(any())).thenReturn(true); - when(gitHubClient.getFileContent(any(), any())).thenReturn(gitHubApiResponse); + when(repositoryClient.getFileContent(any())).thenReturn( + CompletableFuture.completedFuture(content)); + when(pipelineManifestRepository.findLatestByNameAndStatus(any(), any())) .thenReturn(Optional.of(oldPipelineManifestWithId)); try { @@ -259,7 +265,8 @@ public class PipelineManifestServiceTest extends ExternalIntegrationProvider { } catch (Exception e) { e.printStackTrace(); } - doNothing().when(gitHubClient).createOrUpdateFileContent(any(), any(), any()); + when(repositoryClient.createFileContent(any(), any())).thenReturn( + CompletableFuture.completedFuture(any())); newPipelineManifest.setStatus(Gocd.MERGED.name()); when(pipelineManifestRepository.save(newPipelineManifest)).thenReturn( newPipelineManifestWithId);