From 47ddd6c20af971f1a8ca5d71667d277aacddd552 Mon Sep 17 00:00:00 2001 From: chandresh pancholi Date: Wed, 6 Oct 2021 12:18:58 +0530 Subject: [PATCH] segment creation with redis bloom filters Signed-off-by: chandresh pancholi --- litmus-cache/pom.xml | 6 ++ .../com/navi/medici/cache/RedisCache.java | 10 ++++ .../navi/medici/command/CacheCommands.java | 4 ++ .../navi/medici/container/RedisContainer.java | 4 +- litmus-core/pom.xml | 16 ++--- .../com/navi/medici/config/LitmusConfig.java | 12 ++++ .../controller/v1/SegmentController.java | 13 ++-- .../java/com/navi/medici/s3/S3Service.java | 42 +++++++++++++ .../navi/medici/service/SegmentService.java | 11 ++++ .../medici/service/SegmentServiceImpl.java | 60 +++++++++++++++++++ .../java/com/navi/medici/util/CsvUtil.java | 34 +++++++++++ .../com/navi/medici/util/JacksonUtils.java | 47 +++++++++++++++ .../src/main/resources/application.properties | 13 +++- litmus-db/pom.xml | 7 +-- .../navi/medici/entity/ExperimentEntity.java | 6 +- .../com/navi/medici/entity/SegmentEntity.java | 8 +++ .../query/experiment/ExperimentQueryImpl.java | 29 +++++++++ .../query/experiment/IExperimentQuery.java | 13 ++++ .../medici/query/segment/ISegmentQuery.java | 12 ++++ .../query/segment/SegmentQueryImpl.java | 28 +++++++++ .../repository/ExperimentRepository.java | 16 +++++ .../medici/repository/SegmentRepository.java | 13 ++++ litmus-liquibase/pom.xml | 46 ++++++++++++-- .../src/main/java/com/navi/medici/App.java | 13 ---- .../main/java/com/navi/medici/Database.java | 12 ++++ .../src/main/resources/application.properties | 10 ++++ .../202110051558-create-segment-table.sql | 18 ++++++ .../resources/db/liquibase-changelog.yaml | 3 + litmus-model/pom.xml | 9 ++- .../com/navi/medici/enums/ExperimentType.java | 7 +++ .../com/navi/medici/enums/SegmentType.java | 6 ++ .../navi/medici/request/v1/SegmentCsv.java | 17 ++++++ .../medici/request/v1/SegmentRequest.java | 8 ++- local.env | 2 +- pom.xml | 5 ++ 35 files changed, 512 insertions(+), 48 deletions(-) create mode 100644 litmus-core/src/main/java/com/navi/medici/config/LitmusConfig.java create mode 100644 litmus-core/src/main/java/com/navi/medici/s3/S3Service.java create mode 100644 litmus-core/src/main/java/com/navi/medici/service/SegmentService.java create mode 100644 litmus-core/src/main/java/com/navi/medici/service/SegmentServiceImpl.java create mode 100644 litmus-core/src/main/java/com/navi/medici/util/CsvUtil.java create mode 100644 litmus-core/src/main/java/com/navi/medici/util/JacksonUtils.java create mode 100644 litmus-db/src/main/java/com/navi/medici/query/experiment/ExperimentQueryImpl.java create mode 100644 litmus-db/src/main/java/com/navi/medici/query/experiment/IExperimentQuery.java create mode 100644 litmus-db/src/main/java/com/navi/medici/query/segment/ISegmentQuery.java create mode 100644 litmus-db/src/main/java/com/navi/medici/query/segment/SegmentQueryImpl.java create mode 100644 litmus-db/src/main/java/com/navi/medici/repository/ExperimentRepository.java create mode 100644 litmus-db/src/main/java/com/navi/medici/repository/SegmentRepository.java delete mode 100644 litmus-liquibase/src/main/java/com/navi/medici/App.java create mode 100644 litmus-liquibase/src/main/java/com/navi/medici/Database.java create mode 100644 litmus-liquibase/src/main/resources/application.properties create mode 100644 litmus-liquibase/src/main/resources/db/changelog/202110051558-create-segment-table.sql create mode 100644 litmus-liquibase/src/main/resources/db/liquibase-changelog.yaml create mode 100644 litmus-model/src/main/java/com/navi/medici/enums/ExperimentType.java create mode 100644 litmus-model/src/main/java/com/navi/medici/enums/SegmentType.java create mode 100644 litmus-model/src/main/java/com/navi/medici/request/v1/SegmentCsv.java diff --git a/litmus-cache/pom.xml b/litmus-cache/pom.xml index a8bf506..226b243 100644 --- a/litmus-cache/pom.xml +++ b/litmus-cache/pom.xml @@ -13,6 +13,12 @@ litmus-cache + + org.springframework.boot + spring-boot-starter-data-redis + 2.5.5 + + org.redisson redisson diff --git a/litmus-cache/src/main/java/com/navi/medici/cache/RedisCache.java b/litmus-cache/src/main/java/com/navi/medici/cache/RedisCache.java index f72c8f9..28acd0b 100644 --- a/litmus-cache/src/main/java/com/navi/medici/cache/RedisCache.java +++ b/litmus-cache/src/main/java/com/navi/medici/cache/RedisCache.java @@ -20,6 +20,11 @@ public class RedisCache implements CacheCommands { return bloomFilter.tryInit(redisConfiguration.getExpectedInsertions(), redisConfiguration.getFalseProbability()); } + @Override + public RBloomFilter getBf(String name) { + return redissonClient.getBloomFilter(name); + } + @Override public Boolean add(RBloomFilter bf, String key) { return bf.add(key); @@ -29,4 +34,9 @@ public class RedisCache implements CacheCommands { public Boolean contains(RBloomFilter bf, String key) { return bf.contains(key); } + + @Override + public Boolean delete(String bucket) { + return redissonClient.getBucket(bucket).delete(); + } } diff --git a/litmus-cache/src/main/java/com/navi/medici/command/CacheCommands.java b/litmus-cache/src/main/java/com/navi/medici/command/CacheCommands.java index 31e3375..801395f 100644 --- a/litmus-cache/src/main/java/com/navi/medici/command/CacheCommands.java +++ b/litmus-cache/src/main/java/com/navi/medici/command/CacheCommands.java @@ -5,7 +5,11 @@ import org.redisson.api.RBloomFilter; public interface CacheCommands { Boolean reserveBF(String name); + RBloomFilter getBf(String name); + Boolean add(RBloomFilter bf, String key); Boolean contains(RBloomFilter bf, String key); + + Boolean delete(String bucket); } diff --git a/litmus-cache/src/main/java/com/navi/medici/container/RedisContainer.java b/litmus-cache/src/main/java/com/navi/medici/container/RedisContainer.java index 45075a0..9eeb91a 100644 --- a/litmus-cache/src/main/java/com/navi/medici/container/RedisContainer.java +++ b/litmus-cache/src/main/java/com/navi/medici/container/RedisContainer.java @@ -15,12 +15,12 @@ public class RedisContainer { private final RedisConfiguration redisConfiguration; @Bean - public RedissonClient redissonClient(Config config) { + public RedissonClient redissonClient() { Config c = new Config(); c.useSingleServer() .setAddress(String.format("redis://%s:%s", redisConfiguration.getRedisHost(), redisConfiguration.getRedisPort())); - return Redisson.create(config); + return Redisson.create(c); } diff --git a/litmus-core/pom.xml b/litmus-core/pom.xml index 412c310..0e1ae04 100644 --- a/litmus-core/pom.xml +++ b/litmus-core/pom.xml @@ -48,7 +48,7 @@ org.springframework.boot - spring-boot-starter-webflux + spring-boot-starter-web @@ -63,17 +63,13 @@ 2.17.51 - - com.navi.medici.utils - event-bus-client - 0.1.3-SNAPSHOT - + - junit - junit - 4.13.2 - test + com.opencsv + opencsv + 5.5.2 + diff --git a/litmus-core/src/main/java/com/navi/medici/config/LitmusConfig.java b/litmus-core/src/main/java/com/navi/medici/config/LitmusConfig.java new file mode 100644 index 0000000..397243d --- /dev/null +++ b/litmus-core/src/main/java/com/navi/medici/config/LitmusConfig.java @@ -0,0 +1,12 @@ +package com.navi.medici.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Data +public class LitmusConfig { + @Value("${segment.s3.bucket}") + String s3Bucket; +} diff --git a/litmus-core/src/main/java/com/navi/medici/controller/v1/SegmentController.java b/litmus-core/src/main/java/com/navi/medici/controller/v1/SegmentController.java index cb6cd98..a45bc6f 100644 --- a/litmus-core/src/main/java/com/navi/medici/controller/v1/SegmentController.java +++ b/litmus-core/src/main/java/com/navi/medici/controller/v1/SegmentController.java @@ -1,23 +1,28 @@ package com.navi.medici.controller.v1; import com.navi.medici.request.v1.SegmentRequest; +import com.navi.medici.service.SegmentService; import lombok.RequiredArgsConstructor; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; @RestController @RequiredArgsConstructor -@RequestMapping("/v1/bucket") +@RequestMapping("/v1/segments") public class SegmentController { + private final SegmentService segmentService; - @PostMapping(value = "/create") - public void createSegment(@RequestParam("file") MultipartFile file, - @RequestParam("segment") SegmentRequest segmentRequest) { + @PostMapping(value = "/create", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public void createSegment(@RequestPart("file") MultipartFile file, + @RequestPart("segment") SegmentRequest segmentRequest) { + segmentService.createSegment(file, segmentRequest); } } diff --git a/litmus-core/src/main/java/com/navi/medici/s3/S3Service.java b/litmus-core/src/main/java/com/navi/medici/s3/S3Service.java new file mode 100644 index 0000000..37fdcf1 --- /dev/null +++ b/litmus-core/src/main/java/com/navi/medici/s3/S3Service.java @@ -0,0 +1,42 @@ +package com.navi.medici.s3; + +import com.navi.medici.config.LitmusConfig; +import java.io.FileWriter; +import java.io.IOException; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +@Service +@Log4j2 +@RequiredArgsConstructor +public class S3Service { + private final S3Client s3Client; + private final LitmusConfig litmusConfig; + + public void upload(MultipartFile file, String destinationBucket) { + try (FileWriter writer = new FileWriter("/tmp/"+destinationBucket)) { + writer.write(new String(file.getBytes())); + } catch (IOException e) { + log.error("file upload failed", e); + } + +// try { +// PutObjectRequest putOb = PutObjectRequest.builder() +// .bucket(litmusConfig.getS3Bucket()) +// .key(destinationBucket) +// .build(); +// +// PutObjectResponse response = s3Client.putObject(putOb, +// RequestBody.fromBytes(file.getBytes())); +// } catch (IOException e) { +// log.error("s3 upload failed", e); +// } + + } +} diff --git a/litmus-core/src/main/java/com/navi/medici/service/SegmentService.java b/litmus-core/src/main/java/com/navi/medici/service/SegmentService.java new file mode 100644 index 0000000..f9bf958 --- /dev/null +++ b/litmus-core/src/main/java/com/navi/medici/service/SegmentService.java @@ -0,0 +1,11 @@ +package com.navi.medici.service; + +import com.navi.medici.request.v1.SegmentRequest; +import org.springframework.web.multipart.MultipartFile; + +public interface SegmentService { + void createSegment(MultipartFile file, SegmentRequest segmentRequest); + + void segmentIdExist(String segmentName, String id); + +} diff --git a/litmus-core/src/main/java/com/navi/medici/service/SegmentServiceImpl.java b/litmus-core/src/main/java/com/navi/medici/service/SegmentServiceImpl.java new file mode 100644 index 0000000..5ae7b74 --- /dev/null +++ b/litmus-core/src/main/java/com/navi/medici/service/SegmentServiceImpl.java @@ -0,0 +1,60 @@ +package com.navi.medici.service; + +import com.navi.medici.command.CacheCommands; +import com.navi.medici.entity.SegmentEntity; +import com.navi.medici.query.segment.ISegmentQuery; +import com.navi.medici.request.v1.SegmentCsv; +import com.navi.medici.request.v1.SegmentRequest; +import com.navi.medici.s3.S3Service; +import com.navi.medici.util.CsvUtil; +import java.util.List; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +@Service +@Log4j2 +@RequiredArgsConstructor +public class SegmentServiceImpl implements SegmentService { + + private final CsvUtil csvUtil; + private final S3Service s3Service; + private final CacheCommands cacheCommands; + private final ISegmentQuery segmentQuery; + + @Override + public void createSegment(MultipartFile file, SegmentRequest segmentRequest) { + String destinationBucketName = UUID.randomUUID().toString(); + + s3Service.upload(file, destinationBucketName); + List segmentCsvList = csvUtil.csvToList(file); + + var bfCreated = cacheCommands.reserveBF(segmentRequest.getName()); + var bloomFilter = cacheCommands.getBf(segmentRequest.getName()); + log.info("ingesting data to bloom filter. segment_name: {}", segmentRequest.getName()); + + segmentCsvList.forEach(segment ->{ + var temp = cacheCommands.add(bloomFilter, segment.getSegmentId()); + log.info("key: {}, result: {}", segment.getSegmentId(), temp); + }); + + var segmentEntity = SegmentEntity.builder() + .segmentId(UUID.randomUUID().toString()) + .segmentName(segmentRequest.getName()) + .segmentDescription(segmentRequest.getDescription()) + .segmentType(segmentRequest.getSegmentType()) + .destinationBucket(destinationBucketName) + .build(); + + segmentQuery.save(segmentEntity); + } + + @Override + public void segmentIdExist(String segmentName, String id) { + var bloomFilter = cacheCommands.getBf(segmentName); + + var result = cacheCommands.contains(bloomFilter, id); + } +} diff --git a/litmus-core/src/main/java/com/navi/medici/util/CsvUtil.java b/litmus-core/src/main/java/com/navi/medici/util/CsvUtil.java new file mode 100644 index 0000000..896faa0 --- /dev/null +++ b/litmus-core/src/main/java/com/navi/medici/util/CsvUtil.java @@ -0,0 +1,34 @@ +package com.navi.medici.util; + +import com.navi.medici.request.v1.SegmentCsv; +import com.opencsv.bean.CsvToBeanBuilder; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; + +@Component +@Log4j2 +public class CsvUtil { + public List csvToList(MultipartFile file) { + try { + BufferedReader reader = new BufferedReader(new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8)); + + var cb = new CsvToBeanBuilder(reader) + .withType(SegmentCsv.class) + .withIgnoreLeadingWhiteSpace(true) + .build(); + + return cb.parse(); + + } catch (IOException e) { + log.error("data csv read failed. error: {}", e.getMessage()); + + throw new RuntimeException("data csv read failed.", e); + } + } +} diff --git a/litmus-core/src/main/java/com/navi/medici/util/JacksonUtils.java b/litmus-core/src/main/java/com/navi/medici/util/JacksonUtils.java new file mode 100644 index 0000000..c6bce92 --- /dev/null +++ b/litmus-core/src/main/java/com/navi/medici/util/JacksonUtils.java @@ -0,0 +1,47 @@ +package com.navi.medici.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.CollectionType; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +@Log4j2 +public class JacksonUtils { + private final ObjectMapper objectMapper; + + public String objectToString(Object o) { + try { + return objectMapper.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException("object to string conversion failed", e); + } + } + + public T stringToObject(String s, Class klazz) { + try { + return objectMapper.readValue(s, klazz); + } catch (JsonProcessingException e) { + throw new RuntimeException("string to object conversion failed", e); + } + } + + public List stringToListObject(String s, Class klazz) { + try { + CollectionType listType = objectMapper.getTypeFactory().constructCollectionType(ArrayList.class, klazz); + + return objectMapper.readValue(s, listType); + } catch (JsonProcessingException e) { + throw new RuntimeException("string to list object conversion failed", e); + } + } + + public T convertValue(Object o, Class klazz) { + return objectMapper.convertValue(o, klazz); + } +} \ No newline at end of file diff --git a/litmus-core/src/main/resources/application.properties b/litmus-core/src/main/resources/application.properties index 9eeb1ff..dbce67a 100644 --- a/litmus-core/src/main/resources/application.properties +++ b/litmus-core/src/main/resources/application.properties @@ -1,7 +1,7 @@ spring.datasource.hikari.maximum-pool-size=${DB_POOL_MAX_SIZE:2} spring.datasource.hikari.minimum-idle=${DB_POOL_MIN_IDLE:1} spring.datasource.hikari.idle-timeout=${DB_POOL_IDLE_TIMEOUT_IN_MS:30000} -spring.datasource.url=${DATASOURCE_URL:jdbc:postgresql://localhost:5432/santa}?stringtype=unspecified +spring.datasource.url=${DATASOURCE_URL:jdbc:postgresql://localhost:5432/litmus}?stringtype=unspecified spring.datasource.username=${DATASOURCE_USERNAME:postgres} spring.datasource.password=${DATASOURCE_PASSWORD:admin} spring.datasource.initialization-mode=${DATA_INITIALIZATION_MODE:never} @@ -15,4 +15,13 @@ spring.jpa.hibernate.ddl-auto=none kafka.servers=${KAFKA_SERVER:localhost:9092} audit.kafka.servers=${AUDIT_KAFKA_SERVER:localhost:9092} kafka.auditlog.topic=${AUDIT_LOG_TOPIC:audit-logs} -kms.base-url=${KMS_BASE_URL:http://google.com} \ No newline at end of file +kms.base-url=${KMS_BASE_URL:http://google.com} + + +redis.host=127.0.0.1 +redis.port=6379 +redis.expected.insertions=99999 +redis.false.probability=0.001 + +segment.s3.bucket=navi-test +report.s3.region=temp \ No newline at end of file diff --git a/litmus-db/pom.xml b/litmus-db/pom.xml index 1084b09..dddf8bf 100644 --- a/litmus-db/pom.xml +++ b/litmus-db/pom.xml @@ -25,10 +25,9 @@ - junit - junit - 4.11 - test + com.navi.medici + litmus-model + 1.0-SNAPSHOT diff --git a/litmus-db/src/main/java/com/navi/medici/entity/ExperimentEntity.java b/litmus-db/src/main/java/com/navi/medici/entity/ExperimentEntity.java index ecdb5ca..1cb28fb 100644 --- a/litmus-db/src/main/java/com/navi/medici/entity/ExperimentEntity.java +++ b/litmus-db/src/main/java/com/navi/medici/entity/ExperimentEntity.java @@ -1,8 +1,11 @@ package com.navi.medici.entity; +import com.navi.medici.enums.ExperimentType; import java.time.LocalDateTime; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -53,7 +56,8 @@ public class ExperimentEntity { String variants; @Column(name = "type") - String type; + @Enumerated(EnumType.STRING) + ExperimentType type; @Column(name = "start_time") LocalDateTime startTime; diff --git a/litmus-db/src/main/java/com/navi/medici/entity/SegmentEntity.java b/litmus-db/src/main/java/com/navi/medici/entity/SegmentEntity.java index a13afce..44911e8 100644 --- a/litmus-db/src/main/java/com/navi/medici/entity/SegmentEntity.java +++ b/litmus-db/src/main/java/com/navi/medici/entity/SegmentEntity.java @@ -1,13 +1,17 @@ package com.navi.medici.entity; +import com.navi.medici.enums.SegmentType; import java.time.LocalDateTime; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; import javax.persistence.Version; +import javax.swing.text.Segment; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -44,6 +48,10 @@ public class SegmentEntity { @Column(name = "bucket") String destinationBucket; + @Column(name = "segment_type") + @Enumerated(EnumType.STRING) + SegmentType segmentType; + @Version private Integer version; diff --git a/litmus-db/src/main/java/com/navi/medici/query/experiment/ExperimentQueryImpl.java b/litmus-db/src/main/java/com/navi/medici/query/experiment/ExperimentQueryImpl.java new file mode 100644 index 0000000..ab5d70e --- /dev/null +++ b/litmus-db/src/main/java/com/navi/medici/query/experiment/ExperimentQueryImpl.java @@ -0,0 +1,29 @@ +package com.navi.medici.query.experiment; + +import com.navi.medici.entity.ExperimentEntity; +import com.navi.medici.repository.ExperimentRepository; +import java.util.List; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class ExperimentQueryImpl implements IExperimentQuery { + private final ExperimentRepository experimentRepository; + + @Override + public List findByEnabled(Boolean enabled) { + return experimentRepository.findByEnabled(enabled); + } + + @Override + public Optional findByName(String name) { + return experimentRepository.findByName(name); + } + + @Override + public Optional findByExperimentId(String experimentId) { + return experimentRepository.findByExperimentId(experimentId); + } +} diff --git a/litmus-db/src/main/java/com/navi/medici/query/experiment/IExperimentQuery.java b/litmus-db/src/main/java/com/navi/medici/query/experiment/IExperimentQuery.java new file mode 100644 index 0000000..384953a --- /dev/null +++ b/litmus-db/src/main/java/com/navi/medici/query/experiment/IExperimentQuery.java @@ -0,0 +1,13 @@ +package com.navi.medici.query.experiment; + +import com.navi.medici.entity.ExperimentEntity; +import java.util.List; +import java.util.Optional; + +public interface IExperimentQuery { + List findByEnabled(Boolean enabled); + + Optional findByName(String name); + + Optional findByExperimentId(String experimentId); +} diff --git a/litmus-db/src/main/java/com/navi/medici/query/segment/ISegmentQuery.java b/litmus-db/src/main/java/com/navi/medici/query/segment/ISegmentQuery.java new file mode 100644 index 0000000..b347352 --- /dev/null +++ b/litmus-db/src/main/java/com/navi/medici/query/segment/ISegmentQuery.java @@ -0,0 +1,12 @@ +package com.navi.medici.query.segment; + +import com.navi.medici.entity.SegmentEntity; +import java.util.Optional; + +public interface ISegmentQuery { + Optional findBySegmentId(String segmentId); + + void save(SegmentEntity segmentEntity); + + Optional findBySegmentName(String segmentName); +} diff --git a/litmus-db/src/main/java/com/navi/medici/query/segment/SegmentQueryImpl.java b/litmus-db/src/main/java/com/navi/medici/query/segment/SegmentQueryImpl.java new file mode 100644 index 0000000..9b89c35 --- /dev/null +++ b/litmus-db/src/main/java/com/navi/medici/query/segment/SegmentQueryImpl.java @@ -0,0 +1,28 @@ +package com.navi.medici.query.segment; + +import com.navi.medici.entity.SegmentEntity; +import com.navi.medici.repository.SegmentRepository; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class SegmentQueryImpl implements ISegmentQuery{ + private final SegmentRepository segmentRepository; + + @Override + public Optional findBySegmentId(String segmentId) { + return segmentRepository.findBySegmentId(segmentId); + } + + @Override + public void save(SegmentEntity segmentEntity) { + segmentRepository.save(segmentEntity); + } + + @Override + public Optional findBySegmentName(String segmentName) { + return segmentRepository.findBySegmentName(segmentName); + } +} diff --git a/litmus-db/src/main/java/com/navi/medici/repository/ExperimentRepository.java b/litmus-db/src/main/java/com/navi/medici/repository/ExperimentRepository.java new file mode 100644 index 0000000..141d2d1 --- /dev/null +++ b/litmus-db/src/main/java/com/navi/medici/repository/ExperimentRepository.java @@ -0,0 +1,16 @@ +package com.navi.medici.repository; + +import com.navi.medici.entity.ExperimentEntity; +import java.util.List; +import java.util.Optional; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface ExperimentRepository extends CrudRepository { + List findByEnabled(Boolean enabled); + + Optional findByName(String name); + + Optional findByExperimentId(String experimentId); +} diff --git a/litmus-db/src/main/java/com/navi/medici/repository/SegmentRepository.java b/litmus-db/src/main/java/com/navi/medici/repository/SegmentRepository.java new file mode 100644 index 0000000..d6ab118 --- /dev/null +++ b/litmus-db/src/main/java/com/navi/medici/repository/SegmentRepository.java @@ -0,0 +1,13 @@ +package com.navi.medici.repository; + +import com.navi.medici.entity.SegmentEntity; +import java.util.Optional; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface SegmentRepository extends CrudRepository { + Optional findBySegmentId(String segmentId); + + Optional findBySegmentName(String segmentName); +} diff --git a/litmus-liquibase/pom.xml b/litmus-liquibase/pom.xml index dd68dcb..e925257 100644 --- a/litmus-liquibase/pom.xml +++ b/litmus-liquibase/pom.xml @@ -15,11 +15,49 @@ - junit - junit - 4.11 - test + org.springframework.boot + spring-boot-starter-data-jpa + + + org.liquibase + liquibase-core + runtime + 4.3.5 + + + + org.postgresql + postgresql + 42.2.20 + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + 11 + + + org.projectlombok + lombok + 1.18.20 + + + + + + + diff --git a/litmus-liquibase/src/main/java/com/navi/medici/App.java b/litmus-liquibase/src/main/java/com/navi/medici/App.java deleted file mode 100644 index c0d56af..0000000 --- a/litmus-liquibase/src/main/java/com/navi/medici/App.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.navi.medici; - -/** - * Hello world! - * - */ -public class App -{ - public static void main( String[] args ) - { - System.out.println( "Hello World!" ); - } -} diff --git a/litmus-liquibase/src/main/java/com/navi/medici/Database.java b/litmus-liquibase/src/main/java/com/navi/medici/Database.java new file mode 100644 index 0000000..e8e3053 --- /dev/null +++ b/litmus-liquibase/src/main/java/com/navi/medici/Database.java @@ -0,0 +1,12 @@ +package com.navi.medici; + + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class Database { + public static void main( String[] args ) { + SpringApplication.run(Database.class, args); + } +} diff --git a/litmus-liquibase/src/main/resources/application.properties b/litmus-liquibase/src/main/resources/application.properties new file mode 100644 index 0000000..9e60b80 --- /dev/null +++ b/litmus-liquibase/src/main/resources/application.properties @@ -0,0 +1,10 @@ +spring.datasource.hikari.maximum-pool-size=${DB_POOL_MAX_SIZE:2} +spring.datasource.hikari.minimum-idle=${DB_POOL_MIN_IDLE:1} +spring.datasource.hikari.idle-timeout=${DB_POOL_IDLE_TIMEOUT_IN_MS:30000} +spring.datasource.url=${DATASOURCE_URL:jdbc:postgresql://localhost:5432/litmus}?stringtype=unspecified +spring.datasource.username=${DATASOURCE_USERNAME:postgres} +spring.datasource.password=${DATASOURCE_PASSWORD:admin} +spring.datasource.initialization-mode=${DATA_INITIALIZATION_MODE:never} +spring.jpa.hibernate.ddl-auto=none +spring.liquibase.change-log=classpath:db/liquibase-changelog.yaml +spring.main.web-application-type=NONE \ No newline at end of file diff --git a/litmus-liquibase/src/main/resources/db/changelog/202110051558-create-segment-table.sql b/litmus-liquibase/src/main/resources/db/changelog/202110051558-create-segment-table.sql new file mode 100644 index 0000000..76dcfe6 --- /dev/null +++ b/litmus-liquibase/src/main/resources/db/changelog/202110051558-create-segment-table.sql @@ -0,0 +1,18 @@ +--liquibase formatted sql + +--changeset author:chandresh id:202110051558 +CREATE TABLE segments ( + id SERIAL PRIMARY KEY, + segment_id varchar(36) NOT NULL, + name varchar(100) NOT NULL, + description text NOT NULL, + bucket varchar(100) NOT NULL, + segment_type varchar(20) NOT NULL, + version BIGINT, + created_at timestamp, + updated_at timestamp +); + + +CREATE INDEX idx_segment_id ON segments(segment_id); +CREATE INDEX idx_name ON segments(name); diff --git a/litmus-liquibase/src/main/resources/db/liquibase-changelog.yaml b/litmus-liquibase/src/main/resources/db/liquibase-changelog.yaml new file mode 100644 index 0000000..6cb27e2 --- /dev/null +++ b/litmus-liquibase/src/main/resources/db/liquibase-changelog.yaml @@ -0,0 +1,3 @@ +databaseChangeLog: + - includeAll: + path: db/changelog/ \ No newline at end of file diff --git a/litmus-model/pom.xml b/litmus-model/pom.xml index 9e1cdcf..4ea32cf 100644 --- a/litmus-model/pom.xml +++ b/litmus-model/pom.xml @@ -46,13 +46,12 @@ 2.13.0 - - junit - junit - 4.11 - test + com.opencsv + opencsv + 5.5.2 + diff --git a/litmus-model/src/main/java/com/navi/medici/enums/ExperimentType.java b/litmus-model/src/main/java/com/navi/medici/enums/ExperimentType.java new file mode 100644 index 0000000..263e64c --- /dev/null +++ b/litmus-model/src/main/java/com/navi/medici/enums/ExperimentType.java @@ -0,0 +1,7 @@ +package com.navi.medici.enums; + +public enum ExperimentType { + EXPERIMENT, + RELEASE, + KILL_SWITCH +} diff --git a/litmus-model/src/main/java/com/navi/medici/enums/SegmentType.java b/litmus-model/src/main/java/com/navi/medici/enums/SegmentType.java new file mode 100644 index 0000000..545ebb1 --- /dev/null +++ b/litmus-model/src/main/java/com/navi/medici/enums/SegmentType.java @@ -0,0 +1,6 @@ +package com.navi.medici.enums; + +public enum SegmentType { + USER_ID, + DEVICE_ID +} diff --git a/litmus-model/src/main/java/com/navi/medici/request/v1/SegmentCsv.java b/litmus-model/src/main/java/com/navi/medici/request/v1/SegmentCsv.java new file mode 100644 index 0000000..73dda56 --- /dev/null +++ b/litmus-model/src/main/java/com/navi/medici/request/v1/SegmentCsv.java @@ -0,0 +1,17 @@ +package com.navi.medici.request.v1; + +import com.opencsv.bean.CsvBindByName; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@AllArgsConstructor +@Getter +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class SegmentCsv { + @CsvBindByName(column = "segment_id") + String segmentId; +} diff --git a/litmus-model/src/main/java/com/navi/medici/request/v1/SegmentRequest.java b/litmus-model/src/main/java/com/navi/medici/request/v1/SegmentRequest.java index f57ceec..1ab4c23 100644 --- a/litmus-model/src/main/java/com/navi/medici/request/v1/SegmentRequest.java +++ b/litmus-model/src/main/java/com/navi/medici/request/v1/SegmentRequest.java @@ -1,6 +1,8 @@ package com.navi.medici.request.v1; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.navi.medici.enums.SegmentType; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -17,10 +19,12 @@ import lombok.experimental.FieldDefaults; @JsonIgnoreProperties(ignoreUnknown = true) @FieldDefaults(level = AccessLevel.PRIVATE) public class SegmentRequest { + @JsonProperty("name") String name; + @JsonProperty("description") String description; - - + @JsonProperty("segment_type") + SegmentType segmentType; } diff --git a/local.env b/local.env index e78f0cd..6dbbdee 100644 --- a/local.env +++ b/local.env @@ -1,3 +1,3 @@ -DATASOURCE_URL=jdbc:postgresql://dev-db-service-db.np.navi-tech.in:5432/santa +DATASOURCE_URL=jdbc:postgresql://dev-db-service-db.np.navi-tech.in:5432/litmus DATASOURCE_USERNAME=service_user DATASOURCE_PASSWORD=JRCFCMXUXBJHGZVTPBNTXHYCCFVMWN \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9f99210..58bbf32 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,11 @@ spring-boot-autoconfigure + + org.springframework.boot + spring-boot-devtools + + org.projectlombok lombok