varnitgoyal95/cybertron
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tp 55555/integrate document service client (#5)

Browse Source 
* TP-55555 | document client and kafka integration

* TP-55555 | introduce service concept refactor code
This commit is contained in:
Varnit Goyal
2024-07-27 17:00:47 +05:30
committed by GitHub
parent f75297880d
commit f20af81520
21 changed files with 764 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
configs/application.yml
View File

@@ -33,3 +33,35 @@ prometheus:
timeout: 10
flush.interval.in.ms: 200
histogram.buckets: 50.0,75.0,90.0,95.0,99.0
http:
max:
idle.connection.pool: 200
connection: 200
idle.timeout.seconds: 10
# Kafka config
kafka:
password: KAFKA_PASSWORD
username: KAFKA_USERNAME
brokers: KAFKA_BROKERS
group:
names: KAFKA_GROUP_NAMES
topics: KAFKA_TOPICS
tls:
insecureSkipVerify: KAFKA_TLS_INSECURE_SKIP_VERIFY
enabled: KAFKA_TLS_ENABLED
sasl:
enabled: KAFKA_SASL_ENABLED
consumer:
batch.size: KAFKA_CONSUMER_BATCH_SIZE
goroutines.max.pool.size: KAFKA_CONSUMER_GOROUTINES_MAX_POOL_SIZE
max.timeout.ms: KAFKA_CONSUMER_MAX_TIMEOUT_MS
auto.offset.reset: latest
#document service conf
DocumentService:
baseurl: DOCUMENT_SERVICE_BASEURL
mock:
generate_token: DOCUMENT_SERVICE_MOCK_GENERATE_TOKEN

27
configs/client_configs.go Normal file
View File

@@ -0,0 +1,27 @@
package configs
type ClientConfigs struct {
DocumentServiceHttpClientConfigs *DocumentServiceHttpClientConfigs
}
type DocumentServiceHttpClientConfigs struct {
BaseUrl string
MockEnabled bool
}
func loadClientConfigs() *ClientConfigs {
return &ClientConfigs{
DocumentServiceHttpClientConfigs: loadDocumentServiceHttpClientConfigs(),
}
}
func loadDocumentServiceHttpClientConfigs() *DocumentServiceHttpClientConfigs {
return &DocumentServiceHttpClientConfigs{
BaseUrl: getString("DocumentService.baseurl", true),
MockEnabled: getBool("DocumentService.mock.generate_token", true),
}
}
func GetDocumentServiceHttpClientConfigs() *DocumentServiceHttpClientConfigs {
return appConfig.clientConfigs.DocumentServiceHttpClientConfigs
}

8
configs/config.go
View File

@@ -16,6 +16,8 @@ type AppConfig struct {
prometheus *Prometheus
postgres Postgres
timezone string
httpConfig *HttpConfig
clientConfigs *ClientConfigs
}
type MigConfig struct {
@@ -37,6 +39,8 @@ func LoadConfig() {
prometheus: GetPrometheusConfig(),
postgres: getPostgresConfig(),
timezone: getString("timezone", true),
clientConfigs: loadClientConfigs(),
httpConfig: NewHttpConfig(),
}
}
@@ -86,3 +90,7 @@ func readConfig() {
log.Log.GetLog().Panic("Error while loading configuration", zap.Error(err))
}
}
func GetHttpConfig() *HttpConfig {
return appConfig.httpConfig
}

17
configs/http_config.go Normal file
View File

@@ -0,0 +1,17 @@
package configs
import "github.com/spf13/viper"
type HttpConfig struct {
MaxIdleConnectionPool int
MaxConnection int
MaxTimeoutInSeconds int64
}
func NewHttpConfig() *HttpConfig {
return &HttpConfig{
MaxIdleConnectionPool: viper.GetInt("http.max.idle.connection.pool"),
MaxConnection: viper.GetInt("http.max.connection"),
MaxTimeoutInSeconds: viper.GetInt64("http.max.idle.timeout.seconds"),
}
}

40
configs/kafka.go Normal file
View File

@@ -0,0 +1,40 @@
package configs
import (
"github.com/spf13/viper"
"strings"
)
type KafkaConfig struct {
Brokers []string
Username string
Password string
Group map[string]string
Topics map[string]string
TlsInsureSkipVerification bool
TlsEnabled bool
SaslEnabled bool
ConsumerBatchSize int
GoroutinesMaxPoolSize int
ConsumerMaxTimeoutMs string
}
func NewKafkaConfig() *KafkaConfig {
return &KafkaConfig{
Brokers: strings.Split(viper.GetString("kafka.brokers"), ","),
Username: viper.GetString("kafka.username"),
Password: viper.GetString("kafka.password"),
Group: viper.GetStringMapString("kafka.group.names"),
Topics: viper.GetStringMapString("kafka.topics"),
TlsInsureSkipVerification: viper.GetBool("kafka.tls.insecureSkipVerify"),
SaslEnabled: viper.GetBool("kafka.sasl.enabled"),
TlsEnabled: viper.GetBool("kafka.tls.enabled"),
ConsumerBatchSize: viper.GetInt("kafka.consumer.batch.size"),
GoroutinesMaxPoolSize: viper.GetInt("kafka.consumer.goroutines.max.pool.size"),
ConsumerMaxTimeoutMs: viper.GetString("kafka.consumer.max.timeout.ms"),
}
}
func (kc *KafkaConfig) GetTopic(topic string) string {
return kc.Topics[topic]
}

4
configs/utils.go
View File

@@ -39,7 +39,7 @@ func getFloatSlice(key string, required bool) []float64 {
for _, val := range stringValues {
floatVal, err := strconv.ParseFloat(val, 64)
if err != nil {
log.Panic("config value is not float type, err : " + err.Error())
log.Log.Panic("config value is not float type, err : " + err.Error())
}
floatValues = append(floatValues, floatVal)
}
@@ -49,7 +49,7 @@ func getFloatSlice(key string, required bool) []float64 {
func checkKey(key string) {
if !viper.IsSet(key) {
log.Panic("Missing key: " + key)
log.Log.Panic("Missing key: " + key)
}
}

28
go.mod
View File

@@ -3,6 +3,7 @@ module cybertron
go 1.21.1
require (
github.com/IBM/sarama v1.43.2
github.com/gin-contrib/zap v0.2.0
github.com/gin-gonic/gin v1.9.1
github.com/golang-migrate/migrate/v4 v4.17.1
@@ -10,8 +11,10 @@ require (
github.com/prometheus/client_golang v1.19.1
github.com/spf13/cobra v1.7.0
github.com/spf13/viper v1.17.0
github.com/xdg-go/scram v1.1.1
go.elastic.co/ecszap v1.0.2
go.uber.org/zap v1.26.0
google.golang.org/protobuf v1.33.0
gorm.io/driver/postgres v1.5.3
gorm.io/gorm v1.25.5
)
@@ -22,6 +25,10 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/eapache/go-resiliency v1.6.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
@@ -29,17 +36,25 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.15.5 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.5.4 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/lib/pq v1.10.9 // indirect
@@ -49,10 +64,12 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -63,16 +80,17 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.5.0 // indirect
golang.org/x/crypto v0.20.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

25
internal/client/document/document_client.ts.go Normal file
View File

@@ -0,0 +1,25 @@
package document
import (
"cybertron/configs"
httpclient "cybertron/pkg/httpClient"
"cybertron/pkg/log"
)
type Client interface {
}
type HttpClient struct {
Client
HttpClient httpclient.HttpClient
Logger *log.Logger
DocumentServiceHttpClientConfigs *configs.DocumentServiceHttpClientConfigs
}
func NewDocumentServiceHttpClient(httpClient httpclient.HttpClient, logger *log.Logger, documentServiceHttpClientConfigs *configs.DocumentServiceHttpClientConfigs) *HttpClient {
return &HttpClient{
HttpClient: httpClient,
Logger: logger,
DocumentServiceHttpClientConfigs: documentServiceHttpClientConfigs,
}
}

25
internal/dependencies/dependencies.go
View File

@@ -1,10 +1,14 @@
package dependencies
import (
"cybertron/configs"
"cybertron/internal/client/document"
"cybertron/internal/database"
"cybertron/internal/transport/handler"
"cybertron/pkg/db"
httpclient "cybertron/pkg/httpClient"
"cybertron/pkg/log"
"cybertron/service"
"go.uber.org/zap"
"gorm.io/gorm"
)
@@ -18,6 +22,8 @@ type Dependencies struct {
}
type Service struct {
DocumentService *document.HttpClient
ProjectService *service.ProjectCreator
// Add your service here
}
@@ -30,22 +36,27 @@ type Repositories struct {
}
func InitDependencies() *Dependencies {
services := initServices()
dbClient := db.NewDBClient()
handlers := initHandlers(dbClient)
repositories := initRepositories(dbClient)
logger := log.Log
httpClient := httpclient.NewHttpClient(*configs.GetHttpConfig())
documentServiceClient := document.NewDocumentServiceHttpClient(httpClient, logger, configs.GetDocumentServiceHttpClientConfigs())
projectServiceClient := service.NewProjectCreator(logger, dbClient)
services := initServices(documentServiceClient, projectServiceClient)
handlers := initHandlers(projectServiceClient)
return &Dependencies{
Service: services,
DBClient: dbClient,
Logger: log.Log.GetLog(),
Logger: logger.GetLog(),
Handler: handlers,
Repositories: repositories,
}
}
func initServices() *Service {
func initServices(documentService *document.HttpClient, projectService *service.ProjectCreator) *Service {
return &Service{
// Add your service here
DocumentService: documentService,
ProjectService: projectService,
}
}
@@ -55,8 +66,8 @@ func initRepositories(dbClient *gorm.DB) *Repositories {
}
}
func initHandlers(dbClient *gorm.DB) *Handler {
projectHandler := handler.NewProjectHandler(dbClient)
func initHandlers(projectService *service.ProjectCreator) *Handler {
projectHandler := handler.NewProjectHandler(projectService)
return &Handler{
ProjectHandler: projectHandler,
}

36
internal/transport/handler/project.go
View File

@@ -1,15 +1,12 @@
package handler
import (
"cybertron/models/db"
"cybertron/service"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"gorm.io/gorm"
"net/http"
)
type ProjectHandler struct {
dbClient *gorm.DB
projectCreatorService *service.ProjectCreator
}
type ProjectBody struct {
@@ -18,36 +15,15 @@ type ProjectBody struct {
}
func (h *ProjectHandler) ProjectCreate(c *gin.Context) {
var projectBody ProjectBody
if err := c.BindJSON(&projectBody); err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"message": "Invalid Request",
})
return
}
// Write to database
h.dbClient.Create(&db.Project{
ProjectReferenceId: uuid.New(),
Name: projectBody.Name,
Team: projectBody.Team,
})
c.JSON(http.StatusOK, gin.H{
"message": "Project created",
})
h.projectCreatorService.CreateProject(c)
}
func (h *ProjectHandler) ProjectGet(c *gin.Context) {
var project db.Project
h.dbClient.First(&project, 1)
c.JSON(http.StatusOK, gin.H{
"message": project,
})
h.projectCreatorService.GetProject(c)
}
func NewProjectHandler(dbClient *gorm.DB) *ProjectHandler {
func NewProjectHandler(projectCreatorService *service.ProjectCreator) *ProjectHandler {
return &ProjectHandler{
dbClient: dbClient,
projectCreatorService: projectCreatorService,
}
}

5
internal/transport/router/http-client.env.json Normal file
View File

@@ -0,0 +1,5 @@
{
"dev": {
"name": "value"
}
}

2
internal/transport/router/project.go
View File

@@ -7,7 +7,7 @@ import (
)
func ProjectRouter(r *gin.Engine, dep *dependencies.Dependencies) {
projectHandler := handler.NewProjectHandler(dep.DBClient)
projectHandler := handler.NewProjectHandler(dep.Service.ProjectService)
projectRouterGroup := r.Group("/api/v1")
{
projectRouterGroup.POST("/project", projectHandler.ProjectCreate)

28
pkg/encoder/encoder.go Normal file
View File

@@ -0,0 +1,28 @@
package encoder
import (
"encoding/json"
"fmt"
"google.golang.org/protobuf/proto"
)
type Encoder interface {
Encode(v interface{}) ([]byte, error)
}
type JsonEncoder struct{}
type ProtoEncoder struct{}
var JsonEncoderInstance = &JsonEncoder{}
var ProtoEncoderInstance = &ProtoEncoder{}
func (j *JsonEncoder) Encode(v interface{}) ([]byte, error) {
return json.Marshal(v)
}
func (p *ProtoEncoder) Encode(v interface{}) ([]byte, error) {
if message, ok := v.(proto.Message); ok {
return proto.Marshal(message)
}
return nil, fmt.Errorf("not a proto message")
}

139
pkg/httpClient/httpClient.go Normal file
View File

@@ -0,0 +1,139 @@
package httpclient
import (
"bytes"
"context"
"cybertron/configs"
"cybertron/pkg/log"
"cybertron/pkg/metrics"
"cybertron/pkg/utils"
"errors"
"fmt"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
"net/http"
"net/url"
"path"
"time"
)
type HttpClient interface {
Do(req *http.Request) (*http.Response, error)
}
type CircuitBreaker interface {
ExecuteRequest(req func() (interface{}, error)) (interface{}, error)
}
var logger = log.Log
// todo - custom configs for all clients
func NewHttpClient(httpConfig configs.HttpConfig) HttpClient {
return &http.Client{
Transport: &http.Transport{
MaxIdleConns: httpConfig.MaxIdleConnectionPool,
MaxConnsPerHost: httpConfig.MaxConnection,
},
Timeout: time.Duration(httpConfig.MaxTimeoutInSeconds * time.Second.Nanoseconds()),
}
}
// GetHttpRequest is a generic function to create a http request with the given method, url and body.
// Accepts the body as a proto message. Adds the necessary headers.
func GetHttpRequest(requestMetadata *context.Context, method string, url string, body proto.Message) (*http.Request, error) {
requestBody, _ := proto.Marshal(body)
req, err := http.NewRequest(method, url, bytes.NewBuffer(requestBody))
if err != nil {
return nil, err
}
err = addHeadersToRequest(req, requestMetadata)
if err != nil {
return nil, err
}
return req, nil
}
// addHeadersToRequest adds the necessary headers to the request.
func addHeadersToRequest(req *http.Request, requestMetadata *context.Context) error {
//correlationId := metadata.GetRequestMetadata(requestMetadata, utils.CORRELATION_ID_HEADER)
//if correlationId == nil {
// return fmt.Errorf("correlation id not found in request metadata")
//}
//req.Header.Add(utils.CORRELATION_ID_HEADER, *correlationId)
//
//saCustomerId := metadata.GetRequestMetadata(requestMetadata, utils.CUSTOMER_ID_HEADER)
//if saCustomerId != nil {
// req.Header.Add(utils.CUSTOMER_ID_HEADER, *saCustomerId)
//}
return nil
}
// GenerateUrl builds a url for the given endpoint and query params. To append path params to the url,
// pass each part of the path as a parameter. For example, if the endpoint is /billers/{billerId}/bills,
// then pass "billers", "{billerId}" and "bills" as parameters, in the same order.
func GenerateUrl(baseUrl, apiVersion string, queryParams map[string]string, endpoint ...string) (*url.URL, error) {
url, err := url.Parse(baseUrl)
if err != nil {
logger.Error(err.Error())
return nil, err
}
url.Path = path.Join(url.Path, apiVersion, path.Join(endpoint...))
if queryParams != nil {
query := url.Query()
for key, value := range queryParams {
query.Add(key, value)
}
url.RawQuery = query.Encode()
}
return url, nil
}
// HttpCallToClientWithCB is a generic function to make an api call to client service with circuit breaker pattern.
// Returns error only when http response is nil. This is because, in case of internal failures, downstream services are expected
// to share the error details in the response body.
func HttpCallToClientWithCB(requestMetadata *context.Context, req *http.Request, httpClient *HttpClient, circuitBreaker CircuitBreaker) (*http.Response, error) {
resp, err := circuitBreaker.ExecuteRequest(func() (interface{}, error) {
return HttpCallToClient(req, httpClient)
})
if resp == nil {
return nil, err
}
if httpResponse, ok := resp.(*http.Response); ok && httpResponse != nil {
if utils.IsErrorStatusCode(httpResponse.StatusCode) {
logger.ErrorWithCtx(requestMetadata, "api call failed", zap.String("url", req.URL.String()), zap.Int("status_code", httpResponse.StatusCode))
if httpResponse.Body == nil {
return nil, errors.New("api call failed: response body is nil")
}
} else {
logger.InfoWithCtx(requestMetadata, "api call successful", zap.String("url", req.URL.String()), zap.Int("status_code", httpResponse.StatusCode))
}
return httpResponse, nil
}
log.Log.ErrorWithCtx(requestMetadata, "unexpected response", zap.String("response", utils.ConvertToString(resp)))
return nil, errors.New("unexpected response type")
}
// HttpCallToClient is a generic function to make an api call to client service. It expects
// all the headers to be set in the request. Also records the metrics for the api call.
// In error scenario, downstream service is expected to return 4xx or 5xx status code with error details in the response body.
// Errors returned by this method will increase the failure counter in circuit breaker. Error is only returned if status code is 5XX.
func HttpCallToClient(req *http.Request, httpClient *HttpClient) (*http.Response, error) {
httpMethodCall := func(req *http.Request) (*http.Response, error) {
resp, err := (*httpClient).Do(req)
if err != nil {
return nil, err
}
// Only returning error in cases we want CB to consider failure.
if resp.StatusCode >= 500 {
return resp, errors.New("api call failed with status code: " + fmt.Sprintf("%d", resp.StatusCode))
}
return resp, nil
}
return metrics.RecordClientHttpCallMetrics(req, httpMethodCall)
}

77
pkg/kafka/config.go Normal file
View File

@@ -0,0 +1,77 @@
package kafka
import (
"crypto/tls"
"cybertron/configs"
"github.com/IBM/sarama"
"time"
)
func SaramaSyncProducer(env string, baseConfig *configs.KafkaConfig) (sarama.AsyncProducer, error) {
return sarama.NewAsyncProducer(baseConfig.Brokers, kafkaProducerConfig(env, baseConfig))
}
func SaramaKafkaConsumer(env string, baseConfig *configs.KafkaConfig, groupID string) (sarama.ConsumerGroup, error) {
consumerGroup, err := sarama.NewConsumerGroup(baseConfig.Brokers, groupID, kafkaConsumerConfig(env, baseConfig, groupID))
if err != nil {
return nil, err
}
return consumerGroup, nil
}
func kafkaProducerConfig(env string, baseConfig *configs.KafkaConfig) *sarama.Config {
kafkaConfig := kafkaConfiguration(env, baseConfig)
kafkaConfig.Producer.Retry.Max = 3
kafkaConfig.Producer.RequiredAcks = sarama.WaitForLocal
kafkaConfig.Producer.Compression = sarama.CompressionSnappy
kafkaConfig.Producer.Return.Successes = true
kafkaConfig.Producer.Flush.Bytes = 31000
kafkaConfig.Producer.Flush.Frequency = 100 * time.Millisecond
kafkaConfig.Producer.Flush.Messages = 100
kafkaConfig.ChannelBufferSize = 512
kafkaConfig.Producer.MaxMessageBytes = 2000000
kafkaConfig.Metadata.RefreshFrequency = 1 * time.Minute
return kafkaConfig
}
func kafkaConsumerConfig(env string, baseConfig *configs.KafkaConfig, groupId string) *sarama.Config {
kConfig := kafkaConfiguration(env, baseConfig)
kConfig.Version = sarama.V3_6_0_0
kConfig.Consumer.Offsets.Initial = sarama.OffsetNewest
kConfig.Consumer.MaxProcessingTime = 50 * time.Millisecond
kConfig.Consumer.Return.Errors = true
kConfig.ClientID = groupId
kConfig.Consumer.Group.Rebalance.GroupStrategies = []sarama.BalanceStrategy{sarama.NewBalanceStrategyRoundRobin()}
return kConfig
}
func kafkaConfiguration(env string, baseConfig *configs.KafkaConfig) *sarama.Config {
kafkaConfig := sarama.NewConfig()
if env == "local" {
return kafkaConfig
} else if env == "prod" {
kafkaConfig.Net.SASL.Mechanism = sarama.SASLTypePlaintext
} else {
kafkaConfig.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
kafkaConfig.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &XDGSCRAMClient{HashGeneratorFcn: SHA512}
}
}
kafkaConfig.Net.SASL.User = baseConfig.Username
kafkaConfig.Net.SASL.Password = baseConfig.Password
kafkaConfig.Net.SASL.Enable = baseConfig.SaslEnabled
kafkaConfig.Net.TLS.Enable = baseConfig.TlsEnabled
kafkaConfig.Net.TLS.Config = &tls.Config{
InsecureSkipVerify: baseConfig.TlsInsureSkipVerification,
}
return kafkaConfig
}

99
pkg/kafka/producer/produce.go Normal file
View File

@@ -0,0 +1,99 @@
package producer
import (
"cybertron/configs"
"cybertron/pkg/encoder"
"cybertron/pkg/kafka"
"cybertron/pkg/log"
"github.com/IBM/sarama"
"go.uber.org/zap"
"os"
)
type KProducer interface {
Errors()
Successes()
PublishEvent(msgPayload interface{}, topic, key string, headers map[string]string, marshaller encoder.Encoder) error
sendMessage(msgPayload []byte, topic, key string, headers map[string]string) error
}
type KProducerImpl struct {
AsyncProducer sarama.AsyncProducer
}
func NewKProducer(env string, baseConfig *configs.KafkaConfig) *KProducerImpl {
producer, err := kafka.SaramaSyncProducer(env, baseConfig)
if err != nil {
log.Log.Error("sarama kafka producer failed", zap.Error(err))
os.Exit(1)
}
kProducer := &KProducerImpl{
AsyncProducer: producer,
}
go func() {
kProducer.Errors()
}()
go func() {
kProducer.Successes()
}()
return kProducer
}
// Errors keep the track of failed messages.
func (kp *KProducerImpl) Errors() {
for err := range kp.AsyncProducer.Errors() {
keyBytes, errEncode := err.Msg.Key.Encode()
if errEncode != nil {
log.Log.Error("key encoding failed for failed message", zap.String("topic", err.Msg.Topic))
}
// todo - add metrics
log.Log.Error("failed to emit event to kafka", zap.String("topic", err.Msg.Topic),
zap.String("key", string(keyBytes)), zap.String("value", string(keyBytes)), zap.String("error", err.Error()))
}
}
// Successes is to check if message successfully delivered to kafka
func (kp *KProducerImpl) Successes() {
for msg := range kp.AsyncProducer.Successes() {
_, errEncode := msg.Key.Encode()
if errEncode != nil {
log.Log.Error("key encoding failed for failed message", zap.String("topic", msg.Topic))
}
// todo - add metrics
}
}
func (kp *KProducerImpl) sendMessage(msgPayload []byte, topic, key string, headers map[string]string) error {
var saramaHeaders []sarama.RecordHeader
for k, v := range headers {
saramaHeaders = append(saramaHeaders, sarama.RecordHeader{
Key: []byte(k),
Value: []byte(v),
})
}
message := &sarama.ProducerMessage{
Topic: topic,
Key: sarama.StringEncoder(key),
Value: sarama.StringEncoder(msgPayload),
Headers: saramaHeaders,
}
kp.AsyncProducer.Input() <- message
// todo - add metrics
return nil
}
func (kp *KProducerImpl) PublishEvent(msgPayload interface{}, topic, key string, headers map[string]string, marshaller encoder.Encoder) error {
msg, err := marshaller.Encode(msgPayload)
if err != nil {
log.Log.Error("error while marshalling msg payload", zap.Error(err))
return err
}
return kp.sendMessage(msg, topic, key, headers)
}

35
pkg/kafka/scram_client.go Normal file
View File

@@ -0,0 +1,35 @@
package kafka
import (
"crypto/sha512"
"github.com/xdg-go/scram"
)
var (
SHA512 scram.HashGeneratorFcn = sha512.New
)
type XDGSCRAMClient struct {
*scram.Client
*scram.ClientConversation
scram.HashGeneratorFcn
}
func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {
x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID)
if err != nil {
return err
}
x.ClientConversation = x.Client.NewConversation()
return nil
}
func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {
response, err = x.ClientConversation.Step(challenge)
return
}
func (x *XDGSCRAMClient) Done() bool {
return x.ClientConversation.Done()
}

129
pkg/log/log.go
View File

@@ -1,11 +1,15 @@
package log
import (
"github.com/gin-gonic/gin"
"context"
"go.elastic.co/ecszap"
"go.uber.org/zap"
)
const (
CORRELATION_ID_HEADER = "X-Correlation-Id"
)
type Logger struct {
log *zap.Logger
}
@@ -15,7 +19,8 @@ var Log *Logger
func initiateLogger() *zap.Logger {
config := zap.NewProductionConfig()
config.EncoderConfig = ecszap.ECSCompatibleEncoderConfig(config.EncoderConfig)
log, err := config.Build(ecszap.WrapCoreOption(), zap.AddCaller())
log, err := config.Build(ecszap.WrapCoreOption(), zap.AddCallerSkip(1))
log = log.With(zap.String("service", "t-store"))
if err != nil {
panic(err)
@@ -23,48 +28,6 @@ func initiateLogger() *zap.Logger {
return log
}
func Error(message string, fields ...zap.Field) {
Log.log.Error(appendBaseMessage(message), fields...)
}
func Warn(message string, fields ...zap.Field) {
Log.log.Warn(appendBaseMessage(message), fields...)
}
func Info(message string, fields ...zap.Field) {
Log.log.Info(appendBaseMessage(message), fields...)
}
func Fatal(message string, fields ...zap.Field) {
Log.log.Fatal(appendBaseMessage(message), fields...)
}
func Panic(message string, fields ...zap.Field) {
Log.log.Panic(appendBaseMessage(message), fields...)
}
func ErrorWithContext(c *gin.Context, message string, fields ...zap.Field) {
requestLogEntryWithCorrelationId(c).Error(appendBaseMessage(message), fields...)
}
func WarnWithContext(c *gin.Context, message string, fields ...zap.Field) {
requestLogEntryWithCorrelationId(c).Warn(appendBaseMessage(message), fields...)
}
func InfoWithContext(c *gin.Context, message string, fields ...zap.Field) {
requestLogEntryWithCorrelationId(c).Info(appendBaseMessage(message), fields...)
}
func requestLogEntryWithCorrelationId(c *gin.Context) *zap.Logger {
return Log.log.With(
zap.String("CorrelationId", c.Value("X-Correlation-Id").(string)),
)
}
func appendBaseMessage(message string) string {
return "cybertron" + message
}
func (l *Logger) GetLog() *zap.Logger {
return Log.log
}
@@ -74,3 +37,81 @@ func init() {
log: initiateLogger(),
}
}
func (l *Logger) Info(msg string, fields ...zap.Field) {
l.log.Info(msg, fields...)
}
func (l *Logger) Error(msg string, fields ...zap.Field) {
l.log.Error(msg, fields...)
}
func (l *Logger) Fatal(msg string, fields ...zap.Field) {
l.log.Fatal(msg, fields...)
}
func (l *Logger) Warn(msg string, fields ...zap.Field) {
l.log.Warn(msg, fields...)
}
func (l *Logger) Panic(msg string, fields ...zap.Field) {
l.log.Panic(msg, fields...)
}
func (l *Logger) InfoWithCtx(ctx *context.Context, msg string, fields ...zap.Field) {
correlationId := getCorrelationId(ctx)
if correlationId != "" {
fields = append(fields, zap.String("correlation_id", correlationId))
}
l.log.Info(msg, fields...)
}
func (l *Logger) ErrorWithCtx(ctx *context.Context, msg string, fields ...zap.Field) {
correlationId := getCorrelationId(ctx)
if correlationId != "" {
fields = append(fields, zap.String("correlation_id", correlationId))
}
l.log.Error(msg, fields...)
}
func (l *Logger) FatalWithCtx(ctx *context.Context, msg string, fields ...zap.Field) {
correlationId := getCorrelationId(ctx)
if correlationId != "" {
fields = append(fields, zap.String("correlation_id", correlationId))
}
l.log.Fatal(msg, fields...)
}
func (l *Logger) WarnWithCtx(ctx *context.Context, msg string, fields ...zap.Field) {
correlationId := getCorrelationId(ctx)
if correlationId != "" {
fields = append(fields, zap.String("correlation_id", correlationId))
}
l.log.Warn(msg, fields...)
}
func (l *Logger) PanicWithCtx(ctx *context.Context, msg string, fields ...zap.Field) {
correlationId := getCorrelationId(ctx)
if correlationId != "" {
fields = append(fields, zap.String("correlation_id", correlationId))
}
l.log.Panic(msg, fields...)
}
func getCorrelationId(ctx *context.Context) string {
if ctx == nil {
return ""
}
correlationId, ok := (*ctx).Value(CORRELATION_ID_HEADER).(string)
if ok {
return correlationId
}
return ""
}

6
pkg/utils/response_utils.go Normal file
View File

@@ -0,0 +1,6 @@
package utils
// IsErrorStatusCode checks if the status code is an error status code.
func IsErrorStatusCode(statusCode int) bool {
return statusCode >= 400
}

21
pkg/utils/string_utils.go Normal file
View File

@@ -0,0 +1,21 @@
package utils
import (
"fmt"
"strconv"
)
func ConvertToString(value any) string {
return fmt.Sprintf("%+v", value)
}
func ConvertStringToFloat64(value string) (float64, error) {
return strconv.ParseFloat(value, 64)
}
func GetLastNChars(str string, n int) string {
if n > len(str) {
return str
}
return str[len(str)-n:]
}

56
service/ProjectCreator.go Normal file
View File

@@ -0,0 +1,56 @@
package service
import (
"cybertron/models/db"
"cybertron/pkg/log"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"gorm.io/gorm"
"net/http"
)
type ProjectCreator struct {
logger *log.Logger
dbClient *gorm.DB
}
type ProjectBody struct {
Name string `json:"name" binding:"required"`
Team string `json:"team" binding:"required"`
}
func NewProjectCreator(logger *log.Logger, dbClient *gorm.DB) *ProjectCreator {
return &ProjectCreator{
logger: logger,
dbClient: dbClient,
}
}
func (pc *ProjectCreator) CreateProject(ctx *gin.Context) {
var projectBody ProjectBody
if err := ctx.BindJSON(&projectBody); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": "Invalid Request",
})
return
}
// Write to database
pc.dbClient.Create(&db.Project{
ProjectReferenceId: uuid.New(),
Name: projectBody.Name,
Team: projectBody.Team,
})
ctx.JSON(http.StatusOK, gin.H{
"message": "Project created",
})
}
func (pc *ProjectCreator) GetProject(ctx *gin.Context) {
var project db.Project
pc.dbClient.First(&project, 1)
ctx.JSON(http.StatusOK, gin.H{
"message": project,
})
}