diff --git a/.!78967!.DS_Store b/.!78967!.DS_Store new file mode 100644 index 0000000..e69de29 diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e90e11a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,22 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[{*.go,Makefile,.gitmodules,go.mod,go.sum}] +indent_style = tab + +[*.md] +indent_style = tab +trim_trailing_whitespace = false + +[*.{yml,yaml,json}] +indent_style = space +indent_size = 2 + +[*.{js,jsx,ts,tsx,css,less,sass,scss,vue,py}] +indent_style = space +indent_size = 4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f8d4a9c --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Mac OS X files +.DS_Store + +# IDE files +.idea/ +.vscode/ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output directory +out/ + +# env files +*.env + +# Dependency directories +vendor/ + +go.sum diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ecc0414 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +ARG GOLANG_TAG=193044292705.dkr.ecr.ap-south-1.amazonaws.com/common/golang:1.19 + +# To run locally, use +#ARG GOLANG_TAG=registry.cmd.navi-tech.in/common/golang:1.19 + +FROM ${GOLANG_TAG} as builder +RUN mkdir -p /build +WORKDIR /build +COPY . /build +RUN /bin/bash -c "make build-migrations" +RUN /bin/bash -c "make build-cybertron" + +FROM ${GOLANG_TAG} +RUN mkdir -p /usr/local +WORKDIR /usr/local +COPY --from=0 /build/out/cybertron /usr/local/ +COPY --from=0 /build/out/migrations /usr/local/migrations +COPY --from=0 /build/db/migrations/*.sql /usr/local/db/migrations/ +COPY --from=0 /build/configs/application.yml /usr/local/configs/ +RUN adduser --system --uid 4000 --disabled-password api-user && chown -R 4000:4000 /usr/local && chmod -R g+w /usr/local/ +USER 4000 +CMD /bin/bash -c "./cybertron" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6076a34 --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +.PHONY: clean all + +all: fmt lint build-cybertron + +run-local: + @bash ./scripts/run-local + +setup: + go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + +build-cybertron: + go mod tidy && CGO_ENABLED=1 go build -ldflags="-s -w" -o out/cybertron cmd/cybertron/main.go + +build-migrations: + go mod tidy && CGO_ENABLED=1 go build -ldflags="-s -w" -o out/migrations cmd/migrations/main.go + +clean: + rm -rf out/ + rm -f *.out + +lint: + golangci-lint run + +fmt: + @echo "Running fmt..." + @for p in $(ALL_PACKAGES); do \ + go fmt $$p | { grep -vwE "exported (var|function|method|type|const) \S+ should have comment" || true; } \ + done diff --git a/README.md b/README.md index 0aa57f5..2fe9faa 100644 --- a/README.md +++ b/README.md @@ -1 +1,4 @@ -# cybertron +cybertron service + +Important pointer - When using golang-migrate for migrations, the naming convention for migration files should be - +```202310201651_initial-migration.up.sql``` diff --git a/cmd/cybertron/main.go b/cmd/cybertron/main.go new file mode 100644 index 0000000..ab437cc --- /dev/null +++ b/cmd/cybertron/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "cybertron/internal/transport" + "github.com/spf13/cobra" + "go.uber.org/zap" + + "cybertron/configs" + "cybertron/internal/dependencies" +) + +func main() { + + configs.LoadConfig() + + rootCmd := &cobra.Command{ + Use: "cybertron", + Short: "cybertron", + Long: "cybertron", + Run: func(cmd *cobra.Command, args []string) { + dep := dependencies.InitDependencies() + server := transport.NewServer(dep) + go server.Start() + + server.Close() + }, + } + + if err := rootCmd.Execute(); err != nil { + zap.L().Fatal("Error", zap.Error(err)) + } +} diff --git a/cmd/migrations/main.go b/cmd/migrations/main.go new file mode 100644 index 0000000..8cda7b7 --- /dev/null +++ b/cmd/migrations/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "cybertron/configs" + mig "cybertron/db" + "cybertron/pkg/log" + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +func main() { + configs.LoadMigConfig() + + command := &cobra.Command{ + Use: "migrations", + Short: "migrations", + Long: "running DB migrations", + RunE: func(cmd *cobra.Command, args []string) error { + err := mig.RunDatabaseMigrations() + if err != nil { + log.Fatal("cybertron migrations failed", zap.Error(err)) + } + return nil + }, + } + + if err := command.Execute(); err != nil { + log.Fatal("cybertron migrations command execution failed", zap.Error(err)) + } +} diff --git a/configs/application.yml b/configs/application.yml new file mode 100644 index 0000000..8bca28c --- /dev/null +++ b/configs/application.yml @@ -0,0 +1,35 @@ +port: 9000 +name: cybertron +env: local +metrics: + port: 4001 +timezone: Asia/Kolkata + +#DB config +db: + connection: + max: + lifetime: 3600s + idle: + time: 300s + connections: + max: + idle: 10 + open: 300 + username: postgres + password: admin + host: localhost + port: 5432 + name: cybertron_dev + ssl: + mode: disable + +#Prometheus config +prometheus: + app.name: cybertron + host: localhost + port: 4001 + enabled: true + timeout: 10 + flush.interval.in.ms: 200 + histogram.buckets: 50.0,75.0,90.0,95.0,99.0 diff --git a/configs/config.go b/configs/config.go new file mode 100644 index 0000000..06e3399 --- /dev/null +++ b/configs/config.go @@ -0,0 +1,88 @@ +package configs + +import ( + "cybertron/pkg/log" + "strings" + + "github.com/spf13/viper" + "go.uber.org/zap" +) + +type AppConfig struct { + name string + env string + port int + metricsPort int + prometheus *Prometheus + postgres Postgres + timezone string +} + +type MigConfig struct { + postgresConfig Postgres +} + +var appConfig AppConfig + +var migrationConfig MigConfig + +func LoadConfig() { + readConfig() + + appConfig = AppConfig{ + name: getString("name", true), + env: getString("env", true), + port: getInt("port", true), + metricsPort: getInt("metrics.port", true), + prometheus: GetPrometheusConfig(), + postgres: getPostgresConfig(), + timezone: getString("timezone", true), + } +} + +func LoadMigConfig() { + readConfig() + + migrationConfig = MigConfig{postgresConfig: getPostgresConfig()} +} +func GetAppName() string { + return appConfig.name +} + +func GetEnv() string { + return appConfig.env +} + +func GetMetricsPort() int { + return appConfig.metricsPort +} + +func GetPort() int { + return appConfig.port +} + +func GetPostgresConfig() Postgres { + return appConfig.postgres +} + +func GetPostgresMigConfig() Postgres { + return migrationConfig.postgresConfig +} + +func GetTimezone() *string { + return &appConfig.timezone +} + +func readConfig() { + viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.SetConfigFile("./configs/application.yml") + + err := viper.ReadInConfig() + + if err != nil { + log.Log.GetLog().Panic("Error while loading configuration", zap.Error(err)) + } +} diff --git a/configs/postgres.go b/configs/postgres.go new file mode 100644 index 0000000..d122cfc --- /dev/null +++ b/configs/postgres.go @@ -0,0 +1,61 @@ +package configs + +import "fmt" + +type Postgres struct { + maxIdleConnectionTime string + maxConnectionLifetime string + maxOpenConnections int + maxIdleConnections int + host string + name string + port int + userName string + password string + sSLMode string +} + +func getPostgresConfig() Postgres { + return Postgres{ + maxConnectionLifetime: getString("db.connection.max.lifetime", true), + maxIdleConnectionTime: getString("db.connection.max.idle.time", true), + maxIdleConnections: getInt("db.connections.max.idle", true), + maxOpenConnections: getInt("db.connections.max.open", true), + userName: getString("db.username", true), + password: getString("db.password", true), + name: getString("db.name", true), + host: getString("db.host", true), + port: getInt("db.port", true), + sSLMode: getString("db.ssl.mode", true), + } +} + +func (p Postgres) GetPostgresUrl() string { + return fmt.Sprintf( + "postgres://%s:%s@%s:%d/%s?sslmode=%s", + p.userName, p.password, p.host, p.port, p.name, p.sSLMode, + ) +} + +func (p Postgres) GetPostgresDsn() string { + return fmt.Sprintf( + "host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", + p.host, p.port, p.userName, p.password, p.name, p.sSLMode, + ) +} + +func (p Postgres) GetMaxIdleConnectionTimeout() string { + return p.maxIdleConnectionTime +} + +func (p Postgres) GetMaxConnectionLifetime() string { + return p.maxConnectionLifetime +} + +func (p Postgres) GetMaxIdleConnections() int { + return p.maxIdleConnections +} + +func (p Postgres) GetMaxOpenConnections() int { + return p.maxOpenConnections +} diff --git a/configs/prometheus.go b/configs/prometheus.go new file mode 100644 index 0000000..d23c496 --- /dev/null +++ b/configs/prometheus.go @@ -0,0 +1,31 @@ +package configs + +type Prometheus struct { + appName string + host string + port int + flushInterval int + timeout int + enabled bool + buckets []float64 +} + +func GetPrometheusConfig() *Prometheus { + return &Prometheus{ + appName: getString("prometheus.api.name", false), + host: getString("prometheus.host", true), + port: getInt("prometheus.port", true), + enabled: getBool("prometheus.enabled", false), + timeout: getInt("prometheus.timeout", false), + flushInterval: getInt("prometheus.flush.interval.in.ms", false), + buckets: getFloatSlice("prometheus.histogram.buckets", true), + } +} + +func (p *Prometheus) GetAppName() string { + return p.appName +} + +func (p *Prometheus) GetBuckets() []float64 { + return p.buckets +} diff --git a/configs/utils.go b/configs/utils.go new file mode 100644 index 0000000..6532bc0 --- /dev/null +++ b/configs/utils.go @@ -0,0 +1,63 @@ +package configs + +import ( + "cybertron/pkg/log" + "strconv" + "strings" + + "github.com/spf13/viper" +) + +func getInt(key string, required bool) int { + if required { + checkKey(key) + } + + return viper.GetInt(key) +} + +func getString(key string, required bool) string { + if required { + checkKey(key) + } + + return viper.GetString(key) +} + +func getBool(key string, required bool) bool { + if required { + checkKey(key) + } + + return viper.GetBool(key) +} + +func getFloatSlice(key string, required bool) []float64 { + + stringValues := getStringSlice(key, required) + var floatValues []float64 + for _, val := range stringValues { + floatVal, err := strconv.ParseFloat(val, 64) + if err != nil { + log.Panic("config value is not float type, err : " + err.Error()) + } + floatValues = append(floatValues, floatVal) + } + + return floatValues +} + +func checkKey(key string) { + if !viper.IsSet(key) { + log.Panic("Missing key: " + key) + } +} + +func getStringSlice(key string, required bool) []string { + if required { + checkKey(key) + } + + stringValue := viper.GetString(key) + return strings.Split(stringValue, ",") +} diff --git a/db/migrations.go b/db/migrations.go new file mode 100644 index 0000000..e0b9b63 --- /dev/null +++ b/db/migrations.go @@ -0,0 +1,30 @@ +package db + +import ( + "cybertron/configs" + "cybertron/pkg/log" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" + "go.uber.org/zap" +) + +const dbMigrationsPath = "./db/migrations" + +func RunDatabaseMigrations() error { + var err error + postgresConfig := configs.GetPostgresMigConfig() + appMigrate, err := migrate.New("file://"+dbMigrationsPath, postgresConfig.GetPostgresUrl()) + if err != nil { + log.Log.GetLog().Error("migrations error", zap.Error(err)) + panic(err) + } + err = appMigrate.Up() + if err != nil && err != migrate.ErrNoChange { + log.Error("migrations error", zap.Error(err)) + return err + } + log.Info("migrations successful") + return nil +} diff --git a/db/migrations/202310201651_initial-migration.up.sql b/db/migrations/202310201651_initial-migration.up.sql new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..02c2346 --- /dev/null +++ b/go.mod @@ -0,0 +1,77 @@ +module cybertron + +go 1.21.1 + +require ( + github.com/gin-contrib/zap v0.2.0 + github.com/gin-gonic/gin v1.9.1 + github.com/golang-migrate/migrate/v4 v4.17.1 + github.com/prometheus/client_golang v1.19.1 + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.17.0 + go.elastic.co/ecszap v1.0.2 + go.uber.org/zap v1.26.0 + gorm.io/driver/postgres v1.5.3 + gorm.io/gorm v1.25.5 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic v1.10.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.15.5 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.5.4 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/sagikazarmark/locafero v0.3.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.10.0 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.5.0 // indirect + golang.org/x/crypto v0.20.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/internal/dependencies/dependencies.go b/internal/dependencies/dependencies.go new file mode 100644 index 0000000..5d27c4b --- /dev/null +++ b/internal/dependencies/dependencies.go @@ -0,0 +1,32 @@ +package dependencies + +import ( + "cybertron/pkg/log" + "go.uber.org/zap" + "gorm.io/gorm" +) + +type Dependencies struct { + Service *Service + DBClient *gorm.DB + Logger *zap.Logger +} + +type Service struct { + // Add your service here +} + +func InitDependencies() *Dependencies { + services := initServices() + return &Dependencies{ + Service: services, + //DBClient: db.NewDBClient(), + Logger: log.Log.GetLog(), + } +} + +func initServices() *Service { + return &Service{ + // Add your service here + } +} diff --git a/internal/transport/handler/readiness.go b/internal/transport/handler/readiness.go new file mode 100644 index 0000000..737a19f --- /dev/null +++ b/internal/transport/handler/readiness.go @@ -0,0 +1,19 @@ +package handler + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +type HealthCheckHandler struct{} + +func (h *HealthCheckHandler) Readiness(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "message": "pong", + }) +} + +func NewReadinessHandler() *HealthCheckHandler { + return &HealthCheckHandler{} +} diff --git a/internal/transport/middleware/metric_middleware.go b/internal/transport/middleware/metric_middleware.go new file mode 100644 index 0000000..320328e --- /dev/null +++ b/internal/transport/middleware/metric_middleware.go @@ -0,0 +1,30 @@ +package middleware + +import ( + "cybertron/models/instrumentation" + "cybertron/pkg/metrics" + "time" + + "github.com/gin-gonic/gin" +) + +func MetricMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + startTime := time.Now() + c.Next() + endTime := time.Now() + duration := endTime.Sub(startTime) + + metricsPublisher := metrics.NewMetricPublisher() + apiMetrics := instrumentation.ApiMetric{ + Url: c.FullPath(), + ResponseCode: c.Writer.Status(), + StartTime: startTime.Unix(), + EndTime: endTime.Unix(), + DurationInMs: duration.Milliseconds(), + Method: c.Request.Method, + BytesSent: c.Writer.Size(), + } + metricsPublisher.PublishMetrics(instrumentation.MetricAttributes{ApiMetric: apiMetrics}, instrumentation.API_METRICS) + } +} diff --git a/internal/transport/router/readiness.go b/internal/transport/router/readiness.go new file mode 100644 index 0000000..f09d576 --- /dev/null +++ b/internal/transport/router/readiness.go @@ -0,0 +1,12 @@ +package router + +import ( + "cybertron/internal/transport/handler" + + "github.com/gin-gonic/gin" +) + +func ReadinessRouter(r *gin.Engine) { + readinessHandler := handler.NewReadinessHandler() + r.GET("/ping", readinessHandler.Readiness) +} diff --git a/internal/transport/server.go b/internal/transport/server.go new file mode 100644 index 0000000..7adb827 --- /dev/null +++ b/internal/transport/server.go @@ -0,0 +1,52 @@ +package transport + +import ( + "cybertron/internal/transport/router" + "fmt" + "os" + "os/signal" + "syscall" + + "cybertron/configs" + "cybertron/internal/dependencies" + + ginzap "github.com/gin-contrib/zap" + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type Server struct { + gin *gin.Engine + dependencies *dependencies.Dependencies +} + +func NewServer(dep *dependencies.Dependencies) *Server { + return &Server{ + gin: gin.New(), + dependencies: dep, + } +} + +func (s *Server) router() { + router.ReadinessRouter(s.gin) +} + +func (s *Server) Start() { + s.gin.Use(ginzap.RecoveryWithZap(s.dependencies.Logger, true)) + s.router() + + port := configs.GetPort() + s.dependencies.Logger.Info("Starting server", zap.Int("port", port)) + + err := s.gin.Run(fmt.Sprintf(":%v", port)) + if err != nil { + s.dependencies.Logger.Fatal("error while starting the server", zap.Error(err)) + } +} + +func (s *Server) Close() { + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + s.dependencies.Logger.Info("Shutdown server ...") +} diff --git a/models/instrumentation/performance_metrics.go b/models/instrumentation/performance_metrics.go new file mode 100644 index 0000000..333752f --- /dev/null +++ b/models/instrumentation/performance_metrics.go @@ -0,0 +1,34 @@ +package instrumentation + +type MetricType string + +const ( + API_METRICS MetricType = "API_METRICS" + CLIENT_HTTP_CALL_METRICS MetricType = "CLIENT_HTTP_CALL_METRICS" +) + +type ApiMetric struct { + Url string `json:"url,omitempty"` + Method string `json:"method,omitempty"` + ResponseCode int `json:"response_code,omitempty"` + BytesSent int `json:"bytes_sent,omitempty"` + BytesReceived int64 `json:"bytes_received,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + DurationInMs int64 `json:"duration_in_ms,omitempty"` + ErrorType string `json:"error_type,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +type ClientHttpCallMetric struct { + Url string `json:"url,omitempty"` + ResponseCode int `json:"response_code,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + DurationInMs int64 `json:"duration_in_ms,omitempty"` +} + +type MetricAttributes struct { + ApiMetric ApiMetric + ClientHttpCallMetric ClientHttpCallMetric +} diff --git a/pkg/db/postgres.go b/pkg/db/postgres.go new file mode 100644 index 0000000..dc0546f --- /dev/null +++ b/pkg/db/postgres.go @@ -0,0 +1,50 @@ +package db + +import ( + "cybertron/configs" + "cybertron/pkg/log" + "os" + "time" + + "go.uber.org/zap" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var logger = log.Log.GetLog() + +func NewDBClient() *gorm.DB { + postgresConfig := configs.GetPostgresConfig() + db, err := gorm.Open(postgres.Open(postgresConfig.GetPostgresDsn()), &gorm.Config{}) + if err != nil { + logger.Error("database connection failed", zap.Error(err)) + os.Exit(1) + } + sqlDB, err := db.DB() + if err != nil { + logger.Fatal(err.Error()) + } + + connMaxIdleDuration, err := time.ParseDuration(postgresConfig.GetMaxIdleConnectionTimeout()) + if err != nil { + logger.Fatal(err.Error()) + } + + connMaxLifetimeDuration, err := time.ParseDuration(postgresConfig.GetMaxConnectionLifetime()) + if err != nil { + logger.Fatal(err.Error()) + } + + sqlDB.SetConnMaxIdleTime(time.Duration(connMaxIdleDuration.Seconds())) + sqlDB.SetConnMaxLifetime(time.Duration(connMaxLifetimeDuration.Seconds())) + sqlDB.SetMaxIdleConns(postgresConfig.GetMaxIdleConnections()) + sqlDB.SetMaxOpenConns(postgresConfig.GetMaxOpenConnections()) + + err = sqlDB.Ping() + if err != nil { + logger.Fatal(err.Error()) + } + + logger.Info("database connection established") + return db +} diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..11f972c --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,76 @@ +package log + +import ( + "github.com/gin-gonic/gin" + "go.elastic.co/ecszap" + "go.uber.org/zap" +) + +type Logger struct { + log *zap.Logger +} + +var Log *Logger + +func initiateLogger() *zap.Logger { + config := zap.NewProductionConfig() + config.EncoderConfig = ecszap.ECSCompatibleEncoderConfig(config.EncoderConfig) + log, err := config.Build(ecszap.WrapCoreOption(), zap.AddCaller()) + + if err != nil { + panic(err) + } + return log +} + +func Error(message string, fields ...zap.Field) { + Log.log.Error(appendBaseMessage(message), fields...) +} + +func Warn(message string, fields ...zap.Field) { + Log.log.Warn(appendBaseMessage(message), fields...) +} + +func Info(message string, fields ...zap.Field) { + Log.log.Info(appendBaseMessage(message), fields...) +} + +func Fatal(message string, fields ...zap.Field) { + Log.log.Fatal(appendBaseMessage(message), fields...) +} + +func Panic(message string, fields ...zap.Field) { + Log.log.Panic(appendBaseMessage(message), fields...) +} + +func ErrorWithContext(c *gin.Context, message string, fields ...zap.Field) { + requestLogEntryWithCorrelationId(c).Error(appendBaseMessage(message), fields...) +} + +func WarnWithContext(c *gin.Context, message string, fields ...zap.Field) { + requestLogEntryWithCorrelationId(c).Warn(appendBaseMessage(message), fields...) +} + +func InfoWithContext(c *gin.Context, message string, fields ...zap.Field) { + requestLogEntryWithCorrelationId(c).Info(appendBaseMessage(message), fields...) +} + +func requestLogEntryWithCorrelationId(c *gin.Context) *zap.Logger { + return Log.log.With( + zap.String("CorrelationId", c.Value("X-Correlation-Id").(string)), + ) +} + +func appendBaseMessage(message string) string { + return "cybertron" + message +} + +func (l *Logger) GetLog() *zap.Logger { + return Log.log +} + +func init() { + Log = &Logger{ + log: initiateLogger(), + } +} diff --git a/pkg/metrics/http_client_metrics_recorder.go b/pkg/metrics/http_client_metrics_recorder.go new file mode 100644 index 0000000..8d1ba2b --- /dev/null +++ b/pkg/metrics/http_client_metrics_recorder.go @@ -0,0 +1,36 @@ +package metrics + +import ( + "cybertron/models/instrumentation" + "net/http" + "time" +) + +type ClientHttpCall func(req *http.Request) (*http.Response, error) + +func RecordClientHttpCallMetrics(req *http.Request, method ClientHttpCall) (*http.Response, error) { + startTime := time.Now() + + resp, err := method(req) + + endTime := time.Now() + duration := endTime.Sub(startTime) + + metricsPublisher := NewMetricPublisher() + + clientHttpCallMetrics := instrumentation.ClientHttpCallMetric{ + Url: req.URL.Path, + StartTime: startTime.Unix(), + EndTime: endTime.Unix(), + DurationInMs: duration.Milliseconds(), + } + + if resp != nil { + clientHttpCallMetrics.ResponseCode = resp.StatusCode + } + + metricsPublisher.PublishMetrics(instrumentation.MetricAttributes{ClientHttpCallMetric: clientHttpCallMetrics}, + instrumentation.CLIENT_HTTP_CALL_METRICS) + + return resp, err +} diff --git a/pkg/metrics/metric_publisher.go b/pkg/metrics/metric_publisher.go new file mode 100644 index 0000000..75f4c0f --- /dev/null +++ b/pkg/metrics/metric_publisher.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "cybertron/models/instrumentation" + "strconv" +) + +type Publisher interface { + PublishMetrics(metricAttributes map[string]interface{}, metricType instrumentation.MetricType) +} + +type PublisherImpl struct { +} + +func NewMetricPublisher() *PublisherImpl { + return &PublisherImpl{} +} + +func (amp *PublisherImpl) PublishMetrics(metricAttributes instrumentation.MetricAttributes, metricType instrumentation.MetricType) { + switch metricType { + case instrumentation.API_METRICS: + publishApiMetric(metricAttributes.ApiMetric) + return + case instrumentation.CLIENT_HTTP_CALL_METRICS: + publishClientHttpCallMetric(metricAttributes.ClientHttpCallMetric) + return + default: + return + } +} + +func publishApiMetric(apiMetrics instrumentation.ApiMetric) { + status := strconv.Itoa(apiMetrics.ResponseCode) + duration := float64(apiMetrics.DurationInMs) + ApiRequestCounter.WithLabelValues(apiMetrics.Url, status).Inc() + ApiRequestLatencyHistogram.WithLabelValues(apiMetrics.Url, status).Observe(duration) +} + +func publishClientHttpCallMetric(clientHttpCallMetric instrumentation.ClientHttpCallMetric) { + status := strconv.Itoa(clientHttpCallMetric.ResponseCode) + duration := float64(clientHttpCallMetric.DurationInMs) + HttpCallRequestCounter.WithLabelValues(clientHttpCallMetric.Url, status).Inc() + HttpCallRequestLatencyHistogram.WithLabelValues(clientHttpCallMetric.Url, status).Observe(duration) +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..9b86372 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,42 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var metricsBuckets = []float64{5, 10, 20, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 20000, 30000, 60000} + +var ApiRequestCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "cybertron_api_request_total", + Help: "api request counter", + }, + []string{"url", "response_code"}, +) + +var ApiRequestLatencyHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "cybertron_api_request_latency_histogram", + Help: "api latency histogram", + Buckets: metricsBuckets, + }, + []string{"url", "response_code"}, +) + +var HttpCallRequestCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "cybertron_http_call_request_total", + Help: "http call request counter", + }, + []string{"url", "response_code"}, +) + +var HttpCallRequestLatencyHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "cybertron_http_call_request_latency_histogram", + Help: "http call latency histogram", + Buckets: metricsBuckets, + }, + []string{"url", "response_code"}, +) diff --git a/pkg/metrics/server.go b/pkg/metrics/server.go new file mode 100644 index 0000000..00e7bfc --- /dev/null +++ b/pkg/metrics/server.go @@ -0,0 +1,24 @@ +package metrics + +import ( + "cybertron/configs" + "cybertron/pkg/log" + "fmt" + + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" +) + +func AdminHandler() { + ginServer := gin.New() + port := configs.GetMetricsPort() + log.Log.GetLog().Info("Starting metrics on port", zap.Int("port", port)) + ginServer.GET("/metrics", gin.WrapH(promhttp.Handler())) + go func() { + err := ginServer.Run(fmt.Sprintf(":%v", port)) + if err != nil { + panic(err) + } + }() +} diff --git a/scripts/run-local b/scripts/run-local new file mode 100644 index 0000000..22fa932 --- /dev/null +++ b/scripts/run-local @@ -0,0 +1,11 @@ +# !/usr/bin/env bash +ENVS="" +while read -r line; do + if [[ "$line" == *"#"* ]]; then + continue + fi + ENVS+="$line "; done < local.env + +echo "Building cybertron locally" +eval "make build-cybertron +$ENVS out/cybertron"