diff --git a/.air.toml b/.air.toml new file mode 100644 index 0000000..5ff19f9 --- /dev/null +++ b/.air.toml @@ -0,0 +1,51 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] + args_bin = [] + bin = "./out/cybertron" + cmd = "go mod tidy && CGO_ENABLED=1 go build -ldflags=\"-s -w\" -o out/cybertron cmd/cybertron/main.go" + delay = 1000 + exclude_dir = ["assets", "tmp", "vendor", "testdata"] + exclude_file = [] + exclude_regex = ["_test.go"] + exclude_unchanged = false + follow_symlink = false + full_bin = "" + include_dir = [] + include_ext = ["go", "tpl", "tmpl", "html"] + include_file = [] + kill_delay = "0s" + log = "build-errors.log" + poll = false + poll_interval = 0 + post_cmd = [] + pre_cmd = [] + rerun = false + rerun_delay = 500 + send_interrupt = false + stop_on_error = false + +[color] + app = "" + build = "yellow" + main = "magenta" + runner = "green" + watcher = "cyan" + +[log] + main_only = false + time = false + +[misc] + clean_on_exit = false + +[proxy] + app_port = 0 + enabled = false + proxy_port = 0 + +[screen] + clear_on_rebuild = false + keep_scroll = true diff --git a/.gitignore b/.gitignore index f8d4a9c..c49a829 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ out/ vendor/ go.sum + +tmp/ diff --git a/configs/Elastic.go b/configs/Elastic.go new file mode 100644 index 0000000..b6ed014 --- /dev/null +++ b/configs/Elastic.go @@ -0,0 +1,19 @@ +package configs + +type ElasticConfig struct { + Addresses []string + Username string + Password string + Index string + APIKey string +} + +func NewElasticConfig() *ElasticConfig { + return &ElasticConfig{ + Addresses: getStringSlice("elastic.addresses", true), + Username: getString("elastic.username", true), + Password: getString("elastic.password", true), + Index: getString("elastic.index", true), + APIKey: getString("elastic.api_key", true), + } +} diff --git a/configs/application.yml b/configs/application.yml index 0b505f0..c27e716 100644 --- a/configs/application.yml +++ b/configs/application.yml @@ -42,7 +42,7 @@ http: # Kafka config kafka: - password: xxxa + password: kDia1uC.GI;)Al5eQ)+Q username: varnitgoyal/varnitgoyal95@gmail.com/ocid1.streampool.oc1.ap-mumbai-1.amaaaaaaotdslraanepwp54txqqxkmg4l6dghrhufiezqkx2lqhndgxoq7pa brokers: cell-1.streaming.ap-mumbai-1.oci.oraclecloud.com:9092 group: @@ -66,6 +66,14 @@ DocumentService: mock: generate_token: DOCUMENT_SERVICE_MOCK_GENERATE_TOKEN +elastic: + addresses: https://localhost:9200 + username: elastic + password: 9457611267 + index: cybertron + api_key: U3NUSmFKRUJYaHF5bTJkOUozUU06Z1ZDTE9hZm1RUnlXeHRNY21yeGxfQQ== + + aws: region: ap-south-1 diff --git a/configs/config.go b/configs/config.go index 7f9f8fd..33e4b16 100644 --- a/configs/config.go +++ b/configs/config.go @@ -20,6 +20,7 @@ type AppConfig struct { clientConfigs *ClientConfigs awsConfig *AwsConfig KafkaConfig *KafkaConfig + ElasticConfig *ElasticConfig } type MigConfig struct { @@ -45,6 +46,7 @@ func LoadConfig() { httpConfig: NewHttpConfig(), awsConfig: NewAWSConfig(), KafkaConfig: NewKafkaConfig(), + ElasticConfig: NewElasticConfig(), } } @@ -106,3 +108,7 @@ func GetHttpConfig() *HttpConfig { func GetKafkaConfig() *KafkaConfig { return appConfig.KafkaConfig } + +func GetElasticConfig() *ElasticConfig { + return appConfig.ElasticConfig +} diff --git a/go.mod b/go.mod index 580ea70..fc86b5f 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.27.27 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 github.com/confluentinc/confluent-kafka-go/v2 v2.5.0 + github.com/elastic/go-elasticsearch/v8 v8.14.0 + github.com/gin-contrib/cors v1.7.2 github.com/gin-contrib/zap v0.2.0 github.com/gin-gonic/gin v1.9.1 github.com/golang-migrate/migrate/v4 v4.17.1 @@ -16,7 +18,7 @@ require ( github.com/xdg-go/scram v1.1.1 go.elastic.co/ecszap v1.0.2 go.uber.org/zap v1.26.0 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.0 gorm.io/driver/postgres v1.5.3 gorm.io/gorm v1.25.5 ) @@ -39,16 +41,20 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect github.com/aws/smithy-go v1.20.3 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.10.2 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect - github.com/chenzhuoyu/iasm v0.9.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.15.5 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -62,15 +68,15 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.8 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect @@ -84,17 +90,20 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.5.0 // indirect + golang.org/x/arch v0.7.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.19.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/internal/client/elastic/elastic.go b/internal/client/elastic/elastic.go new file mode 100644 index 0000000..23442d7 --- /dev/null +++ b/internal/client/elastic/elastic.go @@ -0,0 +1,97 @@ +package elastic + +import ( + "context" + "crypto/tls" + "cybertron/configs" + "encoding/json" + elasticsearch8 "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "log" + "net/http" +) + +type ElasticSearchClient struct { + client *elasticsearch8.TypedClient + Config configs.ElasticConfig +} + +func NewElasticClient(elasticConfig configs.ElasticConfig) (*ElasticSearchClient, error) { + cfg := elasticsearch8.Config{ + Addresses: elasticConfig.Addresses, + APIKey: elasticConfig.APIKey, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + + client, err := elasticsearch8.NewTypedClient(cfg) + return &ElasticSearchClient{ + client: client, + Config: elasticConfig, + }, err +} + +func (el *ElasticSearchClient) IndexDocument(document interface{}) { + _, err := el.client.Index(el.Config.Index). + Request(document). + Do(context.TODO()) + + if err != nil { + log.Fatal(err) + } +} + +func (el *ElasticSearchClient) SearchDocuments(query *types.Query, fields []string) ([]map[string]interface{}, error) { + res, err := el.client.Search(). + Index(el.Config.Index). + Request(&search.Request{ + Query: query, + }).Source_(fields). + Do(context.TODO()) + if err != nil { + log.Println("Error getting response: %s", err) + } + var results []map[string]interface{} + for _, hit := range res.Hits.Hits { + var doc map[string]interface{} + + err := json.Unmarshal(hit.Source_, &doc) + doc["id"] = *hit.Id_ + if err != nil { + log.Printf("Error unmarshalling document: %s", err) + continue + } + results = append(results, doc) + } + + return results, nil +} + +func (el *ElasticSearchClient) GetDocument(documentID string) (interface{}, error) { + // Retrieve the document by its ID + res, err := el.client.Get(el.Config.Index, documentID).Do(context.Background()) + var document interface{} + if err != nil { + log.Printf("Error getting response: %s", err) + return nil, err + } + + // Check if the document exists + if !res.Found { + log.Printf("Document with ID %s not found", documentID) + return document, nil + } + + // Unmarshal the JSON response into the provided document interface + err = json.Unmarshal(res.Source_, &document) + if err != nil { + log.Printf("Error unmarshalling document: %s", err) + return nil, err + } + + return document, nil +} diff --git a/internal/dependencies/dependencies.go b/internal/dependencies/dependencies.go index 59cd237..0af925c 100644 --- a/internal/dependencies/dependencies.go +++ b/internal/dependencies/dependencies.go @@ -4,6 +4,7 @@ import ( "cybertron/configs" "cybertron/internal/client/aws" "cybertron/internal/client/document" + "cybertron/internal/client/elastic" "cybertron/internal/database" "cybertron/internal/transport/handler" "cybertron/pkg/db" @@ -30,6 +31,7 @@ type Service struct { SourceMapService *service.SourceMapService ReleaseService *service.ReleaseService ExceptionService *service.ExceptionService + SearchService *service.SearchService S3Client *aws.Actions // Add your service here } @@ -39,6 +41,7 @@ type Handler struct { SourceMapHandler *handler.SourceMapHandler ReleaseHandler *handler.ReleasesHandler ExceptionHandler *handler.ExceptionHandler + SearchHandler *handler.SearchHandler } type Repositories struct { @@ -54,15 +57,17 @@ func InitDependencies() *Dependencies { httpClient := httpclient.NewHttpClient(*configs.GetHttpConfig()) s3Client := aws.NewS3client(*configs.GetAWSConfig()) kafkaProducer := initKafkaProducer() + elasticSearch, _ := elastic.NewElasticClient(*configs.GetElasticConfig()) documentServiceClient := document.NewDocumentServiceHttpClient(httpClient, logger, configs.GetDocumentServiceHttpClientConfigs()) projectServiceClient := service.NewProjectCreator(logger, dbClient, s3Client, kafkaProducer) sourceMapServiceClient := service.NewSourceMapService(dbClient, s3Client, configs.GetAWSConfig()) releaseServiceClient := service.NewReleaseService(logger, dbClient) exceptionServiceClient := service.NewExceptionService(logger, dbClient, kafkaProducer) + searchServiceClient := service.NewSearchService(logger, elasticSearch) - services := initServices(documentServiceClient, projectServiceClient, sourceMapServiceClient, releaseServiceClient, exceptionServiceClient) - handlers := initHandlers(projectServiceClient, sourceMapServiceClient, releaseServiceClient, exceptionServiceClient) + services := initServices(documentServiceClient, projectServiceClient, sourceMapServiceClient, releaseServiceClient, exceptionServiceClient, searchServiceClient) + handlers := initHandlers(projectServiceClient, sourceMapServiceClient, releaseServiceClient, exceptionServiceClient, searchServiceClient) return &Dependencies{ Service: services, DBClient: dbClient, @@ -72,13 +77,14 @@ func InitDependencies() *Dependencies { } } -func initServices(documentService *document.HttpClient, projectService *service.ProjectCreator, sourceMapService *service.SourceMapService, releaseService *service.ReleaseService, exceptionService *service.ExceptionService) *Service { +func initServices(documentService *document.HttpClient, projectService *service.ProjectCreator, sourceMapService *service.SourceMapService, releaseService *service.ReleaseService, exceptionService *service.ExceptionService, searchService *service.SearchService) *Service { return &Service{ DocumentService: documentService, ProjectService: projectService, SourceMapService: sourceMapService, ReleaseService: releaseService, ExceptionService: exceptionService, + SearchService: searchService, } } @@ -90,16 +96,18 @@ func initRepositories(dbClient *gorm.DB) *Repositories { } } -func initHandlers(projectService *service.ProjectCreator, sourceMapService *service.SourceMapService, releaseService *service.ReleaseService, exceotionService *service.ExceptionService) *Handler { +func initHandlers(projectService *service.ProjectCreator, sourceMapService *service.SourceMapService, releaseService *service.ReleaseService, exceotionService *service.ExceptionService, searchService *service.SearchService) *Handler { projectHandler := handler.NewProjectHandler(projectService) sourceMapHandler := handler.NewSourceMapHandler(sourceMapService) releaseHandler := handler.NewReleaseHandler(releaseService) exceptionHandler := handler.NewExceptionHandler(exceotionService) + searchHandler := handler.NewSearchHandler(searchService) return &Handler{ ProjectHandler: projectHandler, SourceMapHandler: sourceMapHandler, ReleaseHandler: releaseHandler, ExceptionHandler: exceptionHandler, + SearchHandler: searchHandler, } } diff --git a/internal/transport/handler/search.go b/internal/transport/handler/search.go new file mode 100644 index 0000000..6286c6a --- /dev/null +++ b/internal/transport/handler/search.go @@ -0,0 +1,27 @@ +package handler + +import ( + "cybertron/service" + "github.com/gin-gonic/gin" +) + +type SearchHandler struct { + searchService *service.SearchService +} + +func (h *SearchHandler) SearchErrors(c *gin.Context) { + h.searchService.GetErrorDetails(c) +} +func (h *SearchHandler) GetErrorDetails(c *gin.Context) { + h.searchService.GetErrorDetails(c) +} + +func (h *SearchHandler) GetErrorList(c *gin.Context) { + h.searchService.GetErrorList(c) +} + +func NewSearchHandler(s *service.SearchService) *SearchHandler { + return &SearchHandler{ + searchService: s, + } +} diff --git a/internal/transport/router/search.go b/internal/transport/router/search.go new file mode 100644 index 0000000..e35a69b --- /dev/null +++ b/internal/transport/router/search.go @@ -0,0 +1,17 @@ +package router + +import ( + "cybertron/internal/dependencies" + "cybertron/internal/transport/handler" + "github.com/gin-gonic/gin" +) + +func SearchRouter(r *gin.Engine, dep *dependencies.Dependencies) { + searchHandler := handler.NewSearchHandler(dep.Service.SearchService) + searchRouterGroup := r.Group("/api/v1") + { + searchRouterGroup.GET("/errors-list", searchHandler.GetErrorList) + searchRouterGroup.GET("/error-detail", searchHandler.GetErrorDetails) + } + +} diff --git a/internal/transport/server.go b/internal/transport/server.go index 6bb643f..84c80f9 100644 --- a/internal/transport/server.go +++ b/internal/transport/server.go @@ -3,6 +3,7 @@ package transport import ( "cybertron/internal/transport/router" "fmt" + "github.com/gin-contrib/cors" "os" "os/signal" "syscall" @@ -34,10 +35,19 @@ func (s *Server) router() { router.SourceMapRouter(s.gin, s.dependencies) router.ReleasesRouter(s.gin, s.dependencies) router.ExceptionRouter(s.gin, s.dependencies) + router.SearchRouter(s.gin, s.dependencies) } func (s *Server) Start() { s.gin.Use(ginzap.Ginzap(s.dependencies.Logger, time.RFC3339, false)) + s.gin.Use(cors.New(cors.Config{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{"PUT", "PATCH", "GET", "POST", "OPTIONS"}, + AllowHeaders: []string{"*", "x-session-token"}, + ExposeHeaders: []string{"Content-Length"}, + AllowCredentials: true, + MaxAge: 24 * time.Hour, + })) s.router() port := configs.GetPort() diff --git a/service/searchService.go b/service/searchService.go new file mode 100644 index 0000000..0c72508 --- /dev/null +++ b/service/searchService.go @@ -0,0 +1,49 @@ +package service + +import ( + "cybertron/internal/client/elastic" + "cybertron/pkg/log" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/gin-gonic/gin" + "net/http" +) + +type SearchService struct { + logger *log.Logger + elasticSearchClient *elastic.ElasticSearchClient +} + +func NewSearchService(logger *log.Logger, elasticSearchClient *elastic.ElasticSearchClient) *SearchService { + return &SearchService{ + logger: logger, + elasticSearchClient: elasticSearchClient, + } +} + +func (s *SearchService) Search(c *gin.Context) { + +} + +func (s *SearchService) GetErrorDetails(c *gin.Context) { + documentId := c.Query("document_id") + response, _ := s.elasticSearchClient.GetDocument(documentId) + c.JSON(http.StatusOK, response) +} + +func (s *SearchService) GetErrorList(c *gin.Context) { + //todo pagination and aggregation of errors + projectId := c.Query("project_id") + println(projectId) + query := &types.Query{ + Term: map[string]types.TermQuery{ + "project_id": { + Value: projectId, + }, + }, + } + fields := []string{"error", "significant_stack", "title"} + var response, _ = s.elasticSearchClient.SearchDocuments(query, fields) + + c.JSON(http.StatusOK, response) + +}