TP-5555 | cleanup (#1)

* TP-5555 | cleanup

* TP-55555 clean up

* TP-55555 clean up

* TP-55555 | clean up

* TP-55555 | clean up

.air.toml

@@ -0,0 +1,51 @@
+root = "."
+testdata_dir = "testdata"
+tmp_dir = "tmp"
+
+[build]
+  args_bin = []
+  bin = "./out/log-enricher"
+  cmd = "go mod tidy && CGO_ENABLED=1 go build -ldflags=\"-s -w\" -o out/log-enricher cmd/log-enricher/main.go"
+  delay = 1000
+  exclude_dir = ["assets", "tmp", "vendor", "testdata"]
+  exclude_file = []
+  exclude_regex = ["_test.go"]
+  exclude_unchanged = false
+  follow_symlink = false
+  full_bin = ""
+  include_dir = ["bins"]
+  include_ext = ["go", "tpl", "tmpl", "html"]
+  include_file = []
+  kill_delay = "0s"
+  log = "build-errors.log"
+  poll = false
+  poll_interval = 0
+  post_cmd = []
+  pre_cmd = []
+  rerun = false
+  rerun_delay = 500
+  send_interrupt = false
+  stop_on_error = false
+
+[color]
+  app = ""
+  build = "yellow"
+  main = "magenta"
+  runner = "green"
+  watcher = "cyan"
+
+[log]
+  main_only = false
+  time = false
+
+[misc]
+  clean_on_exit = false
+
+[proxy]
+  app_port = 0
+  enabled = false
+  proxy_port = 0
+
+[screen]
+  clear_on_rebuild = false
+  keep_scroll = true

.gitignore

@@ -27,3 +27,5 @@ vendor/
 go.sum
 
 source-map-cache/
+
+tmp/

README.md

@@ -1,4 +1,4 @@
-# Cyber tron service
+# Log enricher service for cybertron project
 
 ## Frontend error monitoring service 
 
@@ -29,6 +29,15 @@ make run-all
 
 
 
+### Install Air
+```cmd
+go install github.com/air-verse/air@latest
+```
+
+### Run air
+```cmd
+air
+```
 
 
 

configs/Elastic.go

@@ -4,6 +4,7 @@ type ElasticConfig struct {
 	Addresses []string
 	Username  string
 	Password  string
+	APIKey    string
 }
 
 func getElasticConfig() *ElasticConfig {
@@ -11,5 +12,6 @@ func getElasticConfig() *ElasticConfig {
 		Addresses: getStringSlice("elastic.addresses", true),
 		Username:  getString("elastic.username", true),
 		Password:  getString("elastic.password", true),
+		APIKey:    getString("elastic.api_key", true),
 	}
 }

configs/application.yml

@@ -66,13 +66,12 @@ DocumentService:
   mock:
     generate_token: DOCUMENT_SERVICE_MOCK_GENERATE_TOKEN
 
-
 elastic:
-  addresses: http://localhost:9200
-  username: aa
-  password: aa
-  error:
-    index: cybertron
+  addresses: https://localhost:9200
+  username: elastic
+  password: 9457611267
+  index: cybertron
+  api_key: Uk1ScGFKRUJYaHF5bTJkOWRuUmY6eWhIRGpFc1lUak9sRjcxY0taMzUydw==
 
 aws:
   region: ap-south-1

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/gin-contrib/zap v0.2.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/golang-migrate/migrate/v4 v4.17.1
-	github.com/google/uuid v1.6.0
 	github.com/prometheus/client_golang v1.19.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.17.0

models/es/ErrorDocument.go

@@ -3,9 +3,11 @@ package es
 import "log-enricher/pkg/symbolicator"
 
 type ErrorDocument struct {
-	ErrorHash      string                           `json:"error_hash"`
-	StackTrace     []symbolicator.SymbolicatedFrame `json:"stack_trace"`
-	Error          string                           `json:"error"`
-	ProjectId      string                           `json:"project_id"`
-	ReleaseVersion string                           `json:"release_id"`
+	ErrorHash        string                           `json:"error_hash"`
+	StackTrace       []symbolicator.SymbolicatedFrame `json:"stack_trace"`
+	Title            string                           `json:"title"`
+	Error            string                           `json:"error"`
+	ProjectId        string                           `json:"project_id"`
+	ReleaseVersion   string                           `json:"release_id"`
+	SignificantStack string                           `json:"significant_stack"`
 }

pkg/db/elastic.go

@@ -16,8 +16,7 @@ type ElasticSearchClient struct {
 func NewElasticClient(elasticConfig configs.ElasticConfig) (*ElasticSearchClient, error) {
 	cfg := elasticsearch8.Config{
 		Addresses: elasticConfig.Addresses,
-		Username:  elasticConfig.Username,
-		Password:  elasticConfig.Password,
+		APIKey:    elasticConfig.APIKey,
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: true,
@@ -26,6 +25,9 @@ func NewElasticClient(elasticConfig configs.ElasticConfig) (*ElasticSearchClient
 	}
 
 	client, err := elasticsearch8.NewTypedClient(cfg)
+	if err != nil {
+		log.Fatalf("Error creating the client: %s", err)
+	}
 	return &ElasticSearchClient{
 		client: client,
 	}, err
@@ -34,9 +36,9 @@ func NewElasticClient(elasticConfig configs.ElasticConfig) (*ElasticSearchClient
 func (el *ElasticSearchClient) IndexDocument(document interface{}) {
 	_, err := el.client.Index("cybertron").
 		Request(document).
-		Do(context.TODO())
+		Do(context.Background())
 
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("Error ingesting the doc: %s", err)
 	}
 }

pkg/kafka/consumer/consume.go

@@ -40,6 +40,7 @@ func NewKafkaConsumers(baseConfig *configs.KafkaConfig, errorProcessor *service.
 }
 
 func (consumer *Consumer) AttachErrorListener() {
+	//todo get this from app config no hardcoding of kafka-stream
 	err := consumer.errorConsumer.SubscribeTopics([]string{"kafka-stream"}, nil)
 
 	if err != nil {
@@ -56,6 +57,7 @@ func (consumer *Consumer) AttachErrorListener() {
 			fmt.Printf("Caught signal %v: terminating\n", sig)
 			run = false
 		default:
+			//todo drive poll time from kafka config
 			ev := consumer.errorConsumer.Poll(100)
 			if ev == nil {
 				continue

service/ErrorProcessor.go

@@ -80,6 +80,7 @@ func (ep *ErrorProcessor) ProcessError(error []byte) {
 		projectId := payload.ProjectId
 		frame := &payload.Stacktrace.Frames[i]
 		//releaseId := payload.ReleaseId
+		//todo make release dynamic
 		sourceMapPath, err := ep.sourceMapFetcher.GetSourceMap(fileName, projectId, "release-1")
 		if err != nil {
 			fmt.Printf("error occured in fetching source map")
@@ -101,11 +102,13 @@ func (ep *ErrorProcessor) ProcessError(error []byte) {
 	hash := encoder.Md5Encode(string(error))
 	//creating es document
 	errorDocument := &es.ErrorDocument{
-		Error:          payload.Value,
-		StackTrace:     output.Frames,
-		ErrorHash:      hash,
-		ProjectId:      payload.ProjectId,
-		ReleaseVersion: payload.ReleaseId,
+		Error:            payload.Value,
+		Title:            payload.Type,
+		StackTrace:       output.Frames,
+		SignificantStack: output.Frames[len(output.Frames)-1].Token,
+		ErrorHash:        hash,
+		ProjectId:        payload.ProjectId,
+		ReleaseVersion:   payload.ReleaseId,
 	}
 	ep.elasticSearchClient.IndexDocument(errorDocument)
 }

service/FetchSourceMapService.go

@@ -69,6 +69,7 @@ func (sm *SourceMapService) downloadFromS3(key string, bucket string, filePath s
 
 func (sm *SourceMapService) GetSourceMap(fileName string, projectName string, releaseVersion string) (string, error) {
 	key := path.Join(projectName, releaseVersion, fileName)
+	//todo drive it from config
 	bucket := "navi-cd955a63c4476df0f00c1cea0e4a40d1"
 	cacheFilePath := filepath.Join(cacheDir, projectName, releaseVersion, fileName)
 	//checking local cache