From 4a1328b051eb92b9df3ba29483abf7fa0274c03f Mon Sep 17 00:00:00 2001 From: varnit-goyal_navi Date: Fri, 16 Aug 2024 06:26:21 +0530 Subject: [PATCH 1/3] TP-55555/error consumer --- configs/application.yml | 2 +- internal/transport/router/exception.go | 4 +- models/db/project.go | 3 +- service/ExceptionService.go | 53 ++++++++++++++++++++++---- service/ProjectCreator.go | 17 ++++++--- service/ReleaseService.go | 3 -- 6 files changed, 61 insertions(+), 21 deletions(-) diff --git a/configs/application.yml b/configs/application.yml index 09d257f..2d752b0 100644 --- a/configs/application.yml +++ b/configs/application.yml @@ -42,7 +42,7 @@ http: # Kafka config kafka: - password: XX + password: kDia1uC.GI;)Al5eQ)+Q username: varnitgoyal/varnitgoyal95@gmail.com/ocid1.streampool.oc1.ap-mumbai-1.amaaaaaaotdslraanepwp54txqqxkmg4l6dghrhufiezqkx2lqhndgxoq7pa brokers: cell-1.streaming.ap-mumbai-1.oci.oraclecloud.com:9092 group: diff --git a/internal/transport/router/exception.go b/internal/transport/router/exception.go index e12042b..bd057c6 100644 --- a/internal/transport/router/exception.go +++ b/internal/transport/router/exception.go @@ -8,8 +8,8 @@ import ( func ExceptionRouter(r *gin.Engine, dep *dependencies.Dependencies) { exceptionHandler := handler.NewExceptionHandler(dep.Service.ExceptionService) - exceptionRouterGroup := r.Group("/api/v1") + exceptionRouterGroup := r.Group("/api") { - exceptionRouterGroup.POST("/catch-errors", exceptionHandler.CatchErrors) + exceptionRouterGroup.POST("/:projectId/envelope", exceptionHandler.CatchErrors) } } diff --git a/models/db/project.go b/models/db/project.go index b68a527..1823dcc 100644 --- a/models/db/project.go +++ b/models/db/project.go @@ -1,7 +1,6 @@ package db import ( - "github.com/google/uuid" "time" ) @@ -10,7 +9,7 @@ type Project struct { CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` DeletedAt time.Time `json:"deletedAt"` - ProjectReferenceId uuid.UUID `json:"projectReferenceId" gorm:"primaryKey"` + ProjectReferenceId string `json:"projectReferenceId" gorm:"primaryKey"` Name string `json:"name" gorm:"unique"` Team string `json:"team"` } diff --git a/service/ExceptionService.go b/service/ExceptionService.go index 5753e53..4a42559 100644 --- a/service/ExceptionService.go +++ b/service/ExceptionService.go @@ -1,13 +1,13 @@ package service import ( - "cybertron/configs" + "bufio" "cybertron/pkg/encoder" "cybertron/pkg/kafka/producer" "cybertron/pkg/log" + "encoding/json" "fmt" "github.com/gin-gonic/gin" - "github.com/google/uuid" "gorm.io/gorm" "net/http" ) @@ -18,6 +18,33 @@ type ExceptionService struct { kafkaProducer producer.KProducer } +type Frame struct { + Filename string `json:"filename"` + Function string `json:"function"` + InApp bool `json:"in_app"` + Lineno int `json:"lineno,omitempty"` + Colno int `json:"colno,omitempty"` +} + +type Stacktrace struct { + Frames []Frame `json:"frames"` +} + +type ExceptionValue struct { + Type string `json:"type"` + Value string `json:"value"` + Stacktrace Stacktrace `json:"stacktrace"` + ProjectId string `json:"project_id,omitempty"` + ReleaseId string `json:"release_id,omitempty"` +} + +type Exception struct { + Values []ExceptionValue `json:"values"` +} +type Payload struct { + Exception Exception `json:"exception"` +} + func NewExceptionService(logger *log.Logger, dbClient *gorm.DB, kafkaProducer producer.KProducer) *ExceptionService { return &ExceptionService{ logger: logger, @@ -27,17 +54,27 @@ func NewExceptionService(logger *log.Logger, dbClient *gorm.DB, kafkaProducer pr } func (exceptionService *ExceptionService) CatchErrors(c *gin.Context) { - var errorsPayload []interface{} + scanner := bufio.NewScanner(c.Request.Body) + var lines []string - if err := c.BindJSON(&errorsPayload); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid JSON payload"}) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + var jsonData Payload + projectID := c.Param("projectId") + + err := json.Unmarshal([]byte(lines[2]), &jsonData) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Second line is not valid JSON"}) return } - headerMap := make(map[string]string) + for _, errorItem := range jsonData.Exception.Values { + errorItem.ProjectId = projectID + errorItem.ReleaseId = "release-1" + err := exceptionService.kafkaProducer.PublishEvent(errorItem, "kafka-stream", "", nil, encoder.JsonEncoderInstance) - for _, errorItem := range errorsPayload { - err := exceptionService.kafkaProducer.PublishEvent(errorItem, configs.GetKafkaConfig().GetTopic("js-error-topic"), uuid.NewString(), headerMap, encoder.JsonEncoderInstance) if err != nil { fmt.Println("Failed to push error to kafka") } diff --git a/service/ProjectCreator.go b/service/ProjectCreator.go index 5532e08..5c50edc 100644 --- a/service/ProjectCreator.go +++ b/service/ProjectCreator.go @@ -3,12 +3,15 @@ package service import ( "cybertron/internal/client/aws" "cybertron/models/db" + "cybertron/pkg/encoder" "cybertron/pkg/kafka/producer" "cybertron/pkg/log" "github.com/gin-gonic/gin" "github.com/google/uuid" "gorm.io/gorm" + "math/big" "net/http" + "strings" ) type ProjectCreator struct { @@ -40,10 +43,13 @@ func (pc *ProjectCreator) CreateProject(ctx *gin.Context) { }) return } + var uuidGenerated = uuid.New() + var i big.Int + i.SetString(strings.Replace(uuidGenerated.String(), "-", "", 4), 16) // Write to database result := pc.dbClient.Create(&db.Project{ - ProjectReferenceId: uuid.New(), + ProjectReferenceId: i.String(), Name: projectBody.Name, Team: projectBody.Team, }) @@ -79,10 +85,11 @@ func (pc *ProjectCreator) ProjectGet(c *gin.Context) { //generate Random numbers in go - //err := pc.kafkaProducer.PublishEvent(rand.Int(), "kafka-stream", "", nil, encoder.JsonEncoderInstance) - //if err != nil { - // pc.logger.Info(err.Error()) - //} + err := pc.kafkaProducer.PublishEvent(`{"hello" :"world"}`, "kafka-stream", "", nil, encoder.JsonEncoderInstance) + if err != nil { + pc.logger.Info(err.Error()) + } + pc.logger.Info("published kafka message") var projects []db.Project pc.dbClient.Find(&projects) diff --git a/service/ReleaseService.go b/service/ReleaseService.go index de63137..de0cec7 100644 --- a/service/ReleaseService.go +++ b/service/ReleaseService.go @@ -4,7 +4,6 @@ import ( "cybertron/models/db" "cybertron/pkg/log" "github.com/gin-gonic/gin" - "github.com/google/uuid" "gorm.io/gorm" "net/http" ) @@ -38,10 +37,8 @@ func (releaseService *ReleaseService) AddRelease(c *gin.Context) { } releaseToBeAdded := db.Release{ - ReleaseId: uuid.New(), ProjectReferenceId: releaseBody.ProjectId, ReleaseVersion: releaseBody.Version, - SourceMapUrl: releaseBody.SourceMapUrl, } if result := releaseService.dbClient.Create(&releaseToBeAdded); result.Error != nil { From c4de994f2e188d107c46af5a7528d98755f04d30 Mon Sep 17 00:00:00 2001 From: varnit-goyal_navi Date: Fri, 16 Aug 2024 13:47:08 +0530 Subject: [PATCH 2/3] TP-55555/source-map-flow --- configs/application.yml | 3 + .../database/{project.go => repositories.go} | 2 +- internal/dependencies/dependencies.go | 12 +- internal/transport/handler/sourceMap.go | 15 ++- internal/transport/router/sourceMap.go | 15 +-- internal/transport/server.go | 2 +- models/db/release.go | 9 ++ models/db/{SourceMap.go => sourceMap.go} | 3 +- service/ProjectCreator.go | 29 +--- service/sourceMap.go | 125 +++++++++++++----- 10 files changed, 135 insertions(+), 80 deletions(-) rename internal/database/{project.go => repositories.go} (90%) create mode 100644 models/db/release.go rename models/db/{SourceMap.go => sourceMap.go} (64%) diff --git a/configs/application.yml b/configs/application.yml index 2d752b0..0a70047 100644 --- a/configs/application.yml +++ b/configs/application.yml @@ -67,3 +67,6 @@ DocumentService: generate_token: DOCUMENT_SERVICE_MOCK_GENERATE_TOKEN +aws: + region: ap-south-1 + bucket: navi-cd955a63c4476df0f00c1cea0e4a40d1 diff --git a/internal/database/project.go b/internal/database/repositories.go similarity index 90% rename from internal/database/project.go rename to internal/database/repositories.go index de25b53..7c77486 100644 --- a/internal/database/project.go +++ b/internal/database/repositories.go @@ -16,6 +16,6 @@ func InitReleaseRepository(dbClient *gorm.DB) *gorm.DB { } func InitSourceMapRepository(dbClient *gorm.DB) *gorm.DB { - dbClient.AutoMigrate(&db.SourcMap{}) + dbClient.AutoMigrate(&db.SourceMap{}) return dbClient } diff --git a/internal/dependencies/dependencies.go b/internal/dependencies/dependencies.go index e600b30..59cd237 100644 --- a/internal/dependencies/dependencies.go +++ b/internal/dependencies/dependencies.go @@ -42,8 +42,9 @@ type Handler struct { } type Repositories struct { - ProjectRepository *gorm.DB - ReleaseRepository *gorm.DB + ProjectRepository *gorm.DB + ReleaseRepository *gorm.DB + SourceMapRepository *gorm.DB } func InitDependencies() *Dependencies { @@ -56,7 +57,7 @@ func InitDependencies() *Dependencies { documentServiceClient := document.NewDocumentServiceHttpClient(httpClient, logger, configs.GetDocumentServiceHttpClientConfigs()) projectServiceClient := service.NewProjectCreator(logger, dbClient, s3Client, kafkaProducer) - sourceMapServiceClient := service.NewSourceMapService(dbClient) + sourceMapServiceClient := service.NewSourceMapService(dbClient, s3Client, configs.GetAWSConfig()) releaseServiceClient := service.NewReleaseService(logger, dbClient) exceptionServiceClient := service.NewExceptionService(logger, dbClient, kafkaProducer) @@ -83,8 +84,9 @@ func initServices(documentService *document.HttpClient, projectService *service. func initRepositories(dbClient *gorm.DB) *Repositories { return &Repositories{ - ProjectRepository: database.InitProjectRepository(dbClient), - ReleaseRepository: database.InitReleaseRepository(dbClient), + ProjectRepository: database.InitProjectRepository(dbClient), + ReleaseRepository: database.InitReleaseRepository(dbClient), + SourceMapRepository: database.InitSourceMapRepository(dbClient), } } diff --git a/internal/transport/handler/sourceMap.go b/internal/transport/handler/sourceMap.go index a85e638..a080d76 100644 --- a/internal/transport/handler/sourceMap.go +++ b/internal/transport/handler/sourceMap.go @@ -2,8 +2,6 @@ package handler import ( "cybertron/service" - "net/http" - "github.com/gin-gonic/gin" ) @@ -17,11 +15,14 @@ func NewSourceMapHandler(sourceMapService *service.SourceMapService) *SourceMapH } } -func (h *SourceMapHandler) GetSourceMap(c *gin.Context) { - sourceMap := h.sourceMapService.GetSourceMap() - c.JSON(http.StatusOK, sourceMap) +func (h *SourceMapHandler) GetSourceMapUploadUrl(c *gin.Context) { + h.sourceMapService.GetSourceMapUploadUrl(c) } -func (h *SourceMapHandler) StoreSourceMap(c *gin.Context) { - h.sourceMapService.StoreSourceMap(c) +func (h *SourceMapHandler) SourceMapUploadAck(c *gin.Context) { + h.sourceMapService.SourceMapUploadAck(c) +} + +func (h *SourceMapHandler) ValidateSourceMap(c *gin.Context) { + h.sourceMapService.ValidateSourceMap(c) } diff --git a/internal/transport/router/sourceMap.go b/internal/transport/router/sourceMap.go index 5cb81a2..0c1942d 100644 --- a/internal/transport/router/sourceMap.go +++ b/internal/transport/router/sourceMap.go @@ -1,16 +1,15 @@ package router import ( + "cybertron/internal/dependencies" "cybertron/internal/transport/handler" - "cybertron/service" - "github.com/gin-gonic/gin" - "gorm.io/gorm" ) -func SourceMapRouter(r *gin.Engine, dbClient *gorm.DB) { - sourceMapService := service.NewSourceMapService(dbClient) - sourceMapHandler := handler.NewSourceMapHandler(sourceMapService) - r.GET("/uploadsourcemap", sourceMapHandler.GetSourceMap) - r.POST("/storesourcemap", sourceMapHandler.StoreSourceMap) +func SourceMapRouter(r *gin.Engine, dep *dependencies.Dependencies) { + sourceMapHandler := handler.NewSourceMapHandler(dep.Service.SourceMapService) + sourceMapGroup := r.Group("/api/v1") + sourceMapGroup.GET("/get-sourcemap-upload-url", sourceMapHandler.GetSourceMapUploadUrl) + sourceMapGroup.POST("/source-map-upload-ack", sourceMapHandler.SourceMapUploadAck) + sourceMapGroup.POST("/validate-source-map", sourceMapHandler.ValidateSourceMap) } diff --git a/internal/transport/server.go b/internal/transport/server.go index 4eb3006..6bb643f 100644 --- a/internal/transport/server.go +++ b/internal/transport/server.go @@ -31,7 +31,7 @@ func NewServer(dep *dependencies.Dependencies) *Server { func (s *Server) router() { router.ReadinessRouter(s.gin) router.ProjectRouter(s.gin, s.dependencies) - router.SourceMapRouter(s.gin, s.dependencies.DBClient) + router.SourceMapRouter(s.gin, s.dependencies) router.ReleasesRouter(s.gin, s.dependencies) router.ExceptionRouter(s.gin, s.dependencies) } diff --git a/models/db/release.go b/models/db/release.go new file mode 100644 index 0000000..43eab6d --- /dev/null +++ b/models/db/release.go @@ -0,0 +1,9 @@ +package db + +import "gorm.io/gorm" + +type Release struct { + gorm.Model + ProjectReferenceId string `gorm:"primaryKey"` + ReleaseVersion string `gorm:"column:name"` +} diff --git a/models/db/SourceMap.go b/models/db/sourceMap.go similarity index 64% rename from models/db/SourceMap.go rename to models/db/sourceMap.go index 977935e..631ffb1 100644 --- a/models/db/SourceMap.go +++ b/models/db/sourceMap.go @@ -6,5 +6,6 @@ type SourceMap struct { gorm.Model ReleaseReferenceId string `gorm:"primaryKey"` ProjectReferenceId string `gorm:"column:project_reference_id"` - SourceMapZipUrl string `gorm:"column:source_map_zip_url"` + FileName string `gorm:"column:file_name"` + State string `gorm:"column:state"` } diff --git a/service/ProjectCreator.go b/service/ProjectCreator.go index 5c50edc..1fd9bbd 100644 --- a/service/ProjectCreator.go +++ b/service/ProjectCreator.go @@ -3,7 +3,6 @@ package service import ( "cybertron/internal/client/aws" "cybertron/models/db" - "cybertron/pkg/encoder" "cybertron/pkg/kafka/producer" "cybertron/pkg/log" "github.com/gin-gonic/gin" @@ -44,6 +43,8 @@ func (pc *ProjectCreator) CreateProject(ctx *gin.Context) { return } var uuidGenerated = uuid.New() + //converting uuid to all numbers since sentry only supports + //all digit project uuid var i big.Int i.SetString(strings.Replace(uuidGenerated.String(), "-", "", 4), 16) @@ -65,33 +66,7 @@ func (pc *ProjectCreator) CreateProject(ctx *gin.Context) { } func (pc *ProjectCreator) ProjectGet(c *gin.Context) { - //s3 processing - //buckets, err := pc.s3Client.S3Client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) - //key := "async js.pdf" - //bucket := "navi-cd955a63c4476df0f00c1cea0e4a40d1" - //request, err := pc.s3Client.S3PresignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ - // Bucket: &bucket, - // Key: &key, - //}, func(opts *s3.PresignOptions) { - // opts.Expires = time.Duration(7 * 24 * time.Hour) - //}) - //pc.logger.Info(request.URL) - //if err != nil { - // log.Log.Fatal(err.Error()) - // pc.logger.Error("S3 List Buckets Failed") - //} - - //kafka producer testing - - //generate Random numbers in go - - err := pc.kafkaProducer.PublishEvent(`{"hello" :"world"}`, "kafka-stream", "", nil, encoder.JsonEncoderInstance) - if err != nil { - pc.logger.Info(err.Error()) - } - pc.logger.Info("published kafka message") var projects []db.Project - pc.dbClient.Find(&projects) if result := pc.dbClient.Find(&projects); result.Error != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": result.Error.Error()}) diff --git a/service/sourceMap.go b/service/sourceMap.go index 48a288b..7b7c13b 100644 --- a/service/sourceMap.go +++ b/service/sourceMap.go @@ -1,56 +1,121 @@ package service import ( + "context" + "cybertron/configs" + "cybertron/internal/client/aws" "cybertron/models/db" - "net/http" - "time" - + "fmt" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/gin-gonic/gin" "gorm.io/gorm" + "net/http" + "path" + "time" ) type SourceMapService struct { - dbClient *gorm.DB + dbClient *gorm.DB + s3Client *aws.Actions + awsConfig *configs.AwsConfig } -func NewSourceMapService(dbClient *gorm.DB) *SourceMapService { +type SourceMapAckBody struct { + ProjectId string `json:"project-id" binding:"required"` + ReleaseId string `json:"releaseId" binding:"required"` + FileName string `json:"file_name" binding:"required"` +} + +func NewSourceMapService(dbClient *gorm.DB, s3Client *aws.Actions, config *configs.AwsConfig) *SourceMapService { return &SourceMapService{ - dbClient: dbClient, + dbClient: dbClient, + s3Client: s3Client, + awsConfig: config, } } -func (s *SourceMapService) GetSourceMap() db.SourceMap { - //fetching SourceMap from a client API - sourceMap := db.SourceMap{ - Model: gorm.Model{ - ID: 1, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - DeletedAt: gorm.DeletedAt{}, - }, - ReleaseReferenceId: "some-release-ref-id", - ProjectReferenceId: "some-project-ref-id", - SourceMapZipUrl: "http://example.com/sourcemap.zip", - } +func (s *SourceMapService) GetSourceMapUploadUrl(ctx *gin.Context) { + projectId := ctx.Query("project_id") + releaseId := ctx.Query("release_id") + fileName := ctx.Query("file_name") - return sourceMap + if projectId == "" || releaseId == "" || fileName == "" { + ctx.JSON(http.StatusBadRequest, gin.H{ + "error": "Missing required query parameters: project_id, release_id, and file_name are required.", + }) + return + } + //generate s3 pre-signed url + key := path.Join(projectId, releaseId, fileName) + bucket := s.awsConfig.Bucket + request, err := s.s3Client.S3PresignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }, func(opts *s3.PresignOptions) { + opts.Expires = time.Duration(7 * 24 * time.Hour) + }) + if err != nil { + fmt.Println(err) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": "unable to create S3 object"}) + return + } + // save state in database + result := *s.dbClient.Create(&db.SourceMap{ + ProjectReferenceId: projectId, + ReleaseReferenceId: releaseId, + FileName: fileName, + State: "IN_PROGRESS", + }) + if result.Error != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": result.Error.Error(), "message": "Failed to create project"}) + return + } + ctx.JSON(http.StatusOK, gin.H{ + "url": request.URL, + }) + + return } -func (s *SourceMapService) StoreSourceMap(c *gin.Context) { - var sourceMap db.SourceMap - if err := c.ShouldBindJSON(&sourceMap); err != nil { +func (s *SourceMapService) SourceMapUploadAck(c *gin.Context) { + var sourceMapAckBody SourceMapAckBody + if err := c.BindJSON(&sourceMapAckBody); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } + //find the record to update ack + var existingSourceMap db.SourceMap + existingRecordError := s.dbClient.First(&existingSourceMap, "project_reference_id = ? and release_reference_id= ? and file_name = ?", sourceMapAckBody.ProjectId, sourceMapAckBody.ReleaseId, sourceMapAckBody.FileName).Error + if existingRecordError != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": existingRecordError.Error()}) + return + } - result := s.dbClient.Create(&db.SourceMap{ - ReleaseReferenceId: sourceMap.ReleaseReferenceId, - ProjectReferenceId: sourceMap.ProjectReferenceId, - SourceMapZipUrl: sourceMap.SourceMapZipUrl, - }) + existingSourceMap.State = "DONE" - if result.Error != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to store source map"}) + err := s.dbClient.Save(&existingSourceMap).Error + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } c.JSON(http.StatusOK, gin.H{"status": "Source map stored successfully"}) } + +func (s *SourceMapService) ValidateSourceMap(ctx *gin.Context) { + projectId := ctx.Query("project_id") + releaseId := ctx.Query("release_id") + fileName := ctx.Query("file_name") + if projectId == "" || releaseId == "" || fileName == "" { + ctx.JSON(http.StatusBadRequest, gin.H{ + "error": "Missing required query parameters: project_id, release_id, and file_name are required.", + }) + } + var existingSourceMap db.SourceMap + s.dbClient.First(&existingSourceMap, "ProjectReferenceId = ? and ReleaseReferenceId= ? and file_name = ?", projectId, releaseId, fileName) + if existingSourceMap.State == "DONE" { + ctx.JSON(http.StatusOK, gin.H{"status": "Source map stored successfully"}) + + } + ctx.JSON(http.StatusNotFound, gin.H{"error": "Source map not found"}) + +} From add56b9d2cc9507b4a8a3f7e6ac2fcfd3476db95 Mon Sep 17 00:00:00 2001 From: varnit-goyal_navi Date: Fri, 16 Aug 2024 13:48:08 +0530 Subject: [PATCH 3/3] TP-55555/source-map-flow --- configs/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/application.yml b/configs/application.yml index 0a70047..0b505f0 100644 --- a/configs/application.yml +++ b/configs/application.yml @@ -42,7 +42,7 @@ http: # Kafka config kafka: - password: kDia1uC.GI;)Al5eQ)+Q + password: xxxa username: varnitgoyal/varnitgoyal95@gmail.com/ocid1.streampool.oc1.ap-mumbai-1.amaaaaaaotdslraanepwp54txqqxkmg4l6dghrhufiezqkx2lqhndgxoq7pa brokers: cell-1.streaming.ap-mumbai-1.oci.oraclecloud.com:9092 group: