varnitgoyal95/cybertron
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #12 from navi-ppl/TP-55555/error-log-consumer

Browse Source 
Tp 55555/error log consumer
This commit is contained in:
Podili Varshitha
2024-08-16 13:57:04 +05:30
committed by GitHub
parent 220d8e01bd add56b9d2c
commit f89f1f331d
14 changed files with 190 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
configs/application.yml
View File

@@ -42,7 +42,7 @@ http:
# Kafka config
kafka:
password: XX
password: xxxa
username: varnitgoyal/varnitgoyal95@gmail.com/ocid1.streampool.oc1.ap-mumbai-1.amaaaaaaotdslraanepwp54txqqxkmg4l6dghrhufiezqkx2lqhndgxoq7pa
brokers: cell-1.streaming.ap-mumbai-1.oci.oraclecloud.com:9092
group:
@@ -67,3 +67,6 @@ DocumentService:
generate_token: DOCUMENT_SERVICE_MOCK_GENERATE_TOKEN
aws:
region: ap-south-1
bucket: navi-cd955a63c4476df0f00c1cea0e4a40d1

2
internal/database/project.go → internal/database/repositories.go
View File

@@ -16,6 +16,6 @@ func InitReleaseRepository(dbClient *gorm.DB) *gorm.DB {
}
func InitSourceMapRepository(dbClient *gorm.DB) *gorm.DB {
dbClient.AutoMigrate(&db.SourcMap{})
dbClient.AutoMigrate(&db.SourceMap{})
return dbClient
}

4
internal/dependencies/dependencies.go
View File

@@ -44,6 +44,7 @@ type Handler struct {
type Repositories struct {
ProjectRepository *gorm.DB
ReleaseRepository *gorm.DB
SourceMapRepository *gorm.DB
}
func InitDependencies() *Dependencies {
@@ -56,7 +57,7 @@ func InitDependencies() *Dependencies {
documentServiceClient := document.NewDocumentServiceHttpClient(httpClient, logger, configs.GetDocumentServiceHttpClientConfigs())
projectServiceClient := service.NewProjectCreator(logger, dbClient, s3Client, kafkaProducer)
sourceMapServiceClient := service.NewSourceMapService(dbClient)
sourceMapServiceClient := service.NewSourceMapService(dbClient, s3Client, configs.GetAWSConfig())
releaseServiceClient := service.NewReleaseService(logger, dbClient)
exceptionServiceClient := service.NewExceptionService(logger, dbClient, kafkaProducer)
@@ -85,6 +86,7 @@ func initRepositories(dbClient *gorm.DB) *Repositories {
return &Repositories{
ProjectRepository: database.InitProjectRepository(dbClient),
ReleaseRepository: database.InitReleaseRepository(dbClient),
SourceMapRepository: database.InitSourceMapRepository(dbClient),
}
}

15
internal/transport/handler/sourceMap.go
View File

@@ -2,8 +2,6 @@ package handler
import (
"cybertron/service"
"net/http"
"github.com/gin-gonic/gin"
)
@@ -17,11 +15,14 @@ func NewSourceMapHandler(sourceMapService *service.SourceMapService) *SourceMapH
}
}
func (h *SourceMapHandler) GetSourceMap(c *gin.Context) {
sourceMap := h.sourceMapService.GetSourceMap()
c.JSON(http.StatusOK, sourceMap)
func (h *SourceMapHandler) GetSourceMapUploadUrl(c *gin.Context) {
h.sourceMapService.GetSourceMapUploadUrl(c)
}
func (h *SourceMapHandler) StoreSourceMap(c *gin.Context) {
h.sourceMapService.StoreSourceMap(c)
func (h *SourceMapHandler) SourceMapUploadAck(c *gin.Context) {
h.sourceMapService.SourceMapUploadAck(c)
}
func (h *SourceMapHandler) ValidateSourceMap(c *gin.Context) {
h.sourceMapService.ValidateSourceMap(c)
}

4
internal/transport/router/exception.go
View File

@@ -8,8 +8,8 @@ import (
func ExceptionRouter(r *gin.Engine, dep *dependencies.Dependencies) {
exceptionHandler := handler.NewExceptionHandler(dep.Service.ExceptionService)
exceptionRouterGroup := r.Group("/api/v1")
exceptionRouterGroup := r.Group("/api")
{
exceptionRouterGroup.POST("/catch-errors", exceptionHandler.CatchErrors)
exceptionRouterGroup.POST("/:projectId/envelope", exceptionHandler.CatchErrors)
}
}

15
internal/transport/router/sourceMap.go
View File

@@ -1,16 +1,15 @@
package router
import (
"cybertron/internal/dependencies"
"cybertron/internal/transport/handler"
"cybertron/service"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
func SourceMapRouter(r *gin.Engine, dbClient *gorm.DB) {
sourceMapService := service.NewSourceMapService(dbClient)
sourceMapHandler := handler.NewSourceMapHandler(sourceMapService)
r.GET("/uploadsourcemap", sourceMapHandler.GetSourceMap)
r.POST("/storesourcemap", sourceMapHandler.StoreSourceMap)
func SourceMapRouter(r *gin.Engine, dep *dependencies.Dependencies) {
sourceMapHandler := handler.NewSourceMapHandler(dep.Service.SourceMapService)
sourceMapGroup := r.Group("/api/v1")
sourceMapGroup.GET("/get-sourcemap-upload-url", sourceMapHandler.GetSourceMapUploadUrl)
sourceMapGroup.POST("/source-map-upload-ack", sourceMapHandler.SourceMapUploadAck)
sourceMapGroup.POST("/validate-source-map", sourceMapHandler.ValidateSourceMap)
}

2
internal/transport/server.go
View File

@@ -31,7 +31,7 @@ func NewServer(dep *dependencies.Dependencies) *Server {
func (s *Server) router() {
router.ReadinessRouter(s.gin)
router.ProjectRouter(s.gin, s.dependencies)
router.SourceMapRouter(s.gin, s.dependencies.DBClient)
router.SourceMapRouter(s.gin, s.dependencies)
router.ReleasesRouter(s.gin, s.dependencies)
router.ExceptionRouter(s.gin, s.dependencies)
}

3
models/db/project.go
View File

@@ -1,7 +1,6 @@
package db
import (
"github.com/google/uuid"
"time"
)
@@ -10,7 +9,7 @@ type Project struct {
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt time.Time `json:"deletedAt"`
ProjectReferenceId uuid.UUID `json:"projectReferenceId" gorm:"primaryKey"`
ProjectReferenceId string `json:"projectReferenceId" gorm:"primaryKey"`
Name string `json:"name" gorm:"unique"`
Team string `json:"team"`
}

9
models/db/release.go Normal file
View File

@@ -0,0 +1,9 @@
package db
import "gorm.io/gorm"
type Release struct {
gorm.Model
ProjectReferenceId string `gorm:"primaryKey"`
ReleaseVersion string `gorm:"column:name"`
}

3
models/db/SourceMap.go → models/db/sourceMap.go
View File

@@ -6,5 +6,6 @@ type SourceMap struct {
gorm.Model
ReleaseReferenceId string `gorm:"primaryKey"`
ProjectReferenceId string `gorm:"column:project_reference_id"`
SourceMapZipUrl string `gorm:"column:source_map_zip_url"`
FileName string `gorm:"column:file_name"`
State string `gorm:"column:state"`
}

53
service/ExceptionService.go
View File

@@ -1,13 +1,13 @@
package service
import (
"cybertron/configs"
"bufio"
"cybertron/pkg/encoder"
"cybertron/pkg/kafka/producer"
"cybertron/pkg/log"
"encoding/json"
"fmt"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"gorm.io/gorm"
"net/http"
)
@@ -18,6 +18,33 @@ type ExceptionService struct {
kafkaProducer producer.KProducer
}
type Frame struct {
Filename string `json:"filename"`
Function string `json:"function"`
InApp bool `json:"in_app"`
Lineno int `json:"lineno,omitempty"`
Colno int `json:"colno,omitempty"`
}
type Stacktrace struct {
Frames []Frame `json:"frames"`
}
type ExceptionValue struct {
Type string `json:"type"`
Value string `json:"value"`
Stacktrace Stacktrace `json:"stacktrace"`
ProjectId string `json:"project_id,omitempty"`
ReleaseId string `json:"release_id,omitempty"`
}
type Exception struct {
Values []ExceptionValue `json:"values"`
}
type Payload struct {
Exception Exception `json:"exception"`
}
func NewExceptionService(logger *log.Logger, dbClient *gorm.DB, kafkaProducer producer.KProducer) *ExceptionService {
return &ExceptionService{
logger: logger,
@@ -27,17 +54,27 @@ func NewExceptionService(logger *log.Logger, dbClient *gorm.DB, kafkaProducer pr
}
func (exceptionService *ExceptionService) CatchErrors(c *gin.Context) {
var errorsPayload []interface{}
scanner := bufio.NewScanner(c.Request.Body)
var lines []string
if err := c.BindJSON(&errorsPayload); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid JSON payload"})
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
var jsonData Payload
projectID := c.Param("projectId")
err := json.Unmarshal([]byte(lines[2]), &jsonData)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Second line is not valid JSON"})
return
}
headerMap := make(map[string]string)
for _, errorItem := range jsonData.Exception.Values {
errorItem.ProjectId = projectID
errorItem.ReleaseId = "release-1"
err := exceptionService.kafkaProducer.PublishEvent(errorItem, "kafka-stream", "", nil, encoder.JsonEncoderInstance)
for _, errorItem := range errorsPayload {
err := exceptionService.kafkaProducer.PublishEvent(errorItem, configs.GetKafkaConfig().GetTopic("js-error-topic"), uuid.NewString(), headerMap, encoder.JsonEncoderInstance)
if err != nil {
fmt.Println("Failed to push error to kafka")
}

34
service/ProjectCreator.go
View File

@@ -8,7 +8,9 @@ import (
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"gorm.io/gorm"
"math/big"
"net/http"
"strings"
)
type ProjectCreator struct {
@@ -40,10 +42,15 @@ func (pc *ProjectCreator) CreateProject(ctx *gin.Context) {
})
return
}
var uuidGenerated = uuid.New()
//converting uuid to all numbers since sentry only supports
//all digit project uuid
var i big.Int
i.SetString(strings.Replace(uuidGenerated.String(), "-", "", 4), 16)
// Write to database
result := pc.dbClient.Create(&db.Project{
ProjectReferenceId: uuid.New(),
ProjectReferenceId: i.String(),
Name: projectBody.Name,
Team: projectBody.Team,
})
@@ -59,32 +66,7 @@ func (pc *ProjectCreator) CreateProject(ctx *gin.Context) {
}
func (pc *ProjectCreator) ProjectGet(c *gin.Context) {
//s3 processing
//buckets, err := pc.s3Client.S3Client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
//key := "async js.pdf"
//bucket := "navi-cd955a63c4476df0f00c1cea0e4a40d1"
//request, err := pc.s3Client.S3PresignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{
// Bucket: &bucket,
// Key: &key,
//}, func(opts *s3.PresignOptions) {
// opts.Expires = time.Duration(7 * 24 * time.Hour)
//})
//pc.logger.Info(request.URL)
//if err != nil {
// log.Log.Fatal(err.Error())
// pc.logger.Error("S3 List Buckets Failed")
//}
//kafka producer testing
//generate Random numbers in go
//err := pc.kafkaProducer.PublishEvent(rand.Int(), "kafka-stream", "", nil, encoder.JsonEncoderInstance)
//if err != nil {
// pc.logger.Info(err.Error())
//}
var projects []db.Project
pc.dbClient.Find(&projects)
if result := pc.dbClient.Find(&projects); result.Error != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": result.Error.Error()})

3
service/ReleaseService.go
View File

@@ -4,7 +4,6 @@ import (
"cybertron/models/db"
"cybertron/pkg/log"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"gorm.io/gorm"
"net/http"
)
@@ -38,10 +37,8 @@ func (releaseService *ReleaseService) AddRelease(c *gin.Context) {
}
releaseToBeAdded := db.Release{
ReleaseId: uuid.New(),
ProjectReferenceId: releaseBody.ProjectId,
ReleaseVersion: releaseBody.Version,
SourceMapUrl: releaseBody.SourceMapUrl,
}
if result := releaseService.dbClient.Create(&releaseToBeAdded); result.Error != nil {

123
service/sourceMap.go
View File

@@ -1,56 +1,121 @@
package service
import (
"context"
"cybertron/configs"
"cybertron/internal/client/aws"
"cybertron/models/db"
"net/http"
"time"
"fmt"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"net/http"
"path"
"time"
)
type SourceMapService struct {
dbClient *gorm.DB
s3Client *aws.Actions
awsConfig *configs.AwsConfig
}
func NewSourceMapService(dbClient *gorm.DB) *SourceMapService {
type SourceMapAckBody struct {
ProjectId string `json:"project-id" binding:"required"`
ReleaseId string `json:"releaseId" binding:"required"`
FileName string `json:"file_name" binding:"required"`
}
func NewSourceMapService(dbClient *gorm.DB, s3Client *aws.Actions, config *configs.AwsConfig) *SourceMapService {
return &SourceMapService{
dbClient: dbClient,
s3Client: s3Client,
awsConfig: config,
}
}
func (s *SourceMapService) GetSourceMap() db.SourceMap {
//fetching SourceMap from a client API
sourceMap := db.SourceMap{
Model: gorm.Model{
ID: 1,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
DeletedAt: gorm.DeletedAt{},
},
ReleaseReferenceId: "some-release-ref-id",
ProjectReferenceId: "some-project-ref-id",
SourceMapZipUrl: "http://example.com/sourcemap.zip",
}
func (s *SourceMapService) GetSourceMapUploadUrl(ctx *gin.Context) {
projectId := ctx.Query("project_id")
releaseId := ctx.Query("release_id")
fileName := ctx.Query("file_name")
return sourceMap
if projectId == "" || releaseId == "" || fileName == "" {
ctx.JSON(http.StatusBadRequest, gin.H{
"error": "Missing required query parameters: project_id, release_id, and file_name are required.",
})
return
}
//generate s3 pre-signed url
key := path.Join(projectId, releaseId, fileName)
bucket := s.awsConfig.Bucket
request, err := s.s3Client.S3PresignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
}, func(opts *s3.PresignOptions) {
opts.Expires = time.Duration(7 * 24 * time.Hour)
})
if err != nil {
fmt.Println(err)
ctx.JSON(http.StatusInternalServerError, gin.H{"error": "unable to create S3 object"})
return
}
// save state in database
result := *s.dbClient.Create(&db.SourceMap{
ProjectReferenceId: projectId,
ReleaseReferenceId: releaseId,
FileName: fileName,
State: "IN_PROGRESS",
})
if result.Error != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": result.Error.Error(), "message": "Failed to create project"})
return
}
ctx.JSON(http.StatusOK, gin.H{
"url": request.URL,
})
func (s *SourceMapService) StoreSourceMap(c *gin.Context) {
var sourceMap db.SourceMap
if err := c.ShouldBindJSON(&sourceMap); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
result := s.dbClient.Create(&db.SourceMap{
ReleaseReferenceId: sourceMap.ReleaseReferenceId,
ProjectReferenceId: sourceMap.ProjectReferenceId,
SourceMapZipUrl: sourceMap.SourceMapZipUrl,
})
func (s *SourceMapService) SourceMapUploadAck(c *gin.Context) {
var sourceMapAckBody SourceMapAckBody
if err := c.BindJSON(&sourceMapAckBody); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
//find the record to update ack
var existingSourceMap db.SourceMap
existingRecordError := s.dbClient.First(&existingSourceMap, "project_reference_id = ? and release_reference_id= ? and file_name = ?", sourceMapAckBody.ProjectId, sourceMapAckBody.ReleaseId, sourceMapAckBody.FileName).Error
if existingRecordError != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": existingRecordError.Error()})
return
}
if result.Error != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to store source map"})
existingSourceMap.State = "DONE"
err := s.dbClient.Save(&existingSourceMap).Error
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"status": "Source map stored successfully"})
}
func (s *SourceMapService) ValidateSourceMap(ctx *gin.Context) {
projectId := ctx.Query("project_id")
releaseId := ctx.Query("release_id")
fileName := ctx.Query("file_name")
if projectId == "" || releaseId == "" || fileName == "" {
ctx.JSON(http.StatusBadRequest, gin.H{
"error": "Missing required query parameters: project_id, release_id, and file_name are required.",
})
}
var existingSourceMap db.SourceMap
s.dbClient.First(&existingSourceMap, "ProjectReferenceId = ? and ReleaseReferenceId= ? and file_name = ?", projectId, releaseId, fileName)
if existingSourceMap.State == "DONE" {
ctx.JSON(http.StatusOK, gin.H{"status": "Source map stored successfully"})
}
ctx.JSON(http.StatusNotFound, gin.H{"error": "Source map not found"})
}