varnitgoyal95/cybertron
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TP-5555 | clean up job

Browse Source
This commit is contained in:
varnit goyal
2024-12-10 13:25:50 +05:30
parent eda34db8fc 4d297ed526
commit d27c023186
6 changed files with 108 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/transport/handler/search.go
View File

@@ -10,7 +10,7 @@ type SearchHandler struct {
}
func (h *SearchHandler) SearchErrors(c *gin.Context) {
h.searchService.GetErrorDetails(c)
h.searchService.GetSearchResults(c)
}
func (h *SearchHandler) GetErrorDetails(c *gin.Context) {
h.searchService.GetErrorDetails(c)
@@ -20,6 +20,10 @@ func (h *SearchHandler) GetErrorList(c *gin.Context) {
h.searchService.GetErrorList(c)
}
func (h *SearchHandler) GetErrorListV2(c *gin.Context) {
h.searchService.GetErrorListV2(c)
}
func NewSearchHandler(s *service.SearchService) *SearchHandler {
return &SearchHandler{
searchService: s,

3
internal/transport/router/search.go
View File

@@ -10,8 +10,9 @@ func SearchRouter(r *gin.Engine, dep *dependencies.Dependencies) {
searchHandler := handler.NewSearchHandler(dep.Service.SearchService)
searchRouterGroup := r.Group("/api/v1")
{
searchRouterGroup.GET("/errors-list", searchHandler.GetErrorList)
searchRouterGroup.GET("/errors-list", searchHandler.GetErrorListV2)
searchRouterGroup.GET("/error-detail", searchHandler.GetErrorDetails)
searchRouterGroup.GET("/error-search", searchHandler.SearchErrors)
}
}

24
pkg/utils/elastic_query_util.go
View File

@@ -19,15 +19,16 @@ const (
SortQuery = `"sort": [ { "%s": { "order": "%s" } } ]`
CollapseQuery = `"collapse": { "field": "%s" }`
MatchAllQuery = `{ "match_all": {} }`
MultiMatchQuery = `{"multi_match": { "query": "%s", "fields": [%s] }}`
FromQuery = `"from": %d`
SizeQuery = `"size": %d`
SearchQuery = `"query": %s`
FieldsQuery = `"fields": [ "%s" ]`
EsQuery = "{ %s }"
sourceQuery = `"_source": %t`
sourceQuery = `"_source": [%s]`
AggregationQuery = `"aggs": { %s }`
AggregationQueryFormat = `"%s": { %s }` // aggregation name, aggregation query
TermsAggregationQuery = `"terms": { "field": "%s", "size": %d }`
TermsAggregationQuery = `"terms": { "field": "%s", "size": %d, %s}`
MinAggregationQuery = `"min": { "field": "%s" }`
MaxAggregationQuery = `"max": { "field": "%s" }`
CardinalityAggregationQuery = `"cardinality": { "field": "%s" }`
@@ -46,6 +47,7 @@ const (
SearchAfter = `"search_after": [%s]`
CompositeAggsQuery = `"composite": {"size" : %d, "sources" : [{"%s": {%s}}] }`
CompositeAggsQueryWithAfterKey = `"composite": {"size" : %d, "after": {"%s": "%s"}, "sources" : [{"%s": {%s}}] }`
OrderQuery = `"order": {"%s": "%s"}`
TopHitsAggsQuery = `"top_hits":{"size" : %d, "_source": {"includes" : [ %s ]}}`
ValueCountAggsQuery = `"value_count" : {"field": "%s"}`
@@ -124,6 +126,11 @@ func CreateMatchQuery(key string, value string) string {
return fmt.Sprintf(MatchQuery, key, value)
}
func CreateMultiMatchQuery(query string, fields ...string) string {
// Join fields into a JSON array format: ["field1", "field2", "field3"]
formattedFields := `"` + strings.Join(fields, `", "`) + `"`
return fmt.Sprintf(MultiMatchQuery, query, formattedFields)
}
func CreateShouldQuery(filters ...string) string {
return fmt.Sprintf(ShouldQuery, strings.Join(filters, ","))
}
@@ -167,8 +174,9 @@ func CreateEsQuery(query ...string) string {
return fmt.Sprintf(EsQuery, strings.Join(query, ","))
}
func CreateSourceQuery(source bool) string {
return fmt.Sprintf(sourceQuery, source)
func CreateSourceQuery(fields ...string) string {
formattedFields := `"` + strings.Join(fields, `", "`) + `"`
return fmt.Sprintf(sourceQuery, formattedFields)
}
func CreateFilterAggregationQuery(value ...string) string {
@@ -187,8 +195,8 @@ func CreateMaxAggregationQuery(field string) string {
return fmt.Sprintf(MaxAggregationQuery, field)
}
func CreateTermsAggregationQuery(field string, size int) string {
return fmt.Sprintf(TermsAggregationQuery, field, size)
func CreateTermsAggregationQuery(field string, size int, orderQuery string) string {
return fmt.Sprintf(TermsAggregationQuery, field, size, orderQuery)
}
func CreateTermsAggregationQueryWithoutSize(field string) string {
@@ -266,3 +274,7 @@ func CreateTopHitsAggsQuery(size int, fields []string) string {
func CreateValueCountAggsQuery(field string) string {
return fmt.Sprintf(ValueCountAggsQuery, field)
}
func CreateOrderQuery(field string, order string) string {
return fmt.Sprintf(OrderQuery, field, order)
}

10
service/ProjectCreator.go
View File

@@ -29,12 +29,14 @@ type ProjectBody struct {
Team string `json:"team" binding:"required"`
Icon string `json:"icon"`
GithubUrl string `json:"githubUrl" binding:"required"`
Account string `json:"account" binding:"required"`
}
type UpdateProjectBody struct {
Icon string `json:"logoUrl" binding:"required"`
GithubUrl string `json:"githubUrl" binding:"required"`
IgnorePatterns []string `json:"ignorePatterns" binding:"required"`
IgnorePatterns []string `json:"ignorePatterns" binding:"omitempty"`
Account string `json:"account" binding:"required"`
}
func NewProjectCreator(logger *log.Logger, dbClient *gorm.DB, s3Client *aws.Actions, kafkaProducer producer.KProducer) *ProjectCreator {
@@ -75,6 +77,7 @@ func (pc *ProjectCreator) CreateProject(ctx *gin.Context) {
Icon: projectBody.Icon,
GithubUrl: projectBody.GithubUrl,
Secret: hex.EncodeToString(secret),
Account: projectBody.Account,
})
if result.Error != nil {
@@ -122,7 +125,7 @@ func (pc *ProjectCreator) ProjectUpdate(c *gin.Context) {
var updateProjectBody UpdateProjectBody
// Find the project by ID
if result := pc.dbClient.Where("id = ?", projectId).First(&project); result.Error != nil {
if result := pc.dbClient.Where("project_reference_id = ?", projectId).First(&project); result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "Project not found"})
} else {
@@ -138,10 +141,11 @@ func (pc *ProjectCreator) ProjectUpdate(c *gin.Context) {
}
// Update the project in the database
if result := pc.dbClient.Where("id = ?", projectId).Updates(&db.Project{
if result := pc.dbClient.Where("project_reference_id = ?", projectId).Updates(&db.Project{
GithubUrl: updateProjectBody.GithubUrl,
Icon: updateProjectBody.Icon,
IgnorePatterns: updateProjectBody.IgnorePatterns,
Account: updateProjectBody.Account,
}); result.Error != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": result.Error.Error()})
return

76
service/searchService.go
View File

@@ -62,6 +62,80 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) {
})
}
func (s *SearchService) GetSearchResults(c *gin.Context) {
projectId := c.Query("project_id")
size := c.DefaultQuery("size", "10")
search_key := c.DefaultQuery("search_key", "")
sizeInNumber, sizeParseError := strconv.Atoi(size)
if sizeParseError != nil {
c.JSON(http.StatusBadRequest, "size should be a number")
return
}
size_query := utils.CreateSizeQuery(int64(sizeInNumber))
term_query := utils.CreateTermSubQuery("project_id", projectId)
multiMatchQuery := utils.CreateMultiMatchQuery(search_key, "error", "title", "extra.metadata")
should_query := utils.CreateMustQuery(term_query, multiMatchQuery)
boolQuery := utils.CreateBoolQuery(should_query)
search_query := utils.CreateSearchQuery(boolQuery)
source_query := utils.CreateSourceQuery("error")
finalQuery := utils.CreateEsQuery(source_query, search_query, size_query)
fields := []string{"error", "significant_stack", "title"}
var res, _, total, err = s.elasticSearchClient.SearchDocuments(finalQuery, fields)
if err != nil {
utils.ErrorResponse(c, "Failed to search please try again later")
return
}
//println("final query %s", finalQuery)
c.JSON(http.StatusOK, gin.H{
"results": res,
"total": total,
})
}
func (s *SearchService) GetErrorListV2(c *gin.Context) {
projectId := c.Query("project_id")
sortKey := c.DefaultQuery("sort_key", "lastSeen")
var orderQuery = ""
if sortKey == "lastSeen" {
orderQuery = utils.CreateOrderQuery("last_seen", "desc")
}
if sortKey == "firstSeen" {
orderQuery = utils.CreateOrderQuery("first_seen", "desc")
}
if sortKey == "count" {
orderQuery = utils.CreateOrderQuery("_count", "desc")
}
//size := c.DefaultQuery("size", "100000")
term_query := utils.CreateTermSubQuery("project_id", projectId)
search_query := utils.CreateSearchQuery(term_query)
size_query := utils.CreateSizeQuery(0)
top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery(1, []string{"error", "significant_stack", "created_at", "error_hash"}))
last_seen_aggs_query := utils.BuildAggregationQuery("last_seen", utils.CreateMaxAggregationQuery("created_at"))
first_seen_aggs_query := utils.BuildAggregationQuery("first_seen", utils.CreateMinAggregationQuery("created_at"))
top_hits_aggs_query := utils.CreateAggregationQuery(top_hits_aggs_name_query, last_seen_aggs_query, first_seen_aggs_query)
term_aggs_query := utils.CreateTermsAggregationQuery("error_hash", 100000, orderQuery)
terms_aggs_query_with_name := utils.BuildAggregationQuery("errors_by_hash", term_aggs_query, top_hits_aggs_query)
final_terms_aggs_query := utils.CreateAggregationQuery(terms_aggs_query_with_name)
final_query := utils.CreateEsQuery(search_query, size_query, final_terms_aggs_query)
println("final query %v", final_query)
fields := []string{"error", "significant_stack", "title"}
var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields)
if err != nil {
utils.ErrorResponse(c, "search failed please try again")
return
}
c.JSON(http.StatusOK, gin.H{
"results": aggs,
"total": total,
})
}
func (s *SearchService) GetErrorList(c *gin.Context) {
//todo pagination and aggregation of errors
projectId := c.Query("project_id")
@@ -105,8 +179,8 @@ func (s *SearchService) GetErrorList(c *gin.Context) {
compositeAggsQuery := utils.CreateAggregationQuery(composite_aggs_query)
final_query := utils.CreateEsQuery(search_query, size_query, compositeAggsQuery)
//s.logger.Info("%s", zap.String("final query", final_query))
println("%s", final_query)
fields := []string{"error", "significant_stack", "title"}
var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields)

2
service/sourceMap.go
View File

@@ -44,7 +44,7 @@ func (s *SourceMapService) GetSourceMapUploadUrl(ctx *gin.Context) {
account := ctx.DefaultQuery("account", "ppl")
appendedAccount := "-" + account
if appendedAccount == "ppl" {
if account == "ppl" {
appendedAccount = ""
}