varnitgoyal95/cybertron
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TP-5555 | error list v2

Browse Source
This commit is contained in:
varnit goyal
2024-12-10 11:17:46 +05:30
parent 17a707633e
commit 5c8dd99403
5 changed files with 102 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/transport/handler/search.go
View File

@@ -10,7 +10,7 @@ type SearchHandler struct {
}
func (h *SearchHandler) SearchErrors(c *gin.Context) {
h.searchService.GetErrorDetails(c)
h.searchService.GetSearchResults(c)
}
func (h *SearchHandler) GetErrorDetails(c *gin.Context) {
h.searchService.GetErrorDetails(c)
@@ -20,6 +20,10 @@ func (h *SearchHandler) GetErrorList(c *gin.Context) {
h.searchService.GetErrorList(c)
}
func (h *SearchHandler) GetErrorListV2(c *gin.Context) {
h.searchService.GetErrorListV2(c)
}
func NewSearchHandler(s *service.SearchService) *SearchHandler {
return &SearchHandler{
searchService: s,

3
internal/transport/router/search.go
View File

@@ -10,8 +10,9 @@ func SearchRouter(r *gin.Engine, dep *dependencies.Dependencies) {
searchHandler := handler.NewSearchHandler(dep.Service.SearchService)
searchRouterGroup := r.Group("/api/v1")
{
searchRouterGroup.GET("/errors-list", searchHandler.GetErrorList)
searchRouterGroup.GET("/errors-list", searchHandler.GetErrorListV2)
searchRouterGroup.GET("/error-detail", searchHandler.GetErrorDetails)
searchRouterGroup.GET("/error-search", searchHandler.SearchErrors)
}
}

24
pkg/utils/elastic_query_util.go
View File

@@ -18,15 +18,16 @@ const (
SortQuery = `"sort": [ { "%s": { "order": "%s" } } ]`
CollapseQuery = `"collapse": { "field": "%s" }`
MatchAllQuery = `{ "match_all": {} }`
MultiMatchQuery = `{"multi_match": { "query": "%s", "fields": [%s] }}`
FromQuery = `"from": %d`
SizeQuery = `"size": %d`
SearchQuery = `"query": %s`
FieldsQuery = `"fields": [ "%s" ]`
EsQuery = "{ %s }"
sourceQuery = `"_source": %t`
sourceQuery = `"_source": [%s]`
AggregationQuery = `"aggs": { %s }`
AggregationQueryFormat = `"%s": { %s }` // aggregation name, aggregation query
TermsAggregationQuery = `"terms": { "field": "%s", "size": %d }`
TermsAggregationQuery = `"terms": { "field": "%s", "size": %d, %s}`
MinAggregationQuery = `"min": { "field": "%s" }`
MaxAggregationQuery = `"max": { "field": "%s" }`
CardinalityAggregationQuery = `"cardinality": { "field": "%s" }`
@@ -45,6 +46,7 @@ const (
SearchAfter = `"search_after": [%s]`
CompositeAggsQuery = `"composite": {"size" : %d, "sources" : [{"%s": {%s}}] }`
CompositeAggsQueryWithAfterKey = `"composite": {"size" : %d, "after": {"%s": "%s"}, "sources" : [{"%s": {%s}}] }`
OrderQuery = `"order": {"%s": "%s"}`
TopHitsAggsQuery = `"top_hits":{"size" : %d, "_source": {"includes" : [ %s ]}}`
ValueCountAggsQuery = `"value_count" : {"field": "%s"}`
@@ -119,6 +121,11 @@ func CreateMatchQuery(key string, value string) string {
return fmt.Sprintf(MatchQuery, key, value)
}
func CreateMultiMatchQuery(query string, fields ...string) string {
// Join fields into a JSON array format: ["field1", "field2", "field3"]
formattedFields := `"` + strings.Join(fields, `", "`) + `"`
return fmt.Sprintf(MultiMatchQuery, query, formattedFields)
}
func CreateShouldQuery(filters ...string) string {
return fmt.Sprintf(ShouldQuery, strings.Join(filters, ","))
}
@@ -162,8 +169,9 @@ func CreateEsQuery(query ...string) string {
return fmt.Sprintf(EsQuery, strings.Join(query, ","))
}
func CreateSourceQuery(source bool) string {
return fmt.Sprintf(sourceQuery, source)
func CreateSourceQuery(fields ...string) string {
formattedFields := `"` + strings.Join(fields, `", "`) + `"`
return fmt.Sprintf(sourceQuery, formattedFields)
}
func CreateFilterAggregationQuery(value ...string) string {
@@ -182,8 +190,8 @@ func CreateMaxAggregationQuery(field string) string {
return fmt.Sprintf(MaxAggregationQuery, field)
}
func CreateTermsAggregationQuery(field string, size int) string {
return fmt.Sprintf(TermsAggregationQuery, field, size)
func CreateTermsAggregationQuery(field string, size int, orderQuery string) string {
return fmt.Sprintf(TermsAggregationQuery, field, size, orderQuery)
}
func CreateTermsAggregationQueryWithoutSize(field string) string {
@@ -261,3 +269,7 @@ func CreateTopHitsAggsQuery(size int, fields []string) string {
func CreateValueCountAggsQuery(field string) string {
return fmt.Sprintf(ValueCountAggsQuery, field)
}
func CreateOrderQuery(field string, order string) string {
return fmt.Sprintf(OrderQuery, field, order)
}

140
service/searchService.go
View File

@@ -47,24 +47,6 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) {
after_query := utils.CreateFromQuery(fromInNumber)
es_query := utils.CreateEsQuery(search_query, size_query, sort_query, after_query)
// searchRequest := `
//{
// "size": 1,
// "query": {
// "term": {
// "error_hash": {
// "value": "%s"
// }
// }
// },
// "sort": [
// { "created_at": { "order": "asc" } }
// ],
// "search_after": ["1724732743"]
//}
//
// `
searchRequestformatted := es_query
fields := []string{"error", "significant_stack", "title"}
@@ -80,6 +62,80 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) {
})
}
func (s *SearchService) GetSearchResults(c *gin.Context) {
projectId := c.Query("project_id")
size := c.DefaultQuery("size", "10")
search_key := c.DefaultQuery("search_key", "")
sizeInNumber, sizeParseError := strconv.Atoi(size)
if sizeParseError != nil {
c.JSON(http.StatusBadRequest, "size should be a number")
return
}
size_query := utils.CreateSizeQuery(int64(sizeInNumber))
term_query := utils.CreateTermSubQuery("project_id", projectId)
multiMatchQuery := utils.CreateMultiMatchQuery(search_key, "error", "title", "extra.metadata")
should_query := utils.CreateMustQuery(term_query, multiMatchQuery)
boolQuery := utils.CreateBoolQuery(should_query)
search_query := utils.CreateSearchQuery(boolQuery)
source_query := utils.CreateSourceQuery("error")
finalQuery := utils.CreateEsQuery(source_query, search_query, size_query)
fields := []string{"error", "significant_stack", "title"}
var res, _, total, err = s.elasticSearchClient.SearchDocuments(finalQuery, fields)
if err != nil {
utils.ErrorResponse(c, "Failed to search please try again later")
return
}
println("final query %s", finalQuery)
c.JSON(http.StatusOK, gin.H{
"results": res,
"total": total,
})
}
func (s *SearchService) GetErrorListV2(c *gin.Context) {
projectId := c.Query("project_id")
sortKey := c.DefaultQuery("sort_key", "lastSeen")
var orderQuery = ""
if sortKey == "lastSeen" {
orderQuery = utils.CreateOrderQuery("last_seen", "desc")
}
if sortKey == "firstSeen" {
orderQuery = utils.CreateOrderQuery("first_seen", "desc")
}
if sortKey == "count" {
orderQuery = utils.CreateOrderQuery("_count", "desc")
}
//size := c.DefaultQuery("size", "100000")
term_query := utils.CreateTermSubQuery("project_id", projectId)
search_query := utils.CreateSearchQuery(term_query)
size_query := utils.CreateSizeQuery(0)
top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery(1, []string{"error", "significant_stack", "created_at", "error_hash"}))
last_seen_aggs_query := utils.BuildAggregationQuery("last_seen", utils.CreateMaxAggregationQuery("created_at"))
first_seen_aggs_query := utils.BuildAggregationQuery("first_seen", utils.CreateMinAggregationQuery("created_at"))
top_hits_aggs_query := utils.CreateAggregationQuery(top_hits_aggs_name_query, last_seen_aggs_query, first_seen_aggs_query)
term_aggs_query := utils.CreateTermsAggregationQuery("error_hash", 100000, orderQuery)
terms_aggs_query_with_name := utils.BuildAggregationQuery("errors_by_hash", term_aggs_query, top_hits_aggs_query)
final_terms_aggs_query := utils.CreateAggregationQuery(terms_aggs_query_with_name)
final_query := utils.CreateEsQuery(search_query, size_query, final_terms_aggs_query)
println("final query %v", final_query)
fields := []string{"error", "significant_stack", "title"}
var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields)
if err != nil {
utils.ErrorResponse(c, "search failed please try again")
return
}
c.JSON(http.StatusOK, gin.H{
"results": aggs,
"total": total,
})
}
func (s *SearchService) GetErrorList(c *gin.Context) {
//todo pagination and aggregation of errors
projectId := c.Query("project_id")
@@ -123,52 +179,8 @@ func (s *SearchService) GetErrorList(c *gin.Context) {
compositeAggsQuery := utils.CreateAggregationQuery(composite_aggs_query)
final_query := utils.CreateEsQuery(search_query, size_query, compositeAggsQuery)
println("%s", final_query)
// searchRequest := `
//{
// "size": 0,
// "query": {
// "term": {
// "project_id": {
// "value": "%s"
// }
// }
// },
// "aggs": {
// "errors_by_hash": {
// "composite": {
// "size": 3,
// "sources": [
// {
// "error_hash": {
// "terms": {
// "field": "error_hash.keyword"
// }
// }
// }
// ]
// },
// "aggs": {
// "unique_errors": {
// "top_hits": {
// "_source": {
// "includes": ["error", "error_hash"]
// }
// }
// },
// "error_count": {
// "value_count": {
// "field": "error.keyword"
// }
// }
// }
// }
//
// }
//}
//
// `
println(final_query)
//s.logger.Info("%s", zap.String("final query", final_query))
fields := []string{"error", "significant_stack", "title"}
var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields)

2
service/sourceMap.go
View File

@@ -44,7 +44,7 @@ func (s *SourceMapService) GetSourceMapUploadUrl(ctx *gin.Context) {
account := ctx.DefaultQuery("account", "ppl")
appendedAccount := "-" + account
if appendedAccount == "ppl" {
if account == "ppl" {
appendedAccount = ""
}