diff --git a/internal/transport/handler/search.go b/internal/transport/handler/search.go index 6286c6a..3c5d76d 100644 --- a/internal/transport/handler/search.go +++ b/internal/transport/handler/search.go @@ -10,7 +10,7 @@ type SearchHandler struct { } func (h *SearchHandler) SearchErrors(c *gin.Context) { - h.searchService.GetErrorDetails(c) + h.searchService.GetSearchResults(c) } func (h *SearchHandler) GetErrorDetails(c *gin.Context) { h.searchService.GetErrorDetails(c) @@ -20,6 +20,10 @@ func (h *SearchHandler) GetErrorList(c *gin.Context) { h.searchService.GetErrorList(c) } +func (h *SearchHandler) GetErrorListV2(c *gin.Context) { + h.searchService.GetErrorListV2(c) +} + func NewSearchHandler(s *service.SearchService) *SearchHandler { return &SearchHandler{ searchService: s, diff --git a/internal/transport/router/search.go b/internal/transport/router/search.go index e35a69b..2309267 100644 --- a/internal/transport/router/search.go +++ b/internal/transport/router/search.go @@ -10,8 +10,9 @@ func SearchRouter(r *gin.Engine, dep *dependencies.Dependencies) { searchHandler := handler.NewSearchHandler(dep.Service.SearchService) searchRouterGroup := r.Group("/api/v1") { - searchRouterGroup.GET("/errors-list", searchHandler.GetErrorList) + searchRouterGroup.GET("/errors-list", searchHandler.GetErrorListV2) searchRouterGroup.GET("/error-detail", searchHandler.GetErrorDetails) + searchRouterGroup.GET("/error-search", searchHandler.SearchErrors) } } diff --git a/pkg/utils/elastic_query_util.go b/pkg/utils/elastic_query_util.go index 2513d56..e208e5d 100644 --- a/pkg/utils/elastic_query_util.go +++ b/pkg/utils/elastic_query_util.go @@ -18,15 +18,16 @@ const ( SortQuery = `"sort": [ { "%s": { "order": "%s" } } ]` CollapseQuery = `"collapse": { "field": "%s" }` MatchAllQuery = `{ "match_all": {} }` + MultiMatchQuery = `{"multi_match": { "query": "%s", "fields": [%s] }}` FromQuery = `"from": %d` SizeQuery = `"size": %d` SearchQuery = `"query": %s` FieldsQuery = `"fields": [ "%s" ]` EsQuery = "{ %s }" - sourceQuery = `"_source": %t` + sourceQuery = `"_source": [%s]` AggregationQuery = `"aggs": { %s }` AggregationQueryFormat = `"%s": { %s }` // aggregation name, aggregation query - TermsAggregationQuery = `"terms": { "field": "%s", "size": %d }` + TermsAggregationQuery = `"terms": { "field": "%s", "size": %d, %s}` MinAggregationQuery = `"min": { "field": "%s" }` MaxAggregationQuery = `"max": { "field": "%s" }` CardinalityAggregationQuery = `"cardinality": { "field": "%s" }` @@ -45,6 +46,7 @@ const ( SearchAfter = `"search_after": [%s]` CompositeAggsQuery = `"composite": {"size" : %d, "sources" : [{"%s": {%s}}] }` CompositeAggsQueryWithAfterKey = `"composite": {"size" : %d, "after": {"%s": "%s"}, "sources" : [{"%s": {%s}}] }` + OrderQuery = `"order": {"%s": "%s"}` TopHitsAggsQuery = `"top_hits":{"size" : %d, "_source": {"includes" : [ %s ]}}` ValueCountAggsQuery = `"value_count" : {"field": "%s"}` @@ -119,6 +121,11 @@ func CreateMatchQuery(key string, value string) string { return fmt.Sprintf(MatchQuery, key, value) } +func CreateMultiMatchQuery(query string, fields ...string) string { + // Join fields into a JSON array format: ["field1", "field2", "field3"] + formattedFields := `"` + strings.Join(fields, `", "`) + `"` + return fmt.Sprintf(MultiMatchQuery, query, formattedFields) +} func CreateShouldQuery(filters ...string) string { return fmt.Sprintf(ShouldQuery, strings.Join(filters, ",")) } @@ -162,8 +169,9 @@ func CreateEsQuery(query ...string) string { return fmt.Sprintf(EsQuery, strings.Join(query, ",")) } -func CreateSourceQuery(source bool) string { - return fmt.Sprintf(sourceQuery, source) +func CreateSourceQuery(fields ...string) string { + formattedFields := `"` + strings.Join(fields, `", "`) + `"` + return fmt.Sprintf(sourceQuery, formattedFields) } func CreateFilterAggregationQuery(value ...string) string { @@ -182,8 +190,8 @@ func CreateMaxAggregationQuery(field string) string { return fmt.Sprintf(MaxAggregationQuery, field) } -func CreateTermsAggregationQuery(field string, size int) string { - return fmt.Sprintf(TermsAggregationQuery, field, size) +func CreateTermsAggregationQuery(field string, size int, orderQuery string) string { + return fmt.Sprintf(TermsAggregationQuery, field, size, orderQuery) } func CreateTermsAggregationQueryWithoutSize(field string) string { @@ -261,3 +269,7 @@ func CreateTopHitsAggsQuery(size int, fields []string) string { func CreateValueCountAggsQuery(field string) string { return fmt.Sprintf(ValueCountAggsQuery, field) } + +func CreateOrderQuery(field string, order string) string { + return fmt.Sprintf(OrderQuery, field, order) +} diff --git a/service/searchService.go b/service/searchService.go index dfc492c..e958f9c 100644 --- a/service/searchService.go +++ b/service/searchService.go @@ -47,24 +47,6 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) { after_query := utils.CreateFromQuery(fromInNumber) es_query := utils.CreateEsQuery(search_query, size_query, sort_query, after_query) - // searchRequest := ` - //{ - // "size": 1, - // "query": { - // "term": { - // "error_hash": { - // "value": "%s" - // } - // } - // }, - // "sort": [ - // { "created_at": { "order": "asc" } } - // ], - // "search_after": ["1724732743"] - //} - // - // ` - searchRequestformatted := es_query fields := []string{"error", "significant_stack", "title"} @@ -80,6 +62,80 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) { }) } +func (s *SearchService) GetSearchResults(c *gin.Context) { + projectId := c.Query("project_id") + size := c.DefaultQuery("size", "10") + search_key := c.DefaultQuery("search_key", "") + sizeInNumber, sizeParseError := strconv.Atoi(size) + if sizeParseError != nil { + c.JSON(http.StatusBadRequest, "size should be a number") + return + } + size_query := utils.CreateSizeQuery(int64(sizeInNumber)) + + term_query := utils.CreateTermSubQuery("project_id", projectId) + multiMatchQuery := utils.CreateMultiMatchQuery(search_key, "error", "title", "extra.metadata") + should_query := utils.CreateMustQuery(term_query, multiMatchQuery) + boolQuery := utils.CreateBoolQuery(should_query) + search_query := utils.CreateSearchQuery(boolQuery) + source_query := utils.CreateSourceQuery("error") + finalQuery := utils.CreateEsQuery(source_query, search_query, size_query) + fields := []string{"error", "significant_stack", "title"} + var res, _, total, err = s.elasticSearchClient.SearchDocuments(finalQuery, fields) + if err != nil { + utils.ErrorResponse(c, "Failed to search please try again later") + return + } + println("final query %s", finalQuery) + c.JSON(http.StatusOK, gin.H{ + "results": res, + "total": total, + }) +} + +func (s *SearchService) GetErrorListV2(c *gin.Context) { + projectId := c.Query("project_id") + sortKey := c.DefaultQuery("sort_key", "lastSeen") + var orderQuery = "" + + if sortKey == "lastSeen" { + orderQuery = utils.CreateOrderQuery("last_seen", "desc") + } + if sortKey == "firstSeen" { + orderQuery = utils.CreateOrderQuery("first_seen", "desc") + } + if sortKey == "count" { + orderQuery = utils.CreateOrderQuery("_count", "desc") + } + //size := c.DefaultQuery("size", "100000") + term_query := utils.CreateTermSubQuery("project_id", projectId) + search_query := utils.CreateSearchQuery(term_query) + size_query := utils.CreateSizeQuery(0) + top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery(1, []string{"error", "significant_stack", "created_at", "error_hash"})) + last_seen_aggs_query := utils.BuildAggregationQuery("last_seen", utils.CreateMaxAggregationQuery("created_at")) + first_seen_aggs_query := utils.BuildAggregationQuery("first_seen", utils.CreateMinAggregationQuery("created_at")) + top_hits_aggs_query := utils.CreateAggregationQuery(top_hits_aggs_name_query, last_seen_aggs_query, first_seen_aggs_query) + term_aggs_query := utils.CreateTermsAggregationQuery("error_hash", 100000, orderQuery) + terms_aggs_query_with_name := utils.BuildAggregationQuery("errors_by_hash", term_aggs_query, top_hits_aggs_query) + final_terms_aggs_query := utils.CreateAggregationQuery(terms_aggs_query_with_name) + final_query := utils.CreateEsQuery(search_query, size_query, final_terms_aggs_query) + + println("final query %v", final_query) + fields := []string{"error", "significant_stack", "title"} + var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields) + + if err != nil { + utils.ErrorResponse(c, "search failed please try again") + return + } + + c.JSON(http.StatusOK, gin.H{ + "results": aggs, + "total": total, + }) + +} + func (s *SearchService) GetErrorList(c *gin.Context) { //todo pagination and aggregation of errors projectId := c.Query("project_id") @@ -123,52 +179,8 @@ func (s *SearchService) GetErrorList(c *gin.Context) { compositeAggsQuery := utils.CreateAggregationQuery(composite_aggs_query) final_query := utils.CreateEsQuery(search_query, size_query, compositeAggsQuery) - - println("%s", final_query) - // searchRequest := ` - //{ - // "size": 0, - // "query": { - // "term": { - // "project_id": { - // "value": "%s" - // } - // } - // }, - // "aggs": { - // "errors_by_hash": { - // "composite": { - // "size": 3, - // "sources": [ - // { - // "error_hash": { - // "terms": { - // "field": "error_hash.keyword" - // } - // } - // } - // ] - // }, - // "aggs": { - // "unique_errors": { - // "top_hits": { - // "_source": { - // "includes": ["error", "error_hash"] - // } - // } - // }, - // "error_count": { - // "value_count": { - // "field": "error.keyword" - // } - // } - // } - // } - // - // } - //} - // - // ` + println(final_query) + //s.logger.Info("%s", zap.String("final query", final_query)) fields := []string{"error", "significant_stack", "title"} var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields) diff --git a/service/sourceMap.go b/service/sourceMap.go index d64db16..df84a8a 100644 --- a/service/sourceMap.go +++ b/service/sourceMap.go @@ -44,7 +44,7 @@ func (s *SourceMapService) GetSourceMapUploadUrl(ctx *gin.Context) { account := ctx.DefaultQuery("account", "ppl") appendedAccount := "-" + account - if appendedAccount == "ppl" { + if account == "ppl" { appendedAccount = "" }