diff --git a/configs/application.yml b/configs/application.yml index b56e58c..02df621 100644 --- a/configs/application.yml +++ b/configs/application.yml @@ -42,7 +42,7 @@ http: # Kafka config kafka: - password: xxx + password: kDia1uC.GI;)Al5eQ)+Q username: varnitgoyal/varnitgoyal95@gmail.com/ocid1.streampool.oc1.ap-mumbai-1.amaaaaaaotdslraanepwp54txqqxkmg4l6dghrhufiezqkx2lqhndgxoq7pa brokers: cell-1.streaming.ap-mumbai-1.oci.oraclecloud.com:9092 group: @@ -71,13 +71,14 @@ elastic: username: elastic password: 9457611267 index: cybertron - api_key: + api_key: SFQ1aHhwRUJWRVNkVUlrRWZMYWk6ZFpDTnpWYXdTNDYwdGt1QzlaRV9YQQ== + + aws: region: ap-south-1 bucket: navi-cd955a63c4476df0f00c1cea0e4a40d1 -#mjolnir config mjolnir: service.url: https://qa-mjolnir-service.np.navi-ppl.in realm.id: O3G7sCWk4r diff --git a/internal/client/elastic/elastic.go b/internal/client/elastic/elastic.go index 078477d..f5df45b 100644 --- a/internal/client/elastic/elastic.go +++ b/internal/client/elastic/elastic.go @@ -44,13 +44,15 @@ func (el *ElasticSearchClient) IndexDocument(document interface{}) { } } -func (el *ElasticSearchClient) SearchDocuments(searchRequest string, fields []string) ([]map[string]interface{}, map[string]interface{}, error) { +func (el *ElasticSearchClient) SearchDocuments(searchRequest string, fields []string) ([]map[string]interface{}, map[string]interface{}, int64, error) { res, err := el.client.Search(). Index(el.Config.Index).Raw(strings.NewReader(searchRequest)). Do(context.TODO()) if err != nil { log.Println("Error getting response: %s", err) + return nil, nil, 0, err } + var results []map[string]interface{} for _, hit := range res.Hits.Hits { var doc map[string]interface{} @@ -72,7 +74,7 @@ func (el *ElasticSearchClient) SearchDocuments(searchRequest string, fields []st log.Printf("%s", aggregations) - return results, aggregations, nil + return results, aggregations, res.Hits.Total.Value, nil } func (el *ElasticSearchClient) GetDocument(documentID string) (interface{}, error) { diff --git a/internal/transport/server.go b/internal/transport/server.go index 6b87d50..469c0f2 100644 --- a/internal/transport/server.go +++ b/internal/transport/server.go @@ -1,7 +1,6 @@ package transport import ( - "cybertron/internal/transport/middleware" "cybertron/internal/transport/router" "fmt" "github.com/gin-contrib/cors" @@ -49,7 +48,7 @@ func (s *Server) Start() { AllowCredentials: true, MaxAge: 24 * time.Hour, })) - s.gin.Use(middleware.PermissionMiddleware(s.dependencies.Service.AuthService)) + //s.gin.Use(middleware.PermissionMiddleware(s.dependencies.Service.AuthService)) s.router() port := configs.GetPort() diff --git a/pkg/utils/elastic_query_util.go b/pkg/utils/elastic_query_util.go index 22eaeaa..25c4c2e 100644 --- a/pkg/utils/elastic_query_util.go +++ b/pkg/utils/elastic_query_util.go @@ -43,8 +43,10 @@ const ( MatchQuery = `{ "match": { "%s": "%s" } }` SearchAfter = `"search_after": [%s]` CompositeAggsQuery = `"composite": {"size" : %d, "sources" : [{"%s": {%s}}] }` - TopHitsAggsQuery = `"top_hits":{"_source": {"includes" : ["%s"]}}` - ValueCountAggsQuery = `"value_count" : {"field": "%s"}` + CompositeAggsQueryWithAfterKey = `"composite": {"size" : %d, "after": {"%s": "%s"}, "sources" : [{"%s": {%s}}] }` + + TopHitsAggsQuery = `"top_hits":{"size" : %d, "_source": {"includes" : [ %s ]}}` + ValueCountAggsQuery = `"value_count" : {"field": "%s"}` ) const ( @@ -234,12 +236,21 @@ func CreateSearchAfterQuery(searchQuery ...string) string { return fmt.Sprintf(SearchAfter, strings.Join(searchQuery, ",")) } -func CreateCompositeAggsQuery(size int, key string, sources ...string) string { +func CreateCompositeAggsQuery(size int, key string, afterKey string, afterKeyValue string, sources ...string) string { + if afterKeyValue != EMPTY { + println("in after key query") + return fmt.Sprintf(CompositeAggsQueryWithAfterKey, size, afterKey, afterKeyValue, key, strings.Join(sources, ",")) + + } return fmt.Sprintf(CompositeAggsQuery, size, key, strings.Join(sources, ",")) } -func CreateTopHitsAggsQuery(fields ...string) string { - return fmt.Sprintf(TopHitsAggsQuery, strings.Join(fields, ",")) +func CreateTopHitsAggsQuery(size int, fields []string) string { + var fieldsList []string + for _, field := range fields { + fieldsList = append(fieldsList, fmt.Sprintf(`"%s"`, field)) + } + return fmt.Sprintf(TopHitsAggsQuery, size, strings.Join(fieldsList, ", ")) } func CreateValueCountAggsQuery(field string) string { diff --git a/service/ExceptionService.go b/service/ExceptionService.go index b64e378..0262f32 100644 --- a/service/ExceptionService.go +++ b/service/ExceptionService.go @@ -78,6 +78,7 @@ func (exceptionService *ExceptionService) CatchErrors(c *gin.Context) { for _, errorItem := range jsonData.Exception.Values { errorItem.ProjectId = projectID + //todo update release id errorItem.ReleaseId = "release-1" errorItem.Breadcrumbs = jsonData.Breadcrumbs errorItem.Extras = jsonData.Extra diff --git a/service/searchService.go b/service/searchService.go index 06e4ff7..cf9e68a 100644 --- a/service/searchService.go +++ b/service/searchService.go @@ -6,6 +6,7 @@ import ( "cybertron/pkg/utils" "github.com/gin-gonic/gin" "net/http" + "strconv" ) type SearchService struct { @@ -26,12 +27,20 @@ func (s *SearchService) Search(c *gin.Context) { func (s *SearchService) GetErrorDetails(c *gin.Context) { error_hash := c.Query("error_hash") + from := c.DefaultQuery("from", "0") + + fromInNumber, err := strconv.ParseInt(from, 10, 64) + + if err != nil { + c.JSON(http.StatusBadRequest, "from should be a number") + return + } term_query := utils.CreateTermSubQuery("error_hash", error_hash) search_query := utils.CreateSearchQuery(term_query) size_query := utils.CreateSizeQuery(1) sort_query := utils.CreateSortQuery("created_at", "asc", "") - after_query := utils.CreateSearchAfterQuery("1724732743") + after_query := utils.CreateFromQuery(fromInNumber) es_query := utils.CreateEsQuery(search_query, size_query, sort_query, after_query) println(es_query) @@ -56,23 +65,35 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) { searchRequestformatted := es_query fields := []string{"error", "significant_stack", "title"} - var response, _, _ = s.elasticSearchClient.SearchDocuments(searchRequestformatted, fields) - c.JSON(http.StatusOK, response) + var response, _, total, _ = s.elasticSearchClient.SearchDocuments(searchRequestformatted, fields) + c.JSON(http.StatusOK, gin.H{ + "results": response, + "total": total, + }) } func (s *SearchService) GetErrorList(c *gin.Context) { //todo pagination and aggregation of errors projectId := c.Query("project_id") + size := c.DefaultQuery("size", "10") + afterKey := c.DefaultQuery("after_key", "") + + sizeInNumber, sizeParseError := strconv.Atoi(size) + + if sizeParseError != nil { + c.JSON(http.StatusBadRequest, "size should be a number") + return + } term_query := utils.CreateTermSubQuery("project_id", projectId) search_query := utils.CreateSearchQuery(term_query) size_query := utils.CreateSizeQuery(0) - top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery("error")) + top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery(1, []string{"error", "significant_stack", "created_at", "error_hash"})) top_hits_aggs_query := utils.CreateAggregationQuery(top_hits_aggs_name_query) //building composite aggregation - composite_term_query := utils.CreateTermsAggregationQueryWithoutSize("error_hash.keyword") - composite_aggregation_query_without_name := utils.CreateCompositeAggsQuery(3, "error_hash", composite_term_query) + composite_term_query := utils.CreateTermsAggregationQueryWithoutSize("error_hash") + composite_aggregation_query_without_name := utils.CreateCompositeAggsQuery(sizeInNumber, "error_hash", "error_hash", afterKey, composite_term_query) composite_aggs_query := utils.BuildAggregationQuery("errors_by_hash", composite_aggregation_query_without_name, top_hits_aggs_query) compositeAggsQuery := utils.CreateAggregationQuery(composite_aggs_query) @@ -128,8 +149,16 @@ func (s *SearchService) GetErrorList(c *gin.Context) { // ` fields := []string{"error", "significant_stack", "title"} - var _, aggs, _ = s.elasticSearchClient.SearchDocuments(final_query, fields) + var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields) - c.JSON(http.StatusOK, aggs) + if err != nil { + c.JSON(http.StatusVariantAlsoNegotiates, "something went wrong ") + return + } + + c.JSON(http.StatusOK, gin.H{ + "results": aggs, + "total": total, + }) }