varnitgoyal95/cybertron
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TP-55555 | fix es query for pagination

Browse Source
This commit is contained in:
varnit-goyal_navi
2024-09-10 10:04:08 +05:30
parent 4f4476d550
commit f55ab1e53f
6 changed files with 63 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
configs/application.yml
View File

@@ -42,7 +42,7 @@ http:
# Kafka config
kafka:
password: xxx
password: kDia1uC.GI;)Al5eQ)+Q
username: varnitgoyal/varnitgoyal95@gmail.com/ocid1.streampool.oc1.ap-mumbai-1.amaaaaaaotdslraanepwp54txqqxkmg4l6dghrhufiezqkx2lqhndgxoq7pa
brokers: cell-1.streaming.ap-mumbai-1.oci.oraclecloud.com:9092
group:
@@ -71,13 +71,14 @@ elastic:
username: elastic
password: 9457611267
index: cybertron
api_key:
api_key: SFQ1aHhwRUJWRVNkVUlrRWZMYWk6ZFpDTnpWYXdTNDYwdGt1QzlaRV9YQQ==
aws:
region: ap-south-1
bucket: navi-cd955a63c4476df0f00c1cea0e4a40d1
#mjolnir config
mjolnir:
service.url: https://qa-mjolnir-service.np.navi-ppl.in
realm.id: O3G7sCWk4r

6
internal/client/elastic/elastic.go
View File

@@ -44,13 +44,15 @@ func (el *ElasticSearchClient) IndexDocument(document interface{}) {
}
}
func (el *ElasticSearchClient) SearchDocuments(searchRequest string, fields []string) ([]map[string]interface{}, map[string]interface{}, error) {
func (el *ElasticSearchClient) SearchDocuments(searchRequest string, fields []string) ([]map[string]interface{}, map[string]interface{}, int64, error) {
res, err := el.client.Search().
Index(el.Config.Index).Raw(strings.NewReader(searchRequest)).
Do(context.TODO())
if err != nil {
log.Println("Error getting response: %s", err)
return nil, nil, 0, err
}
var results []map[string]interface{}
for _, hit := range res.Hits.Hits {
var doc map[string]interface{}
@@ -72,7 +74,7 @@ func (el *ElasticSearchClient) SearchDocuments(searchRequest string, fields []st
log.Printf("%s", aggregations)
return results, aggregations, nil
return results, aggregations, res.Hits.Total.Value, nil
}
func (el *ElasticSearchClient) GetDocument(documentID string) (interface{}, error) {

3
internal/transport/server.go
View File

@@ -1,7 +1,6 @@
package transport
import (
"cybertron/internal/transport/middleware"
"cybertron/internal/transport/router"
"fmt"
"github.com/gin-contrib/cors"
@@ -49,7 +48,7 @@ func (s *Server) Start() {
AllowCredentials: true,
MaxAge: 24 * time.Hour,
}))
s.gin.Use(middleware.PermissionMiddleware(s.dependencies.Service.AuthService))
//s.gin.Use(middleware.PermissionMiddleware(s.dependencies.Service.AuthService))
s.router()
port := configs.GetPort()

21
pkg/utils/elastic_query_util.go
View File

@@ -43,8 +43,10 @@ const (
MatchQuery = `{ "match": { "%s": "%s" } }`
SearchAfter = `"search_after": [%s]`
CompositeAggsQuery = `"composite": {"size" : %d, "sources" : [{"%s": {%s}}] }`
TopHitsAggsQuery = `"top_hits":{"_source": {"includes" : ["%s"]}}`
ValueCountAggsQuery = `"value_count" : {"field": "%s"}`
CompositeAggsQueryWithAfterKey = `"composite": {"size" : %d, "after": {"%s": "%s"}, "sources" : [{"%s": {%s}}] }`
TopHitsAggsQuery = `"top_hits":{"size" : %d, "_source": {"includes" : [ %s ]}}`
ValueCountAggsQuery = `"value_count" : {"field": "%s"}`
)
const (
@@ -234,12 +236,21 @@ func CreateSearchAfterQuery(searchQuery ...string) string {
return fmt.Sprintf(SearchAfter, strings.Join(searchQuery, ","))
}
func CreateCompositeAggsQuery(size int, key string, sources ...string) string {
func CreateCompositeAggsQuery(size int, key string, afterKey string, afterKeyValue string, sources ...string) string {
if afterKeyValue != EMPTY {
println("in after key query")
return fmt.Sprintf(CompositeAggsQueryWithAfterKey, size, afterKey, afterKeyValue, key, strings.Join(sources, ","))
}
return fmt.Sprintf(CompositeAggsQuery, size, key, strings.Join(sources, ","))
}
func CreateTopHitsAggsQuery(fields ...string) string {
return fmt.Sprintf(TopHitsAggsQuery, strings.Join(fields, ","))
func CreateTopHitsAggsQuery(size int, fields []string) string {
var fieldsList []string
for _, field := range fields {
fieldsList = append(fieldsList, fmt.Sprintf(`"%s"`, field))
}
return fmt.Sprintf(TopHitsAggsQuery, size, strings.Join(fieldsList, ", "))
}
func CreateValueCountAggsQuery(field string) string {

1
service/ExceptionService.go
View File

@@ -78,6 +78,7 @@ func (exceptionService *ExceptionService) CatchErrors(c *gin.Context) {
for _, errorItem := range jsonData.Exception.Values {
errorItem.ProjectId = projectID
//todo update release id
errorItem.ReleaseId = "release-1"
errorItem.Breadcrumbs = jsonData.Breadcrumbs
errorItem.Extras = jsonData.Extra

45
service/searchService.go
View File

@@ -6,6 +6,7 @@ import (
"cybertron/pkg/utils"
"github.com/gin-gonic/gin"
"net/http"
"strconv"
)
type SearchService struct {
@@ -26,12 +27,20 @@ func (s *SearchService) Search(c *gin.Context) {
func (s *SearchService) GetErrorDetails(c *gin.Context) {
error_hash := c.Query("error_hash")
from := c.DefaultQuery("from", "0")
fromInNumber, err := strconv.ParseInt(from, 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, "from should be a number")
return
}
term_query := utils.CreateTermSubQuery("error_hash", error_hash)
search_query := utils.CreateSearchQuery(term_query)
size_query := utils.CreateSizeQuery(1)
sort_query := utils.CreateSortQuery("created_at", "asc", "")
after_query := utils.CreateSearchAfterQuery("1724732743")
after_query := utils.CreateFromQuery(fromInNumber)
es_query := utils.CreateEsQuery(search_query, size_query, sort_query, after_query)
println(es_query)
@@ -56,23 +65,35 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) {
searchRequestformatted := es_query
fields := []string{"error", "significant_stack", "title"}
var response, _, _ = s.elasticSearchClient.SearchDocuments(searchRequestformatted, fields)
c.JSON(http.StatusOK, response)
var response, _, total, _ = s.elasticSearchClient.SearchDocuments(searchRequestformatted, fields)
c.JSON(http.StatusOK, gin.H{
"results": response,
"total": total,
})
}
func (s *SearchService) GetErrorList(c *gin.Context) {
//todo pagination and aggregation of errors
projectId := c.Query("project_id")
size := c.DefaultQuery("size", "10")
afterKey := c.DefaultQuery("after_key", "")
sizeInNumber, sizeParseError := strconv.Atoi(size)
if sizeParseError != nil {
c.JSON(http.StatusBadRequest, "size should be a number")
return
}
term_query := utils.CreateTermSubQuery("project_id", projectId)
search_query := utils.CreateSearchQuery(term_query)
size_query := utils.CreateSizeQuery(0)
top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery("error"))
top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery(1, []string{"error", "significant_stack", "created_at", "error_hash"}))
top_hits_aggs_query := utils.CreateAggregationQuery(top_hits_aggs_name_query)
//building composite aggregation
composite_term_query := utils.CreateTermsAggregationQueryWithoutSize("error_hash.keyword")
composite_aggregation_query_without_name := utils.CreateCompositeAggsQuery(3, "error_hash", composite_term_query)
composite_term_query := utils.CreateTermsAggregationQueryWithoutSize("error_hash")
composite_aggregation_query_without_name := utils.CreateCompositeAggsQuery(sizeInNumber, "error_hash", "error_hash", afterKey, composite_term_query)
composite_aggs_query := utils.BuildAggregationQuery("errors_by_hash", composite_aggregation_query_without_name, top_hits_aggs_query)
compositeAggsQuery := utils.CreateAggregationQuery(composite_aggs_query)
@@ -128,8 +149,16 @@ func (s *SearchService) GetErrorList(c *gin.Context) {
// `
fields := []string{"error", "significant_stack", "title"}
var _, aggs, _ = s.elasticSearchClient.SearchDocuments(final_query, fields)
var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields)
c.JSON(http.StatusOK, aggs)
if err != nil {
c.JSON(http.StatusVariantAlsoNegotiates, "something went wrong ")
return
}
c.JSON(http.StatusOK, gin.H{
"results": aggs,
"total": total,
})
}