TP-5555 | clean up job

internal/transport/handler/search.go

@@ -10,7 +10,7 @@ type SearchHandler struct {
 }
 
 func (h *SearchHandler) SearchErrors(c *gin.Context) {
-	h.searchService.GetErrorDetails(c)
+	h.searchService.GetSearchResults(c)
 }
 func (h *SearchHandler) GetErrorDetails(c *gin.Context) {
 	h.searchService.GetErrorDetails(c)
@@ -20,6 +20,10 @@ func (h *SearchHandler) GetErrorList(c *gin.Context) {
 	h.searchService.GetErrorList(c)
 }
 
+func (h *SearchHandler) GetErrorListV2(c *gin.Context) {
+	h.searchService.GetErrorListV2(c)
+}
+
 func NewSearchHandler(s *service.SearchService) *SearchHandler {
 	return &SearchHandler{
 		searchService: s,

internal/transport/router/search.go

@@ -10,8 +10,9 @@ func SearchRouter(r *gin.Engine, dep *dependencies.Dependencies) {
 	searchHandler := handler.NewSearchHandler(dep.Service.SearchService)
 	searchRouterGroup := r.Group("/api/v1")
 	{
-		searchRouterGroup.GET("/errors-list", searchHandler.GetErrorList)
+		searchRouterGroup.GET("/errors-list", searchHandler.GetErrorListV2)
 		searchRouterGroup.GET("/error-detail", searchHandler.GetErrorDetails)
+		searchRouterGroup.GET("/error-search", searchHandler.SearchErrors)
 	}
 
 }

pkg/utils/elastic_query_util.go

@@ -19,15 +19,16 @@ const (
 	SortQuery                        = `"sort": [ { "%s": { "order": "%s" } } ]`
 	CollapseQuery                    = `"collapse": { "field": "%s" }`
 	MatchAllQuery                    = `{ "match_all": {} }`
+	MultiMatchQuery                  = `{"multi_match": { "query": "%s", "fields": [%s] }}`
 	FromQuery                        = `"from": %d`
 	SizeQuery                        = `"size": %d`
 	SearchQuery                      = `"query": %s`
 	FieldsQuery                      = `"fields": [ "%s" ]`
 	EsQuery                          = "{ %s }"
-	sourceQuery                      = `"_source": %t`
+	sourceQuery                      = `"_source": [%s]`
 	AggregationQuery                 = `"aggs": { %s }`
 	AggregationQueryFormat           = `"%s": { %s }` // aggregation name, aggregation query
-	TermsAggregationQuery            = `"terms": { "field": "%s", "size": %d }`
+	TermsAggregationQuery            = `"terms": { "field": "%s", "size": %d, %s}`
 	MinAggregationQuery              = `"min": { "field": "%s" }`
 	MaxAggregationQuery              = `"max": { "field": "%s" }`
 	CardinalityAggregationQuery      = `"cardinality": { "field": "%s" }`
@@ -46,6 +47,7 @@ const (
 	SearchAfter                      = `"search_after": [%s]`
 	CompositeAggsQuery               = `"composite": {"size" : %d, "sources" : [{"%s": {%s}}] }`
 	CompositeAggsQueryWithAfterKey   = `"composite": {"size" : %d, "after": {"%s": "%s"}, "sources" : [{"%s": {%s}}] }`
+	OrderQuery                       = `"order": {"%s": "%s"}`
 
 	TopHitsAggsQuery    = `"top_hits":{"size" : %d, "_source": {"includes" : [ %s ]}}`
 	ValueCountAggsQuery = `"value_count" : {"field": "%s"}`
@@ -124,6 +126,11 @@ func CreateMatchQuery(key string, value string) string {
 	return fmt.Sprintf(MatchQuery, key, value)
 }
 
+func CreateMultiMatchQuery(query string, fields ...string) string {
+	// Join fields into a JSON array format: ["field1", "field2", "field3"]
+	formattedFields := `"` + strings.Join(fields, `", "`) + `"`
+	return fmt.Sprintf(MultiMatchQuery, query, formattedFields)
+}
 func CreateShouldQuery(filters ...string) string {
 	return fmt.Sprintf(ShouldQuery, strings.Join(filters, ","))
 }
@@ -167,8 +174,9 @@ func CreateEsQuery(query ...string) string {
 	return fmt.Sprintf(EsQuery, strings.Join(query, ","))
 }
 
-func CreateSourceQuery(source bool) string {
+func CreateSourceQuery(fields ...string) string {
-	return fmt.Sprintf(sourceQuery, source)
+	formattedFields := `"` + strings.Join(fields, `", "`) + `"`
+	return fmt.Sprintf(sourceQuery, formattedFields)
 }
 
 func CreateFilterAggregationQuery(value ...string) string {
@@ -187,8 +195,8 @@ func CreateMaxAggregationQuery(field string) string {
 	return fmt.Sprintf(MaxAggregationQuery, field)
 }
 
-func CreateTermsAggregationQuery(field string, size int) string {
+func CreateTermsAggregationQuery(field string, size int, orderQuery string) string {
-	return fmt.Sprintf(TermsAggregationQuery, field, size)
+	return fmt.Sprintf(TermsAggregationQuery, field, size, orderQuery)
 }
 
 func CreateTermsAggregationQueryWithoutSize(field string) string {
@@ -266,3 +274,7 @@ func CreateTopHitsAggsQuery(size int, fields []string) string {
 func CreateValueCountAggsQuery(field string) string {
 	return fmt.Sprintf(ValueCountAggsQuery, field)
 }
+
+func CreateOrderQuery(field string, order string) string {
+	return fmt.Sprintf(OrderQuery, field, order)
+}

service/ProjectCreator.go

@@ -29,12 +29,14 @@ type ProjectBody struct {
 	Team      string `json:"team" binding:"required"`
 	Icon      string `json:"icon"`
 	GithubUrl string `json:"githubUrl" binding:"required"`
+	Account   string `json:"account" binding:"required"`
 }
 
 type UpdateProjectBody struct {
 	Icon           string   `json:"logoUrl" binding:"required"`
 	GithubUrl      string   `json:"githubUrl" binding:"required"`
-	IgnorePatterns []string `json:"ignorePatterns" binding:"required"`
+	IgnorePatterns []string `json:"ignorePatterns" binding:"omitempty"`
+	Account        string   `json:"account" binding:"required"`
 }
 
 func NewProjectCreator(logger *log.Logger, dbClient *gorm.DB, s3Client *aws.Actions, kafkaProducer producer.KProducer) *ProjectCreator {
@@ -75,6 +77,7 @@ func (pc *ProjectCreator) CreateProject(ctx *gin.Context) {
 		Icon:               projectBody.Icon,
 		GithubUrl:          projectBody.GithubUrl,
 		Secret:             hex.EncodeToString(secret),
+		Account:            projectBody.Account,
 	})
 
 	if result.Error != nil {
@@ -122,7 +125,7 @@ func (pc *ProjectCreator) ProjectUpdate(c *gin.Context) {
 	var updateProjectBody UpdateProjectBody
 
 	// Find the project by ID
-	if result := pc.dbClient.Where("id = ?", projectId).First(&project); result.Error != nil {
+	if result := pc.dbClient.Where("project_reference_id = ?", projectId).First(&project); result.Error != nil {
 		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
 			c.JSON(http.StatusNotFound, gin.H{"error": "Project not found"})
 		} else {
@@ -138,10 +141,11 @@ func (pc *ProjectCreator) ProjectUpdate(c *gin.Context) {
 	}
 
 	// Update the project in the database
-	if result := pc.dbClient.Where("id = ?", projectId).Updates(&db.Project{
+	if result := pc.dbClient.Where("project_reference_id = ?", projectId).Updates(&db.Project{
 		GithubUrl:      updateProjectBody.GithubUrl,
 		Icon:           updateProjectBody.Icon,
 		IgnorePatterns: updateProjectBody.IgnorePatterns,
+		Account:        updateProjectBody.Account,
 	}); result.Error != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": result.Error.Error()})
 		return

service/searchService.go

@@ -62,6 +62,80 @@ func (s *SearchService) GetErrorDetails(c *gin.Context) {
 	})
 }
 
+func (s *SearchService) GetSearchResults(c *gin.Context) {
+	projectId := c.Query("project_id")
+	size := c.DefaultQuery("size", "10")
+	search_key := c.DefaultQuery("search_key", "")
+	sizeInNumber, sizeParseError := strconv.Atoi(size)
+	if sizeParseError != nil {
+		c.JSON(http.StatusBadRequest, "size should be a number")
+		return
+	}
+	size_query := utils.CreateSizeQuery(int64(sizeInNumber))
+
+	term_query := utils.CreateTermSubQuery("project_id", projectId)
+	multiMatchQuery := utils.CreateMultiMatchQuery(search_key, "error", "title", "extra.metadata")
+	should_query := utils.CreateMustQuery(term_query, multiMatchQuery)
+	boolQuery := utils.CreateBoolQuery(should_query)
+	search_query := utils.CreateSearchQuery(boolQuery)
+	source_query := utils.CreateSourceQuery("error")
+	finalQuery := utils.CreateEsQuery(source_query, search_query, size_query)
+	fields := []string{"error", "significant_stack", "title"}
+	var res, _, total, err = s.elasticSearchClient.SearchDocuments(finalQuery, fields)
+	if err != nil {
+		utils.ErrorResponse(c, "Failed to search please try again later")
+		return
+	}
+	//println("final query %s", finalQuery)
+	c.JSON(http.StatusOK, gin.H{
+		"results": res,
+		"total":   total,
+	})
+}
+
+func (s *SearchService) GetErrorListV2(c *gin.Context) {
+	projectId := c.Query("project_id")
+	sortKey := c.DefaultQuery("sort_key", "lastSeen")
+	var orderQuery = ""
+
+	if sortKey == "lastSeen" {
+		orderQuery = utils.CreateOrderQuery("last_seen", "desc")
+	}
+	if sortKey == "firstSeen" {
+		orderQuery = utils.CreateOrderQuery("first_seen", "desc")
+	}
+	if sortKey == "count" {
+		orderQuery = utils.CreateOrderQuery("_count", "desc")
+	}
+	//size := c.DefaultQuery("size", "100000")
+	term_query := utils.CreateTermSubQuery("project_id", projectId)
+	search_query := utils.CreateSearchQuery(term_query)
+	size_query := utils.CreateSizeQuery(0)
+	top_hits_aggs_name_query := utils.BuildAggregationQuery("unique_errors", utils.CreateTopHitsAggsQuery(1, []string{"error", "significant_stack", "created_at", "error_hash"}))
+	last_seen_aggs_query := utils.BuildAggregationQuery("last_seen", utils.CreateMaxAggregationQuery("created_at"))
+	first_seen_aggs_query := utils.BuildAggregationQuery("first_seen", utils.CreateMinAggregationQuery("created_at"))
+	top_hits_aggs_query := utils.CreateAggregationQuery(top_hits_aggs_name_query, last_seen_aggs_query, first_seen_aggs_query)
+	term_aggs_query := utils.CreateTermsAggregationQuery("error_hash", 100000, orderQuery)
+	terms_aggs_query_with_name := utils.BuildAggregationQuery("errors_by_hash", term_aggs_query, top_hits_aggs_query)
+	final_terms_aggs_query := utils.CreateAggregationQuery(terms_aggs_query_with_name)
+	final_query := utils.CreateEsQuery(search_query, size_query, final_terms_aggs_query)
+
+	println("final query %v", final_query)
+	fields := []string{"error", "significant_stack", "title"}
+	var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields)
+
+	if err != nil {
+		utils.ErrorResponse(c, "search failed please try again")
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"results": aggs,
+		"total":   total,
+	})
+
+}
+
 func (s *SearchService) GetErrorList(c *gin.Context) {
 	//todo pagination and aggregation of errors
 	projectId := c.Query("project_id")
@@ -105,8 +179,8 @@ func (s *SearchService) GetErrorList(c *gin.Context) {
 	compositeAggsQuery := utils.CreateAggregationQuery(composite_aggs_query)
 
 	final_query := utils.CreateEsQuery(search_query, size_query, compositeAggsQuery)
+	//s.logger.Info("%s", zap.String("final query", final_query))
 
-	println("%s", final_query)
 	fields := []string{"error", "significant_stack", "title"}
 	var _, aggs, total, err = s.elasticSearchClient.SearchDocuments(final_query, fields)
 

service/sourceMap.go

@@ -44,7 +44,7 @@ func (s *SourceMapService) GetSourceMapUploadUrl(ctx *gin.Context) {
 	account := ctx.DefaultQuery("account", "ppl")
 	appendedAccount := "-" + account
 
-	if appendedAccount == "ppl" {
+	if account == "ppl" {
 		appendedAccount = ""
 	}