TP-52077 | Implemented retry option for rca generation failure (#344)

* TP-38709 | Merging the changes to master on the logfix

* TP-52077 | Added on demand rca generation entry point and its implementation

* TP-52077 | slack client ack request issue fix

* TP-52077 | implemented retry button for rca generation failure

* TP-52077 | Added rca generation unit tests

* TP-52077 | Addressed review comments

* TP-52077 | Added retyr button on rca generation failure incoming webhook

* TP-52077 | Fixed unit test conflicts

Makefile

@@ -63,3 +63,4 @@ generatemocks:
 	cd $(CURDIR)/service/incident_jira && minimock -i IncidentJiraService -s _mock.go -o $(CURDIR)/mocks
 	cd $(CURDIR)/model/incident_jira && minimock -i IncidentJiraRepository -s _mock.go -o $(CURDIR)/mocks
 	cd $(CURDIR)/service/google && minimock -i IDriveService -s _mock.go -o $(CURDIR)/mocks
+	cd $(CURDIR)/service/rca && minimock -i IRCAService -s _mock.go -o $(CURDIR)/mocks

common/util/constant.go

@@ -15,6 +15,7 @@ const (
 	SetIncidentTitle                       = "set_incident_title"
 	SetIncidentDescription                 = "set_incident_description"
 	SetRCADetails                          = "set_rca_details"
+	GenerateRCA                            = "generate_rca"
 	RCASection                             = "rca_section"
 	ShowRCADetails                         = "show_rca_details"
 	SetRCASummary                          = "set_rca_summary"
@@ -56,7 +57,8 @@ const (
 const ()
 
 const (
-	ConferenceMessage = "To discuss, use this *<%s|Meet link>*"
+	ConferenceMessage         = "To discuss, use this *<%s|Meet link>*"
+	RCAGenerationErrorMessage = "Some issue occurred while generating RCA."
 )
 
 type ContentType string
@@ -121,3 +123,5 @@ const (
 	ExtensionPNG = ".png"
 	ExtensionCSV = ".csv"
 )
+
+const RedColorCode = "#FF0000"

internal/processor/action/incident_generate_rca_action.go

@@ -0,0 +1,47 @@
+package action
+
+import (
+	"fmt"
+	"github.com/slack-go/slack"
+	"github.com/slack-go/slack/socketmode"
+	"houston/appcontext"
+	"houston/internal/processor/action/view"
+	"houston/logger"
+	"houston/model/incident"
+	incidentService "houston/service/incident"
+	"houston/service/rca"
+	slackService "houston/service/slack"
+)
+
+type IncidentGenerateRCAAction struct {
+	incidentServiceV2 incidentService.IIncidentService
+	slackService      slackService.ISlackService
+	rcaService        rca.IRCAService
+	client            *socketmode.Client
+}
+
+func NewIncidentGenerateRCAAction(client *socketmode.Client) *IncidentGenerateRCAAction {
+	return &IncidentGenerateRCAAction{
+		incidentServiceV2: appcontext.GetIncidentService(),
+		slackService:      appcontext.GetSlackService(),
+		rcaService:        appcontext.GetRCAService(),
+		client:            client,
+	}
+}
+func (igr *IncidentGenerateRCAAction) GenerateRCAProcess(callback slack.InteractionCallback, request *socketmode.Request) {
+	err := igr.slackService.UpdateMessageWithAttachments(callback.Channel.ID, callback.Message.Msg.Timestamp, view.GetRCAFailureAttachmentWithUpdatedMessage(callback.User.ID))
+	if err != nil {
+		logger.Error(fmt.Sprintf("Rca gneration error message update failed due to %s", err.Error()))
+	}
+	igr.client.Ack(*request)
+	incidentEntity, err := igr.incidentServiceV2.GetIncidentByChannelID(callback.Channel.ID)
+	if err != nil {
+		_ = igr.slackService.PostEphemeralByChannelID("Incident not found", callback.User.ID, false, callback.Channel.ID)
+		return
+	}
+	if incidentEntity.Status == incident.ResolvedId {
+		_ = igr.rcaService.GenerateRCA(incidentEntity, callback.User.ID)
+	} else {
+		_ = igr.slackService.PostEphemeralByChannelID("RCA generation is not allowed as Incident is not resolved", callback.User.ID, false, callback.Channel.ID)
+	}
+}

internal/processor/action/incident_resolve_action.go

@@ -2,7 +2,6 @@ package action
 
 import (
 	"fmt"
-	"github.com/spf13/viper"
 	"houston/appcontext"
 	"houston/common/util"
 	"houston/logger"
@@ -119,22 +118,7 @@ func (irp *ResolveIncidentAction) IncidentResolveProcess(callback slack.Interact
 					}
 				}
 			}
-			if viper.GetBool("RCA_GENERATION_ENABLED") {
-				err = irp.rcaService.SendConversationDataForGeneratingRCA(incidentEntity.ID, incidentEntity.IncidentName,
-					channelId)
-				if err != nil {
-					logger.Error(fmt.Sprintf("failed to generate rca for incident id: %d of channel id: %s", incidentEntity.ID, channelId), zap.Error(err))
-					_, _, errMessage := irp.client.PostMessage(channelId, slack.MsgOptionText("`Some issue occurred while generating RCA`", false))
-					if errMessage != nil {
-						logger.Error("post response failed for rca failure message", zap.Error(errMessage))
-					}
-				} else {
-					_, _, errMessage := irp.client.PostMessage(channelId, slack.MsgOptionText("System RCA generation is in progress and might take 2 to 4 minutes.", false))
-					if errMessage != nil {
-						logger.Error("post response failed for rca generated message", zap.Error(errMessage))
-					}
-				}
-			}
+			_ = irp.rcaService.GenerateRCA(incidentEntity, callback.User.ID)
 		}()
 	} else {
 		msgOption := slack.MsgOptionText(fmt.Sprintf("`Please set tag value`"), false)

internal/processor/action/view/incident_rca_failure_message.go

@@ -0,0 +1,35 @@
+package view
+
+import (
+	"fmt"
+	"github.com/slack-go/slack"
+	"houston/common/util"
+)
+
+func GetRCAFailureMessageAttachmentWithRetryButton() slack.Attachment {
+	field := slack.TextBlockObject{Type: util.MarkDownElementType, Text: util.RCAGenerationErrorMessage, Verbatim: false}
+	accessory := slack.NewAccessory(getRCARetryButtonElement())
+	block := slack.NewSectionBlock(&field, nil, accessory)
+	return getFailureAttachment([]slack.Block{block})
+}
+
+func GetRCAFailureAttachmentWithUpdatedMessage(userId string) slack.Attachment {
+	field := slack.TextBlockObject{Type: util.MarkDownElementType, Text: util.RCAGenerationErrorMessage + fmt.Sprintf(" <@%s> re-triggered.", userId), Verbatim: false}
+	block := slack.NewSectionBlock(&field, nil, nil)
+	return getFailureAttachment([]slack.Block{block})
+}
+
+func getFailureAttachment(blocks []slack.Block) slack.Attachment {
+	return slack.Attachment{Blocks: slack.Blocks{BlockSet: blocks}, Color: util.RedColorCode}
+}
+
+func getRCARetryButtonElement() *slack.ButtonBlockElement {
+	return slack.NewButtonBlockElement(
+		util.GenerateRCA,
+		util.GenerateRCA,
+		&slack.TextBlockObject{
+			Type: slack.PlainTextType,
+			Text: "Retry",
+		},
+	)
+}

internal/processor/event_type_interactive_processor.go

@@ -41,6 +41,7 @@ type BlockActionProcessor struct {
 	incidentServiceV2               *incidentService.IncidentServiceV2
 	slackService                    *slack2.SlackService
 	incidentRCASectionAction        *action.IncidentRCASectionAction
+	incidentGenerateRCAAction       *action.IncidentGenerateRCAAction
 }
 
 func NewBlockActionProcessor(
@@ -75,7 +76,8 @@ func NewBlockActionProcessor(
 		incidentUpdateDescriptionAction: action.NewIncidentUpdateDescriptionAction(socketModeClient, incidentRepository),
 		incidentDuplicateAction: action.NewDuplicateIncidentProcessor(socketModeClient, incidentRepository,
 			tagService, teamService, severityService, incidentServiceV2),
-		incidentRCASectionAction: action.NewIncidentRCASectionAction(socketModeClient, incidentRepository, teamService, tagService, severityService, action.NewIncidentTagsAction(incidentRepository, tagService), action.NewIncidentRCASummaryAction(incidentRepository), action.NewIncidentJiraLinksAction(incidentServiceV2, slackService), rcaService),
+		incidentRCASectionAction:  action.NewIncidentRCASectionAction(socketModeClient, incidentRepository, teamService, tagService, severityService, action.NewIncidentTagsAction(incidentRepository, tagService), action.NewIncidentRCASummaryAction(incidentRepository), action.NewIncidentJiraLinksAction(incidentServiceV2, slackService), rcaService),
+		incidentGenerateRCAAction: action.NewIncidentGenerateRCAAction(socketModeClient),
 	}
 }
 
@@ -115,6 +117,11 @@ func (bap *BlockActionProcessor) ProcessCommand(callback slack.InteractionCallba
 		{
 			bap.processRCASectionCommands(callback, request)
 		}
+	case util.GenerateRCA:
+		{
+			bap.incidentGenerateRCAAction.GenerateRCAProcess(callback, request)
+		}
+
 	default:
 		{
 			msgOption := slack.MsgOptionText(fmt.Sprintf("We are working on it"), false)

service/rca/rca_service.go

@@ -9,6 +9,7 @@ import (
 	"go.uber.org/zap"
 	"gorm.io/gorm"
 	"houston/common/util"
+	"houston/internal/processor/action/view"
 	"houston/logger"
 	"houston/model"
 	incident2 "houston/model/incident"
@@ -31,6 +32,11 @@ import (
 	"time"
 )
 
+type IRCAService interface {
+	GenerateRCA(incidentEntity *incident2.IncidentEntity, userId string) error
+	SendConversationDataForGeneratingRCA(incidentId uint, incidentName string, channelId string) error
+}
+
 type RcaService struct {
 	incidentService    incident.IIncidentService
 	slackService       slackService.ISlackService
@@ -151,11 +157,7 @@ func (r *RcaService) PostRcaToIncidentChannel(postRcaRequest service.PostRcaRequ
 			return err
 		}
 
-		_, err = r.slackService.PostMessageByChannelID(
-			"`Some issue occurred while generating RCA`",
-			false,
-			incidentEntity.SlackChannel,
-		)
+		_, err = r.slackService.PostMessageWithAttachments(incidentEntity.SlackChannel, view.GetRCAFailureMessageAttachmentWithRetryButton())
 
 		if err != nil {
 			logger.Error(
@@ -175,7 +177,7 @@ func (r *RcaService) PostRcaToIncidentChannel(postRcaRequest service.PostRcaRequ
 func (r *RcaService) SendConversationDataForGeneratingRCA(incidentId uint, incidentName string, channelId string) error {
 	logger.Info(fmt.Sprintf("Sending conversation data for generating RCA for incident id %d", incidentId))
 	directoryPathResponseCh := make(chan model.DirectoryPathResponse)
-	defer close(directoryPathResponseCh)
+	//defer close(directoryPathResponseCh)
 
 	dataResponse, err := r.getConversationDataForGeneratingRCA(incidentId, incidentName, channelId, directoryPathResponseCh)
 	if err != nil {
@@ -473,3 +475,31 @@ func (r *RcaService) createRcaInputEntity(incidentId uint, documentType string,
 	}
 	return nil
 }
+
+func (r *RcaService) GenerateRCA(incidentEntity *incident2.IncidentEntity, userId string) error {
+	if viper.GetBool("RCA_GENERATION_ENABLED") {
+		err := r.SendConversationDataForGeneratingRCA(incidentEntity.ID, incidentEntity.IncidentName,
+			incidentEntity.SlackChannel)
+		if err != nil {
+			logger.Error(fmt.Sprintf("failed to generate rca for incident id: %d of channel id: %s", incidentEntity.ID, incidentEntity.SlackChannel), zap.Error(err))
+			_, errMessage := r.slackService.PostMessageWithAttachments(incidentEntity.SlackChannel, view.GetRCAFailureMessageAttachmentWithRetryButton())
+			if errMessage != nil {
+				logger.Error("post response failed for rca failure message", zap.Error(errMessage))
+				return errMessage
+			}
+			return err
+		} else {
+			_, errMessage := r.slackService.PostMessageByChannelID("System RCA generation is in progress and might take 2 to 4 minutes.", false, incidentEntity.SlackChannel)
+			if errMessage != nil {
+				logger.Error("post response failed for rca generated message", zap.Error(errMessage))
+			}
+			return errMessage
+		}
+	} else {
+		errMessage := r.slackService.PostEphemeralByChannelID("RCA generation is disabled, please connect with Houston team to enable it", userId, false, incidentEntity.SlackChannel)
+		if errMessage != nil {
+			logger.Error("post response failed for rca generated message", zap.Error(errMessage))
+		}
+		return errMessage
+	}
+}

service/rca/rca_service_test.go

@@ -89,6 +89,7 @@ func TestRcaService_PostRcaToIncidentChannel_Failure(t *testing.T) {
 	rcaRepository.FetchRcaByIncidentIdMock.When(1).Then(testRca, nil)
 
 	rcaRepository.UpdateRcaMock.When(testRca).Then(nil)
+	slackService.PostMessageWithAttachmentsMock.Return("", nil)
 
 	slackService.PostMessageByChannelIDMock.When(
 		"`Some issue occurred while generating RCA`",
@@ -199,6 +200,7 @@ func TestRcaService_PostRcaToIncidentChannel_SlackFailure_RcaFailureCase(t *test
 	rcaRepository.FetchRcaByIncidentIdMock.When(1).Then(testRca, nil)
 
 	rcaRepository.UpdateRcaMock.When(testRca).Then(nil)
+	slackService.PostMessageWithAttachmentsMock.Return("", errors.New("Could not post failure message to the given incident channel"))
 
 	slackService.PostMessageByChannelIDMock.When(
 		"`Some issue occurred while generating RCA`",
@@ -216,6 +218,7 @@ func TestRcaService_PostRcaToIncidentChannel_SlackFailure_RcaFailureCase(t *test
 
 	assert.EqualError(t, err, "Could not post failure message to the given incident channel")
 }
+
 func TestUploadSlackConversationHistory(t *testing.T) {
 	incidentService, slackService, _, rcaService, _, documentService, _, userRepository, _ := setupTest(t)
 
@@ -736,3 +739,51 @@ func TestGetIncidentResponseWithRCALinkSuccessForIdentifiedStatus(t *testing.T)
 	assert.NoError(t, err, "Expected no error")
 	assert.Equal(t, incidentResponse.RcaLink, "")
 }
+
+func TestRcaService_GenerateRCA_RCA_Disabled(t *testing.T) {
+	viper.Set("RCA_GENERATION_ENABLED", false)
+	incidentService, slackServiceMock, _, rcaService, _, _, _, _, _ := setupTest(t)
+	mockIncident := GetMockIncident()
+	incidentService.GetIncidentByIdMock.When(mockIncident.ID).Then(mockIncident, nil)
+	slackServiceMock.PostEphemeralByChannelIDMock.Return(errors.New("error"))
+	err := rcaService.GenerateRCA(mockIncident, "user1")
+	assert.Error(t, err, "error")
+}
+
+func TestRcaService_GenerateRCA_Disabled_Success(t *testing.T) {
+	viper.Set("RCA_GENERATION_ENABLED", false)
+	incidentService, slackServiceMock, _, rcaService, _, _, _, _, _ := setupTest(t)
+	mockIncident := GetMockIncident()
+	incidentService.GetIncidentByIdMock.When(mockIncident.ID).Then(mockIncident, nil)
+	slackServiceMock.PostEphemeralByChannelIDMock.Return(nil)
+	err := rcaService.GenerateRCA(mockIncident, "user1")
+	assert.NoError(t, err, "error")
+}
+
+func TestRcaService_GenerateRCA_Enabled_FailureAtGetConversations(t *testing.T) {
+	viper.Set("RCA_GENERATION_ENABLED", true)
+	incidentService, slackServiceMock, _, rcaService, _, _, _, _, driveServiceMock := setupTest(t)
+	mockIncident := GetMockIncident()
+	incidentService.GetIncidentByIdMock.When(mockIncident.ID).Then(mockIncident, nil)
+	rcaAServiceMock := mocks.NewIRCAServiceMock(t)
+	slackServiceMock.GetSlackConversationHistoryWithRepliesMock.Return(nil, errors.New("error"))
+	driveServiceMock.CollectTranscriptsMock.Return(nil, errors.New("error"))
+	slackServiceMock.PostMessageWithAttachmentsMock.Return("", nil)
+	rcaAServiceMock.SendConversationDataForGeneratingRCAMock.Return(errors.New("error"))
+	err := rcaService.GenerateRCA(mockIncident, "user1")
+	assert.Error(t, err, "error")
+}
+
+func TestRcaService_GenerateRCA_Enabled_FailureAtGetConversationsPostRetryButtonMessage(t *testing.T) {
+	viper.Set("RCA_GENERATION_ENABLED", true)
+	incidentService, slackServiceMock, _, rcaService, _, _, _, _, driveServiceMock := setupTest(t)
+	mockIncident := GetMockIncident()
+	incidentService.GetIncidentByIdMock.When(mockIncident.ID).Then(mockIncident, nil)
+	rcaAServiceMock := mocks.NewIRCAServiceMock(t)
+	slackServiceMock.GetSlackConversationHistoryWithRepliesMock.Return(nil, errors.New("error"))
+	driveServiceMock.CollectTranscriptsMock.Return(nil, errors.New("error"))
+	slackServiceMock.PostMessageWithAttachmentsMock.Return("", errors.New("error"))
+	rcaAServiceMock.SendConversationDataForGeneratingRCAMock.Return(errors.New("error"))
+	err := rcaService.GenerateRCA(mockIncident, "user1")
+	assert.Error(t, err, "error")
+}

service/slack/slack_service_interface.go

@@ -44,4 +44,5 @@ type ISlackService interface {
 	UploadFilesToChannel(files []string, channel string)
 	UploadFileByPath(filePath string, channels string) (file *slack.File, err error)
 	GetConversationInfo(channelId string) (*slack.Channel, error)
+	AckRequest(request socketmode.Request)
 }