INFRA-2887 : SLA breach heads up cron refactor/reimplementation (#411)

* INFRA-2887 : SLA breach heads up cron refactor/reimplementation

* INFRA-2887 : Code review comments

appcontext/app.go

@@ -355,7 +355,7 @@ func GetDriveService() google.IDriveService {
 }
 
 func initReminderService() reminder.ReminderService {
-	return reminder.NewReminderService(GetIncidentService(), GetTeamService(), GetSlackService())
+	return reminder.NewReminderService(GetIncidentService(), GetTeamService(), GetSlackService(), GetSeverityService())
 }
 
 func GetReminderService() reminder.ReminderService {

cmd/app/handler/reminder_handler.go

@@ -32,3 +32,13 @@ func (handler *ReminderHandler) HandleTeamIncidents(c *gin.Context) {
 	}
 	c.JSON(http.StatusOK, common.SuccessResponse("Team metrics posted successfully", http.StatusOK))
 }
+
+func (handler *ReminderHandler) HandleSlaBreachReminder(c *gin.Context) {
+	err := handler.service.PostSlaBreachMessages()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, common.ErrorResponse(err, http.StatusInternalServerError, nil))
+		metrics.PublishHoustonFlowFailureMetrics("SLA_BREACH_REMINDER", err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, common.SuccessResponse("SLA breach reminder posted successfully", http.StatusOK))
+}

cmd/app/server.go

@@ -189,6 +189,7 @@ func (s *Server) productTeamsHandler(houstonGroup *gin.RouterGroup) {
 func (s *Server) reminderHandler(houstonGroup *gin.RouterGroup) {
 	reminderHandler := handler.NewReminderHandler(s.gin)
 	houstonGroup.POST("reminder/team-incidents", reminderHandler.HandleTeamIncidents)
+	houstonGroup.POST("reminder/sla-breach", reminderHandler.HandleSlaBreachReminder)
 }
 
 func (s *Server) incidentClientHandlerV2(houstonGroup *gin.RouterGroup) {

common/thread_safe_error.go

@@ -1,6 +1,10 @@
 package common
 
-import "sync"
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
 
 type ThreadSafeErrors struct {
 	mutex  sync.Mutex
@@ -16,3 +20,18 @@ func (errs *ThreadSafeErrors) AddErrors(errsToAdd ...error) {
 func (errs *ThreadSafeErrors) GetErrors() []error {
 	return errs.errors
 }
+
+func (errs *ThreadSafeErrors) CollectErrors() error {
+	if len(errs.errors) == 0 {
+		return nil
+	}
+
+	var errorStrings []string
+	for _, err := range errs.errors {
+		if err != nil {
+			errorStrings = append(errorStrings, err.Error())
+		}
+	}
+
+	return fmt.Errorf("%s", strings.Join(errorStrings, "; "))
+}

common/util/common_test_util.go

@@ -0,0 +1,11 @@
+package util
+
+import "houston/model/severity"
+
+func GetMockSeverityEscalationMap() map[uint]*severity.SeverityDTO {
+	return map[uint]*severity.SeverityDTO{
+		2: {ID: 2, Priority: 2, Name: "P2"},
+		3: {ID: 3, Priority: 3, Name: "P3"},
+		4: {ID: 4, Priority: 4, Name: "P4"},
+	}
+}

common/util/common_util.go

@@ -5,14 +5,9 @@ import (
 	"github.com/lib/pq"
 	"github.com/slack-go/slack"
 	"github.com/slack-go/slack/socketmode"
-	"go.uber.org/zap"
 	"golang.org/x/exp/slices"
 	"gorm.io/gorm"
-	"houston/common/metrics"
 	"houston/logger"
-	"houston/model/incident"
-	"houston/model/severity"
-	"houston/model/team"
 	"math"
 	"reflect"
 	"runtime"
@@ -213,46 +208,12 @@ func RemoveString(slice []string, strToRemove string) []string {
 	return result
 }
 
-func PostIncidentSeverityEscalationHeadsUpMessage(severities *[]severity.SeverityEntity, incidentEntity *incident.IncidentEntity, teamEntity *team.TeamEntity, client *socketmode.Client, jobName string) {
-	fromSeverityName := getSeverityById(severities, incidentEntity.SeverityId)
-	toSeverityName := getSeverityById(severities, incidentEntity.SeverityId-1)
-	daysToEscalation := calculateDifferenceInDays(incidentEntity.SeverityTat, time.Now())
-	teamSlackChannel := teamEntity.WebhookSlackChannel
-	if daysToEscalation != 0 {
-		msgOption := slack.MsgOptionText(fmt.Sprintf("This incident will be auto-escalated to `%v` in `%v day(s)`", toSeverityName, daysToEscalation), false)
-		_, _, err := client.PostMessage(incidentEntity.SlackChannel, msgOption)
-		if err != nil {
-			logger.Info(fmt.Sprintf("Error posting message to incident channel for incident %v", incidentEntity.IncidentName), zap.Error(err))
-			metrics.PublishCronJobFailureMetrics(jobName, err.Error())
-			return
-		}
-		if teamSlackChannel != "" {
-			msgOption := slack.MsgOptionText(fmt.Sprintf("<#%s> (`%v`) will be auto-escalated to `%v` in `%v day(s)`", incidentEntity.SlackChannel, fromSeverityName, toSeverityName, daysToEscalation), false)
-			_, _, err := client.PostMessage(teamSlackChannel, msgOption)
-			if err != nil {
-				logger.Info(fmt.Sprintf("Error posting message to team channel for incident %v", incidentEntity.IncidentName), zap.Error(err))
-				metrics.PublishCronJobFailureMetrics(jobName, err.Error())
-				return
-			}
-		}
-	}
-}
-
 func ExecuteConcurrentAction(waitGroup *sync.WaitGroup, concurrentTask func()) {
 	defer waitGroup.Done()
 	concurrentTask()
 }
 
-func getSeverityById(severities *[]severity.SeverityEntity, severityId uint) string {
-	for _, severityEntity := range *severities {
-		if severityEntity.ID == severityId {
-			return severityEntity.Name
-		}
-	}
-	return ""
-}
-
-func calculateDifferenceInDays(fromTime, toTime time.Time) int {
+func CalculateDifferenceInDays(fromTime, toTime time.Time) int {
 	fromDate := time.Date(fromTime.Year(), fromTime.Month(), fromTime.Day(), 0, 0, 0, 0, time.UTC)
 	toDate := time.Date(toTime.Year(), toTime.Month(), toTime.Day(), 0, 0, 0, 0, time.UTC)
 	return int(math.Abs(toDate.Sub(fromDate).Hours() / 24))

internal/cron/cron.go

@@ -48,16 +48,6 @@ func RunJob(
 		logger.Error("HOUSTON_ADDING_USER error :" + err.Error())
 	}
 
-	//Post SLA Breach Message to Incident Channels
-	err = shedlockConfig.AddFun(viper.GetString("cron.job.sla_breach"), viper.GetString("cron.job.sla_breach_interval"), shedlockRepository, func() {
-		RunJobWithExecutionMetrics(SLA_BREACH, func() {
-			PostSLABreachMessageToIncidentChannels(socketModeClient, teamRepository, incidentRepository, severityRepository)
-		})
-	})
-	if err != nil {
-		logger.Error("HOUSTON_SLA_BREACH error :" + err.Error())
-	}
-
 	//HOUSTON DAILY INCIDENT REMINDER DMs TO USERS
 	err = shedlockConfig.AddFun(viper.GetString("cron.job.incident_reminder"), viper.GetString("cron.job.incident_reminder_interval"), shedlockRepository, func() {
 		RunJobWithExecutionMetrics(INCIDENT_REMINDER, func() {
@@ -205,36 +195,6 @@ func UpsertUsers(socketModeClient *socketmode.Client, userService *user.Reposito
 	fmt.Println("Finishing upsertUsers job at", time.Now().Format(time.RFC3339))
 }
 
-func PostSLABreachMessageToIncidentChannels(socketModeClient *socketmode.Client, teamRepository *team.Repository, incidentRepository *incident.Repository, severityRepository *severity.Repository) {
-	logger.Info(fmt.Sprintf("Running %v job at %v", viper.GetString("cron.job.sla_breach"), time.Now().Format(time.RFC822)))
-	defer func() {
-		if r := recover(); r != nil {
-			logger.Error(fmt.Sprintf("Exception occurred in cron: %v", r.(error)))
-			metrics.PublishCronJobFailureMetrics(SLA_BREACH, r.(error).Error())
-		}
-	}()
-	slaStart := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	slaEnd := time.Now().AddDate(0, 0, 4).Format("2006-01-02")
-	incidents, err := incidentRepository.FetchAllOpenIncidentsWithSeverityTATGreaterThan(slaStart, slaEnd, 2, 3, 4)
-	if err != nil {
-		logger.Error("error occurred fetching open incidents for SLA breach", zap.Error(err))
-		panic(err)
-		return
-	}
-	if len(*incidents) == 0 {
-		logger.Info("No incidents found to be updated by cron job")
-	}
-	severities, _ := severityRepository.GetAllActiveSeverity()
-	for _, incidentEntity := range *incidents {
-		teamEntity, err := teamRepository.FindTeamById(incidentEntity.TeamId)
-		if err != nil {
-			logger.Error("error in fetching team by id", zap.Uint("teamId", incidentEntity.TeamId), zap.Error(err))
-			metrics.PublishCronJobFailureMetrics(SLA_BREACH, err.Error())
-		}
-		util.PostIncidentSeverityEscalationHeadsUpMessage(severities, &incidentEntity, teamEntity, socketModeClient, SLA_BREACH)
-	}
-}
-
 // HoustonIncidentReminderToUsers - sends slack DMs to users with list of Houston incidents they are part of
 func HoustonIncidentReminderToUsers(
 	incidentService *incidentService.IncidentServiceV2,

model/incident/entity.go

@@ -100,6 +100,7 @@ func (i IncidentEntity) ToDTO() IncidentDTO {
 		ConferenceId:    i.ConferenceId,
 		ReportingTeamId: i.ReportingTeamId,
 		Products:        products,
+		SeverityTat:     i.SeverityTat,
 		Team:            *(i.Team).ToDTO(),
 		Severity:        i.Severity.ToDTO(),
 	}

model/incident/incident.go

@@ -774,13 +774,14 @@ func (r *Repository) FetchAllNonTerminalIncidentStatuses() (*[]IncidentStatusEnt
 	return &incidentStatusEntity, nil
 }
 
-func (r *Repository) FetchAllOpenIncidentsWithSeverityTATGreaterThan(slaStart, slaEnd string, severityId ...int) (*[]IncidentEntity, error) {
+func (r *Repository) FetchIncidentsWithSeverityTatBetweenGivenRange(slaStart, slaEnd string) (*[]IncidentEntity, error) {
 	var incidents []IncidentEntity
-	query := fmt.Sprintf("SELECT * FROM incident WHERE severity_tat >= '%v' AND severity_tat < '%v' AND severity_id IN (%v) AND status IN (%s) ORDER BY team_id;", slaStart, slaEnd, strings.Trim(strings.Join(strings.Fields(fmt.Sprint(severityId)), ","), "[]"), SLA_BREACH_ALLOWED_STATUSES)
-	logger.Info("Query: ", zap.String("query", query))
-	result := r.gormClient.Raw(query).Scan(&incidents)
-	if result.Error != nil {
-		return nil, result.Error
+	query := r.gormClient.Where(
+		"severity_tat >= ? AND severity_tat < ? AND severity_id IN (?) AND status IN (?)",
+		slaStart, slaEnd, severitiesForSLABreach, statusesForEscalation).
+		Order("team_id").Preload("Team").Preload("Severity").Find(&incidents)
+	if query.Error != nil {
+		return nil, query.Error
 	}
 	return &incidents, nil
 }
@@ -794,7 +795,4 @@ func (r *Repository) UpdateIncidentChannelEntity(incidentChannelEntity *Incident
 }
 
 var statusesForEscalation = []uint{1, 2}
-
-const (
-	SLA_BREACH_ALLOWED_STATUSES = "1,2"
-)
+var severitiesForSLABreach = []uint{2, 3, 4}

model/incident/incident_repository_interface.go

@@ -43,8 +43,8 @@ type IIncidentRepository interface {
 	GetIncidentRolesByIncidentIdsAndRole(incidentsIds []uint, role string) ([]IncidentRoleEntity, error)
 	FindOpenIncidentsByTeamOrderedByCreationTimeAndSeverity(team string) (*[]IncidentEntity, error)
 	FetchAllNonTerminalIncidentStatuses() (*[]IncidentStatusEntity, error)
-	FetchAllOpenIncidentsWithSeverityTATGreaterThan(slaStart, slaEnd string, severityId ...int) (*[]IncidentEntity, error)
 	UpdateIncidentChannelEntity(incidentChannelEntity *IncidentChannelEntity) error
 	GetOpenIncidentsByCreatorIdForGivenTeam(created_by string, teamId uint) (*[]IncidentEntity, error)
 	GetIncidentTagsByTagIds(incidentId uint, tagIds []uint) (*IncidentTagEntity, error)
+	FetchIncidentsWithSeverityTatBetweenGivenRange(slaStart, slaEnd string) (*[]IncidentEntity, error)
 }

model/incident/model.go

@@ -111,6 +111,7 @@ type IncidentDTO struct {
 	ConferenceLink  string               `json:"conference_link,omitempty"`
 	ReportingTeamId *uint                `json:"reporting_team_id,omitempty"`
 	Products        []product.ProductDTO `json:"products,omitempty"`
+	SeverityTat     time.Time            `json:"severity_tat,omitempty"`
 	Team            team.TeamDTO         `json:"team,omitempty"`
 	Severity        severity.SeverityDTO `json:"severity,omitempty"`
 }

service/incident/impl/escalate_incident.go

@@ -6,8 +6,8 @@ import (
 	"github.com/spf13/viper"
 	"houston/logger"
 	incidentModel "houston/model/incident"
+	"houston/model/severity"
 	service "houston/service/request"
-	"strconv"
 )
 
 const escalateLogTag = "[escalate]"
@@ -30,15 +30,15 @@ func (i *IncidentServiceV2) EscalateIncidents() error {
 	return i.runEscalationOnIncidents(incidents, escalationMap)
 }
 
-func (i *IncidentServiceV2) runEscalationOnIncidents(incidents []incidentModel.IncidentEntity, escalationMap map[uint]*uint) error {
+func (i *IncidentServiceV2) runEscalationOnIncidents(incidents []incidentModel.IncidentEntity, escalationMap map[uint]*severity.SeverityDTO) error {
 	logger.Info(fmt.Sprintf("%s found %d incidents for escalation", escalateLogTag, len(incidents)))
 
 	var incidentsWithFailures []string
 
 	for _, incident := range incidents {
 		logger.Info(fmt.Sprintf("%s escalating incident with id: %d", escalateLogTag, incident.ID))
-		nextSeverityID := escalationMap[incident.SeverityId]
-		if nextSeverityID == nil {
+		nextSeverity, ok := escalationMap[incident.SeverityId]
+		if !ok {
 			logger.Error(fmt.Sprintf("%s no escalation found for incident with id: %d", escalateLogTag, incident.ID))
 			incidentsWithFailures = append(incidentsWithFailures, incident.IncidentName)
 			continue
@@ -46,7 +46,7 @@ func (i *IncidentServiceV2) runEscalationOnIncidents(incidents []incidentModel.I
 
 		_, err := i.UpdateIncident(service.UpdateIncidentRequest{
 			Id:         incident.ID,
-			SeverityId: strconv.Itoa(int(*nextSeverityID)),
+			SeverityId: fmt.Sprintf("%d", nextSeverity.ID),
 		}, viper.GetString("HOUSTON_BOT_SLACK_ID"))
 
 		if err != nil {

service/incident/impl/escalate_incident_test.go

@@ -3,6 +3,7 @@ package impl
 import (
 	"errors"
 	"github.com/slack-go/slack"
+	"houston/common/util"
 	"houston/model/incident"
 	teamUserModel "houston/model/teamUser"
 	"houston/model/user"
@@ -41,7 +42,7 @@ func (suite *IncidentServiceSuite) Test_Escalate_SeverityMapNil() {
 func (suite *IncidentServiceSuite) Test_Escalate_IncidentUpdateError() {
 	incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
 	suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil)
-	suite.severityService.GetSeverityEscalationMapMock.Return(getMockSeverityEscalationMap(), nil)
+	suite.severityService.GetSeverityEscalationMapMock.Return(util.GetMockSeverityEscalationMap(), nil)
 	suite.incidentRepository.FindIncidentByIdMock.Return(nil, errors.New("error"))
 	err := suite.incidentService.EscalateIncidents()
 	suite.Error(err, "service should return error")
@@ -103,20 +104,8 @@ func (suite *IncidentServiceSuite) Test_Escalate_Success() {
 
 	incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
 	suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil)
-	suite.severityService.GetSeverityEscalationMapMock.Return(getMockSeverityEscalationMap(), nil)
+	suite.severityService.GetSeverityEscalationMapMock.Return(util.GetMockSeverityEscalationMap(), nil)
 
 	err := suite.incidentService.EscalateIncidents()
 	suite.NoError(err, "service should not return error")
 }
-
-func getMockSeverityEscalationMap() map[uint]*uint {
-	return map[uint]*uint{
-		2: uintPtr(1),
-		3: uintPtr(2),
-		4: uintPtr(3),
-	}
-}
-
-func uintPtr(i uint) *uint {
-	return &i
-}

service/incident/impl/fetch_incidents.go

@@ -0,0 +1,18 @@
+package impl
+
+import (
+	"houston/common/util"
+	"houston/model/incident"
+	"time"
+)
+
+func (i *IncidentServiceV2) FetchIncidentsApproachingSlaBreach() ([]incident.IncidentDTO, error) {
+	incidents, err := i.incidentRepository.FetchIncidentsWithSeverityTatBetweenGivenRange(
+		time.Now().AddDate(0, 0, 1).Format("2006-01-02"),
+		time.Now().AddDate(0, 0, 4).Format("2006-01-02"),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return util.ToDtoArray[incident.IncidentEntity, incident.IncidentDTO](*incidents), nil
+}

service/incident/impl/fetch_incidents_test.go

@@ -0,0 +1,20 @@
+package impl
+
+import (
+	"errors"
+	"houston/model/incident"
+)
+
+func (suite *IncidentServiceSuite) Test_FetchIncidentsApproachingSlaBreach_GetIncidentsError() {
+	suite.incidentRepository.FetchIncidentsWithSeverityTatBetweenGivenRangeMock.Return(nil, errors.New("error"))
+	_, err := suite.incidentService.FetchIncidentsApproachingSlaBreach()
+	suite.Error(err, "service should return error")
+}
+
+func (suite *IncidentServiceSuite) Test_FetchIncidentsApproachingSlaBreach_Success() {
+	incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
+	suite.incidentRepository.FetchIncidentsWithSeverityTatBetweenGivenRangeMock.Return(&incidents, nil)
+	_, err := suite.incidentService.FetchIncidentsApproachingSlaBreach()
+	suite.NoError(err, "service should not return error")
+	suite.Len(incidents, 2, "service should return 2 incidents")
+}

service/incident/incident_service_v2_interface.go

@@ -39,4 +39,5 @@ type IIncidentService interface {
 		userID,
 		requestType string,
 	) error
+	FetchIncidentsApproachingSlaBreach() ([]incident.IncidentDTO, error)
 }

service/reminder/reminder_service.go

@@ -2,22 +2,26 @@ package reminder
 
 import (
 	"houston/service/incident"
+	"houston/service/severity"
 	slackService "houston/service/slack"
 	"houston/service/teamService"
 )
 
 type ReminderService interface {
 	PostTeamIncidents() error
+	PostSlaBreachMessages() error
 }
 
 func NewReminderService(
 	incidentService incident.IIncidentService,
 	teamService teamService.ITeamServiceV2,
 	slackService slackService.ISlackService,
+	severityService severity.ISeverityService,
 ) ReminderService {
 	return &reminderServiceImpl{
 		incidentService: incidentService,
 		teamService:     teamService,
 		slackService:    slackService,
+		severityService: severityService,
 	}
 }

service/reminder/reminder_service_impl.go

@@ -2,6 +2,7 @@ package reminder
 
 import (
 	"houston/service/incident"
+	"houston/service/severity"
 	"houston/service/slack"
 	"houston/service/teamService"
 )
@@ -10,4 +11,5 @@ type reminderServiceImpl struct {
 	incidentService incident.IIncidentService
 	teamService     teamService.ITeamServiceV2
 	slackService    slack.ISlackService
+	severityService severity.ISeverityService
 }

service/reminder/reminder_service_test.go

@@ -13,6 +13,7 @@ type ReminderServiceSuite struct {
 	incidentService mocks.IIncidentServiceMock
 	teamService     mocks.ITeamServiceV2Mock
 	slackService    mocks.ISlackServiceMock
+	severityService mocks.ISeverityServiceMock
 }
 
 func (suite *ReminderServiceSuite) SetupTest() {
@@ -20,7 +21,8 @@ func (suite *ReminderServiceSuite) SetupTest() {
 	suite.incidentService = *mocks.NewIIncidentServiceMock(suite.T())
 	suite.teamService = *mocks.NewITeamServiceV2Mock(suite.T())
 	suite.slackService = *mocks.NewISlackServiceMock(suite.T())
-	suite.reminderService = NewReminderService(&suite.incidentService, &suite.teamService, &suite.slackService)
+	suite.severityService = *mocks.NewISeverityServiceMock(suite.T())
+	suite.reminderService = NewReminderService(&suite.incidentService, &suite.teamService, &suite.slackService, &suite.severityService)
 }
 
 func TestReminderService(t *testing.T) {

service/reminder/sla_breach_reminder.go

@@ -0,0 +1,98 @@
+package reminder
+
+import (
+	"fmt"
+	"github.com/slack-go/slack"
+	"houston/common"
+	"houston/common/util"
+	"houston/logger"
+	"houston/model/incident"
+	"houston/model/severity"
+	"sync"
+	"time"
+)
+
+const slaBreachLogTag = "SLA_BREACH_REMINDER_LOG_TAG"
+
+func (service *reminderServiceImpl) PostSlaBreachMessages() error {
+	logger.Info(fmt.Sprintf("%s received request to post SLA breach message", slaBreachLogTag))
+
+	severityEscalalationMap, err := service.severityService.GetSeverityEscalationMap()
+	if err != nil {
+		logger.Error(fmt.Sprintf("%s failed to get severity escalation map", slaBreachLogTag))
+		return err
+	}
+
+	incidents, err := service.incidentService.FetchIncidentsApproachingSlaBreach()
+	if err != nil {
+		logger.Error(fmt.Sprintf("%s failed to fetch incidents approaching SLA breach", slaBreachLogTag))
+		return err
+	}
+
+	if len(incidents) == 0 {
+		logger.Info(fmt.Sprintf("%s no incidents approaching SLA breach", slaBreachLogTag))
+		return nil
+	}
+
+	return service.postSlaBreachMessageForIncidents(incidents, severityEscalalationMap)
+}
+
+func (service *reminderServiceImpl) postSlaBreachMessageForIncidents(incidents []incident.IncidentDTO, severityEscalationMap map[uint]*severity.SeverityDTO) error {
+	logger.Info(fmt.Sprintf("%s found %d incidents approaching SLA breach", slaBreachLogTag, len(incidents)))
+
+	var errors common.ThreadSafeErrors
+	var wg sync.WaitGroup
+
+	for _, incidentData := range incidents {
+		fromSeverityName := incidentData.Severity.Name
+		toSeverityName := severityEscalationMap[incidentData.Severity.ID].Name
+		daysToEscalation := util.CalculateDifferenceInDays(incidentData.SeverityTat, time.Now())
+		incidentSlackChannel, teamSlackChannel := incidentData.SlackChannel, incidentData.Team.WebhookSlackChannel
+
+		wg.Add(1)
+		go util.ExecuteConcurrentAction(&wg, func() {
+			err := service.postSlaBreachMessageToChannel(
+				fmt.Sprintf("This incident will be auto-escalated to `%v` in `%v day(s)`", toSeverityName, daysToEscalation),
+				incidentSlackChannel,
+			)
+			if err != nil {
+				errors.AddErrors(err)
+			}
+		})
+
+		if !util.IsBlank(teamSlackChannel) {
+			wg.Add(1)
+			go util.ExecuteConcurrentAction(&wg, func() {
+				err := service.postSlaBreachMessageToChannel(
+					fmt.Sprintf(
+						"<#%s> (`%v`) will be auto-escalated to `%v` in `%v day(s)`",
+						incidentSlackChannel, fromSeverityName, toSeverityName, daysToEscalation,
+					),
+					teamSlackChannel,
+				)
+				if err != nil {
+					errors.AddErrors(err)
+				}
+			})
+		}
+	}
+	wg.Wait()
+
+	return errors.CollectErrors()
+}
+
+func (service *reminderServiceImpl) postSlaBreachMessageToChannel(
+	message,
+	channelId string,
+) error {
+	_, err := service.slackService.PostMessageOption(
+		channelId,
+		slack.MsgOptionText(message, false),
+	)
+
+	if err != nil {
+		logger.Error(fmt.Sprintf("%s failed to post SLA breach message in incident channel", slaBreachLogTag))
+	}
+
+	return err
+}

service/reminder/sla_breach_reminder_test.go

@@ -0,0 +1,43 @@
+package reminder
+
+import (
+	"errors"
+	"houston/common/util"
+	"houston/model/incident"
+)
+
+func (suite *ReminderServiceSuite) Test_PostSlaBreachMessages_GetSeverityEscalationMapError() {
+	suite.severityService.GetSeverityEscalationMapMock.Return(nil, errors.New("error"))
+	err := suite.reminderService.PostSlaBreachMessages()
+	suite.Error(err, "service should return error")
+}
+
+func (suite *ReminderServiceSuite) Test_PostSlaBreachMessages_IncidentFetchingError() {
+	suite.severityService.GetSeverityEscalationMapMock.Return(util.GetMockSeverityEscalationMap(), nil)
+	suite.incidentService.FetchIncidentsApproachingSlaBreachMock.Return(nil, errors.New("error"))
+	err := suite.reminderService.PostSlaBreachMessages()
+	suite.Error(err, "service should return error")
+}
+
+func (suite *ReminderServiceSuite) Test_PostSlaBreachMessages_NoIncidents() {
+	suite.severityService.GetSeverityEscalationMapMock.Return(util.GetMockSeverityEscalationMap(), nil)
+	suite.incidentService.FetchIncidentsApproachingSlaBreachMock.Return([]incident.IncidentDTO{}, nil)
+	err := suite.reminderService.PostSlaBreachMessages()
+	suite.NoError(err, "service should not return error")
+}
+
+func (suite *ReminderServiceSuite) Test_PostSlaBreachMessages_SlackFailure() {
+	suite.severityService.GetSeverityEscalationMapMock.Return(util.GetMockSeverityEscalationMap(), nil)
+	suite.incidentService.FetchIncidentsApproachingSlaBreachMock.Return(getMockIncidentsData(), nil)
+	suite.slackService.PostMessageOptionMock.Return("", errors.New("error"))
+	err := suite.reminderService.PostSlaBreachMessages()
+	suite.Error(err, "service should return error")
+}
+
+func (suite *ReminderServiceSuite) Test_PostSlaBreachMessages_Success() {
+	suite.severityService.GetSeverityEscalationMapMock.Return(util.GetMockSeverityEscalationMap(), nil)
+	suite.incidentService.FetchIncidentsApproachingSlaBreachMock.Return(getMockIncidentsData(), nil)
+	suite.slackService.PostMessageOptionMock.Return("", nil)
+	err := suite.reminderService.PostSlaBreachMessages()
+	suite.NoError(err, "service should not return error")
+}

service/reminder/team_incidents_reminder_test.go

@@ -51,23 +51,23 @@ func getMockIncidentsData() []incidentModel.IncidentDTO {
 	return []incidentModel.IncidentDTO{
 		{
 			ID:       1,
-			Team:     team.TeamDTO{Name: "team1", OncallHandle: "oncall1", ManagerHandle: "manager1"},
-			Severity: severity.SeverityDTO{Priority: 1, Name: "severity1"},
+			Team:     team.TeamDTO{Name: "team1", OncallHandle: "oncall1", ManagerHandle: "manager1", WebhookSlackChannel: "channel"},
+			Severity: severity.SeverityDTO{ID: 2, Priority: 1, Name: "severity1"},
 		},
 		{
 			ID:       2,
 			Team:     team.TeamDTO{Name: "team1", OncallHandle: "oncall1", ManagerHandle: "manager1"},
-			Severity: severity.SeverityDTO{Priority: 2, Name: "severity2"},
+			Severity: severity.SeverityDTO{ID: 3, Priority: 2, Name: "severity2"},
 		},
 		{
 			ID:       3,
 			Team:     team.TeamDTO{Name: "team2"},
-			Severity: severity.SeverityDTO{Priority: 1, Name: "severity1"},
+			Severity: severity.SeverityDTO{ID: 2, Priority: 1, Name: "severity1"},
 		},
 		{
 			ID:       4,
 			Team:     team.TeamDTO{Name: "team2"},
-			Severity: severity.SeverityDTO{Priority: 3, Name: "severity3"},
+			Severity: severity.SeverityDTO{ID: 4, Priority: 3, Name: "severity3"},
 		},
 	}
 }

service/severity/impl/severity_service.go

@@ -28,7 +28,7 @@ func (service *SeverityService) GetAllActiveSeverities() ([]severity.SeverityDTO
 	return dtoConverter.SeverityEntitiesToDTOs(*severityEntities), err
 }
 
-func (service *SeverityService) GetSeverityEscalationMap() (map[uint]*uint, error) {
+func (service *SeverityService) GetSeverityEscalationMap() (map[uint]*severity.SeverityDTO, error) {
 	severities, err := service.GetAllActiveSeverities()
 	if err != nil {
 		return nil, err
@@ -49,25 +49,20 @@ func (service *SeverityService) FindSeverityById(severityId uint) (*severity.Sev
 	return &severityDTO, err
 }
 
-func (service *SeverityService) createSeverityEscalationMap(severities []severity.SeverityDTO) map[uint]*uint {
+func (service *SeverityService) createSeverityEscalationMap(severities []severity.SeverityDTO) map[uint]*severity.SeverityDTO {
 	sort.Slice(severities, func(i, j int) bool {
 		return severities[i].Priority > severities[j].Priority
 	})
 
-	escalationMap := make(map[uint]*uint)
+	escalationMap := make(map[uint]*severity.SeverityDTO)
 
 	for index, severity := range severities {
-		var nextSeverityID *uint
 		for nextIndex := index + 1; nextIndex < len(severities); nextIndex++ {
 			if severities[nextIndex].Priority < severity.Priority {
-				nextSeverityID = &severities[nextIndex].ID
+				escalationMap[severity.ID] = &severities[nextIndex]
 				break
 			}
 		}
-
-		if nextSeverityID != nil {
-			escalationMap[severity.ID] = nextSeverityID
-		}
 	}
 
 	return escalationMap

service/severity/severity_service_interface.go

@@ -4,6 +4,6 @@ import "houston/model/severity"
 
 type ISeverityService interface {
 	GetAllActiveSeverities() ([]severity.SeverityDTO, error)
-	GetSeverityEscalationMap() (map[uint]*uint, error)
+	GetSeverityEscalationMap() (map[uint]*severity.SeverityDTO, error)
 	FindSeverityById(severityId uint) (*severity.SeverityDTO, error)
 }