diff --git a/cmd/app/handler/slack_handler.go b/cmd/app/handler/slack_handler.go index 4bf9a21..93542cc 100644 --- a/cmd/app/handler/slack_handler.go +++ b/cmd/app/handler/slack_handler.go @@ -1,9 +1,11 @@ package handler import ( + "houston/internal/cron" "houston/internal/processor" "houston/model/incident" "houston/model/severity" + "houston/model/shedlock" "houston/model/tag" "houston/model/team" "houston/pkg/slackbot" @@ -29,7 +31,11 @@ func NewSlackHandler(logger *zap.Logger, gormClient *gorm.DB, socketModeClient * incidentService := incident.NewIncidentRepository(logger, gormClient, severityService) tagService := tag.NewTagRepository(logger, gormClient) teamService := team.NewTeamRepository(logger, gormClient) + shedlockService := shedlock.NewShedlockRepository(logger, gormClient) slackbotClient := slackbot.NewSlackClient(logger, socketModeClient) + + cron.RunJob(socketModeClient, gormClient, logger, incidentService, severityService, teamService, shedlockService) + return &slackHandler{ logger: logger, socketModeClient: socketModeClient, diff --git a/cmd/app/server.go b/cmd/app/server.go index 6586f3b..0764f64 100644 --- a/cmd/app/server.go +++ b/cmd/app/server.go @@ -2,11 +2,12 @@ package app import ( "fmt" + "houston/cmd/app/handler" + "github.com/gin-gonic/gin" "github.com/spf13/viper" "go.uber.org/zap" "gorm.io/gorm" - "houston/cmd/app/handler" ) type Server struct { @@ -34,7 +35,6 @@ func (s *Server) Handler() { func (s *Server) houstonHandler() { houstonClient := NewHoustonClient(s.logger) houstonHandler := handler.NewSlackHandler(s.logger, s.db, houstonClient.socketModeClient) - //cron.RunJob(houstonClient.slackClient, s.db, s.logger) houstonHandler.HoustonConnect() } diff --git a/cmd/app/slack.go b/cmd/app/slack.go index a1187f6..39cb660 100644 --- a/cmd/app/slack.go +++ b/cmd/app/slack.go @@ -1,10 +1,11 @@ package app import ( - "github.com/spf13/viper" "os" "strings" + "github.com/spf13/viper" + "github.com/slack-go/slack" "github.com/slack-go/slack/socketmode" "go.uber.org/zap" @@ -25,7 +26,7 @@ func NewHoustonClient(logger *zap.Logger) *HoustonSlack { } func slackConnect(logger *zap.Logger) *socketmode.Client { - appToken := viper.GetString("HOUSTON_SLACK_APP_TOKEN") + appToken := viper.GetString("houston.slack.app.token") if appToken == "" { logger.Error("HOUSTON_SLACK_APP_TOKEN must be set.") os.Exit(1) @@ -36,7 +37,7 @@ func slackConnect(logger *zap.Logger) *socketmode.Client { os.Exit(1) } - botToken := viper.GetString("HOUSTON_SLACK_BOT_TOKEN") + botToken := viper.GetString("houston.slack.bot.token") if botToken == "" { logger.Error("HOUSTON_SLACK_BOT_TOKEN must be set.") os.Exit(1) diff --git a/cmd/main.go b/cmd/main.go index 5ea016a..d75727e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,13 +1,14 @@ package main import ( - "github.com/spf13/viper" "houston/cmd/app" "houston/config" "houston/pkg/postgres" "os" "time" + "github.com/spf13/viper" + ginzap "github.com/gin-contrib/zap" "github.com/gin-gonic/gin" "github.com/joho/godotenv" @@ -30,7 +31,9 @@ func main() { r.Use(ginzap.Ginzap(logger, time.RFC3339, true)) r.Use(ginzap.RecoveryWithZap(logger, true)) - db := postgres.NewGormClient(viper.GetString("POSTGRES_DSN"), logger) + db := postgres.NewGormClient(viper.GetString("postgres.dsn"), viper.GetString("postgres.connection.max.idle.time"), + viper.GetString("postgres.connection.max.lifetime"), viper.GetInt("postgres.connections.max.idle"), + viper.GetInt("postgres.connections.max.open"), logger) sv := app.NewServer(r, logger, db) sv.Handler() diff --git a/config/application.properties b/config/application.properties index 52d97d1..09c46b8 100644 --- a/config/application.properties +++ b/config/application.properties @@ -1,7 +1,25 @@ -HOUSTON_SLACK_APP_TOKEN=xapp-1-A04TBQ7PGSJ-4960174100544-3c648a093c830a718bd81aff36cf0f433633312e16a0a6e11408bf5063a4785d -HOUSTON_SLACK_BOT_TOKEN=token -ENVIRONMENT=local -SHOW_INCIDENTS_LIMIT=10 -PORT=8080 -METRIC_PORT=9090 -POSTGRES_DSN=postgresql://postgres:admin@localhost:5432/houston +env=ENV +port=PORT +metric.port=METRIC_POST + +#houston token config +houston.slack.app.token=HOUSTON_SLACK_APP_TOKEN +houston.slack.bot.token=HOUSTON_SLACK_BOT_TOKEN + +#Postgres config +postgres.connection.max.idle.time=POSTGRES_CONNECTION_MAX_IDLE_TIME +postgres.connection.max.lifetime=POSTGRES_CONNECTION_MAX_LIFETIME +postgres.connections.max.idle=POSTGRES_CONNECTION_MAX_IDLE +postgres.connections.max.open=POSTGRES_CONNECTION_MAX_OPEN +postgres.dsn=POSTGRES_DSN + +#cron job config +cron.job.update.incident.interval=CRON_JOB_UPDATE_INCIDENT_INTERVAL +cron.job.lock.default.time.in.sec=CRON_JOB_LOCK_DEFAULT_TIME_IN_SEC +cron.job.lock.ticker.time.in.sec=CRON_JOB_LOCK_TICKER_TIME_IN_SEC +cron.job.name=CRON_JOB_NAME + +#incidents +incidents.show.limit=INCIDENTS_SHOW_LIMIT + + diff --git a/config/config.go b/config/config.go index 6a593b4..0ac598c 100644 --- a/config/config.go +++ b/config/config.go @@ -1,13 +1,16 @@ package config import ( + "os" + "strings" + "github.com/spf13/viper" "go.uber.org/zap" - "os" ) func LoadHoustonConfig(logger *zap.Logger) { viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.SetConfigName("houston-config") viper.SetConfigType("properties") viper.SetConfigFile("./config/application.properties") diff --git a/config/local.env b/config/local.env new file mode 100644 index 0000000..0e7c7bc --- /dev/null +++ b/config/local.env @@ -0,0 +1,24 @@ +ENV=local +PORT=8080 +METRIC_POST=9090 + +#houston token config +HOUSTON_SLACK_APP_TOKEN=xapp-1-A04TBQ7PGSJ-4960174100544-3c648a093c830a718bd81aff36cf0f433633312e16a0a6e11408bf5063a4785d +HOUSTON_SLACK_BOT_TOKEN=token + +#Postgres config +POSTGRES_CONNECTION_MAX_IDLE_TIME=10s +POSTGRES_CONNECTION_MAX_LIFETIME=10s +POSTGRES_CONNECTION_MAX_IDLE=1 +POSTGRES_CONNECTION_MAX_OPEN=2 +POSTGRES_DSN=postgresql://postgres:admin@localhost:5432/houston + +#cron job config +CRON_JOB_UPDATE_INCIDENT_INTERVAL='0 * * * * *' +CRON_JOB_LOCK_DEFAULT_TIME_IN_SEC=20 +CRON_JOB_LOCK_TICKER_TIME_IN_SEC=3000 +CRON_JOB_NAME=cron + +#incidents +INCIDENTS_SHOW_LIMIT=10 + diff --git a/db/migration/000001_init_schema.down.sql b/db/migration/000001_init_schema.down.sql index 5546438..07b0b01 100644 --- a/db/migration/000001_init_schema.down.sql +++ b/db/migration/000001_init_schema.down.sql @@ -20,4 +20,6 @@ drop table incident_role; drop table incident_channel; -drop table incident_tag; \ No newline at end of file +drop table incident_tag; + +drop table shedlock; \ No newline at end of file diff --git a/db/migration/000001_init_schema.up.sql b/db/migration/000001_init_schema.up.sql index 101b089..99e17f4 100644 --- a/db/migration/000001_init_schema.up.sql +++ b/db/migration/000001_init_schema.up.sql @@ -29,6 +29,7 @@ CREATE TABLE team name varchar(50) unique not null, slack_user_ids varchar[] default '{}', active boolean DEFAULT false, + confluence_link text, version bigint default 0, created_at timestamp without time zone, updated_at timestamp without time zone, @@ -145,4 +146,15 @@ create table incident_tag created_at timestamp without time zone, updated_at timestamp without time zone, deleted_at timestamp without time zone +); + +CREATE TABLE shedlock +( + name VARCHAR(64), + lock_until timestamp without time zone, + locked_at timestamp without time zone, + locked_by VARCHAR(255), + unlocked_by VARCHAR(64), + locked_value boolean DEFAULT true, + PRIMARY KEY (name) ); \ No newline at end of file diff --git a/go.mod b/go.mod index f3f7568..76702d5 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/robfig/cron v1.2.0 github.com/spf13/afero v1.9.4 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect diff --git a/internal/cron/cron.go b/internal/cron/cron.go index 9751265..df6b458 100644 --- a/internal/cron/cron.go +++ b/internal/cron/cron.go @@ -1,88 +1,119 @@ package cron -// -//import ( -// "fmt" -// "houston/internal/processor" -// "houston/pkg/postgres/query" -// "time" -// -// "github.com/robfig/cron" -// "github.com/slack-go/slack" -// "go.uber.org/zap" -// "gorm.io/gorm" -//) -// -//func RunJob(slackClient *slack.Client, db *gorm.DB, logger *zap.Logger) { -// c := cron.New() -// //RUN EVERY HOUR -// c.AddFunc("0 0 * * * *", func() { -// fmt.Println("Running job at", time.Now().Format(time.RFC3339)) -// -// severityUserMap := make(map[int][]string) -// severityData, err := query.FindSeverity(db, logger) -// for _, o := range severityData { -// userIdList, err := query.FindDefaultUserIdToBeAddedBySeverity(db, int(o.ID)) -// if err != nil { -// logger.Error("FindDefaultUserIdToBeAddedBySeverity error in cron job") -// return -// } -// severityUserMap[int(o.ID)] = userIdList -// } -// -// //FETCH INCIDENTS WHICH ARE NOT RESOLVED AND CURRENT TIMESTAMP>=SEVERITY TAT FIELD -// incidents, err := query.FindIncidentsBreachingSevTat(db) -// if err != nil { -// logger.Error("FindIncidentsBreachingSevTat error", -// zap.Error(err)) -// return -// } -// for i := 0; i < len(incidents); i++ { -// var currentSeverityId int = incidents[i].SeverityId -// var severityString string -// //CHECK IF SEVERITY IS ALREADY 0 OR NOT. SEV-0 is saved as id = 1 in db -// if currentSeverityId-1 > 0 { -// incidents[i].SeverityId = currentSeverityId - 1 -// severityEntity, err := query.FindSeverityById(db, incidents[i].SeverityId) -// if err != nil { -// logger.Error("failed to fetch FindSeverityByIdin cron job", -// zap.Int("severityId", incidents[i].SeverityId), zap.Error(err)) -// } -// incidents[i].SeverityTat = time.Now().AddDate(0, 0, severityEntity.Sla) -// severityString = fmt.Sprintln(severityEntity.Name + " (" + severityEntity.Description + ")") -// } -// -// incidents[i].UpdatedAt = time.Now() -// err = query.UpdateIncident(db, &incidents[i]) -// if err != nil { -// logger.Error("failed to update incident in cron job", -// zap.String("channel", incidents[i].SlackChannel), zap.Error(err)) -// } -// -// //DEFAULT USER ADDITION -// for _, o := range severityUserMap[incidents[i].SeverityId] { -// //throws error if the customer is already present in channel -// _, err := slackClient.InviteUsersToConversation(incidents[i].SlackChannel, o) -// if err != nil { -// logger.Error("Slack Client InviteUsersToConversation error in cron job") -// return -// } -// } -// -// //UPDATING MESSAGE -// processor.UpdateMessage(db, &incidents[i], nil, slackClient) -// -// //SENDING MESSAGE IN THE CHANNEL -// if currentSeverityId > 1 { -// msgOption := slack.MsgOptionText(fmt.Sprintf("Issue has been escalated to "+severityString+" as there was TAT breach"), true) -// _, _, errMessage := slackClient.PostMessage(incidents[i].SlackChannel, msgOption) -// if errMessage != nil { -// logger.Error("PostMessage failed for cronJob ", zap.Error(errMessage), zap.Int("incidentId", int(incidents[i].ID))) -// return -// } -// } -// } -// -// }) -// c.Start() -//} +import ( + "fmt" + "strconv" + "time" + + "houston/internal/processor/action" + "houston/model/incident" + "houston/model/severity" + "houston/model/shedlock" + "houston/model/team" + + "github.com/slack-go/slack" + "github.com/slack-go/slack/socketmode" + "github.com/spf13/viper" + "go.uber.org/zap" + "gorm.io/gorm" +) + +func RunJob(socketModeClient *socketmode.Client, db *gorm.DB, logger *zap.Logger, incidentService *incident.Repository, severityService *severity.Repository, teamService *team.Repository, shedlockService *shedlock.Repository) { + + shedlockConfig := NewLockerDbWithLockTime(viper.GetInt("cron.job.lock.default.time.in.sec")) + + err := shedlockConfig.AddFun(viper.GetString("cron.job.name"), viper.GetString("cron.job.update.incident.interval"), shedlockService, func() { + UpdateIncidentByCronJob(socketModeClient, db, logger, incidentService, teamService, severityService, viper.GetString("cron.job.name")) + }) + if err != nil { + logger.Error(err.Error()) + } + shedlockConfig.Start() + +} + +func UpdateIncidentByCronJob(socketModeClient *socketmode.Client, db *gorm.DB, logger *zap.Logger, incidentService *incident.Repository, teamService *team.Repository, severityService *severity.Repository, name string) { + + fmt.Println("Running job at", time.Now().Format(time.RFC3339)) + defer func() { + if r := recover(); r != nil { + logger.Error(fmt.Sprintf("Exception occurred in cron: %v", r.(error))) + } + }() + //Do not need to fetch Severity every time, hence keeping a map outside. + incidentSeverityList, err := severityService.GetAllActiveSeverity() + if err != nil || incidentSeverityList == nil { + logger.Error("GetAllActiveSeverity error in cron Job", + zap.String("cron_name", name), zap.Error(err)) + return + } + + incidentStatusEntity, _ := incidentService.FindIncidentStatusByName(incident.Resolved) + //FETCH INCIDENTS WHICH ARE NOT RESOLVED AND CURRENT TIMESTAMP>=SEVERITY TAT FIELD + if incidentStatusEntity == nil || incidentStatusEntity.ID == 0 { + logger.Error("Error : RESOLVED Incident Status not found in cron job") + return + } + + incidents, err := incidentService.FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0(db, int(incidentStatusEntity.ID)) + if err != nil { + logger.Error("FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0 error", + zap.Error(err)) + return + } + + if len(incidents) == 0 { + logger.Info("No incidents found to be updated by cron job") + } + + //Looping through All incidents which are not resolved and sev tat is breaching + for i := 0; i < len(incidents); i++ { + logger.Info("TAT breached Not Resolved Incident found. Name: " + incidents[i].IncidentName + ". Severity: " + strconv.Itoa(int(incidents[i].SeverityId))) + updatingSevForEachInc(logger, incidents, i, incidentSeverityList, err, incidentService, socketModeClient, teamService, severityService) + } +} + +func updatingSevForEachInc(logger *zap.Logger, incidents []incident.IncidentEntity, i int, incidentSeverityList *[]severity.SeverityEntity, err error, incidentService *incident.Repository, socketModeClient *socketmode.Client, teamService *team.Repository, severityService *severity.Repository) { + + var currentSeverityId = incidents[i].SeverityId + var severityString string = "" + var usersToBeAdded = make([]string, 0) + + incidents[i].SeverityId = currentSeverityId - 1 + + //Find the severity object from list and update the tat of incident + for _, s := range *incidentSeverityList { + if s.ID == incidents[i].SeverityId { + incidents[i].SeverityTat = time.Now().AddDate(0, 0, s.Sla) + usersToBeAdded = s.SlackUserIds + severityString = fmt.Sprintln(s.Name + " (" + s.Description + ")") + } + } + + //Updating Incident + incidents[i].UpdatedAt = time.Now() + err = incidentService.UpdateIncident(&incidents[i]) + if err != nil { + logger.Error("failed to update incident in cron job", + zap.String("channel", incidents[i].SlackChannel), zap.Error(err)) + } + + //Default User Addition According To Sev Tat + for _, o := range usersToBeAdded { + //throws error if the customer is already present in channel + _, err := socketModeClient.Client.InviteUsersToConversation(incidents[i].SlackChannel, o) + if err != nil { + logger.Error("Slack Client InviteUsersToConversation error in cron job") + } + } + + //UPDATING MESSAGE + s := action.NewIncidentChannelMessageUpdateAction(socketModeClient, logger, incidentService, teamService, severityService) + s.ProcessAction(incidents[i].SlackChannel) + + msgOption := slack.MsgOptionText(fmt.Sprintf("hoston escalated incident to %s", severityString), false) + _, _, errMessage := socketModeClient.PostMessage(incidents[i].SlackChannel, msgOption) + if errMessage != nil { + logger.Error("PostMessage failed for cronJob ", zap.Error(errMessage), zap.Int("incidentId", int(incidents[i].ID))) + } +} diff --git a/internal/cron/lock.go b/internal/cron/lock.go new file mode 100644 index 0000000..06a0ab6 --- /dev/null +++ b/internal/cron/lock.go @@ -0,0 +1,13 @@ +package cron + +import ( + "os" +) + +func LocalHostName() string { + hostname, err := os.Hostname() + if err != nil { + return "" + } + return hostname +} diff --git a/internal/cron/shedlocker.go b/internal/cron/shedlocker.go new file mode 100644 index 0000000..50a86d7 --- /dev/null +++ b/internal/cron/shedlocker.go @@ -0,0 +1,72 @@ +package cron + +import ( + "fmt" + "houston/model/shedlock" + "time" + + "github.com/robfig/cron" + "github.com/spf13/viper" +) + +//https://blog.csdn.net/waltonhuang/article/details/106555195 +//https://www.baeldung.com/shedlock-spring + +// LockerDb +type LockerDb struct { + LockTime int `json:"lock_time" 20 sec` + c *cron.Cron +} + +func NewLockerDb() *LockerDb { + return &LockerDb{LockTime: 20, c: cron.New()} +} +func NewLockerDbWithLockTime(lockFor int) *LockerDb { + return &LockerDb{LockTime: lockFor, c: cron.New()} +} + +func (l LockerDb) AddFun(name string, spec string, shedlockService *shedlock.Repository, cmd func()) error { + if l.c == nil { + l.c = cron.New() + } + err := l.c.AddFunc(spec, func() { + if l.DoLock(name, shedlockService) { + defer shedlockService.Unlock(name, "") + cmd() + } + }) + return err +} + +func (l LockerDb) DoLock(name string, shedlockService *shedlock.Repository) bool { + s := shedlockService.Insert(name, LocalHostName(), l.LockTime) + if true == s { + ticker := time.NewTicker(viper.GetDuration("cron.job.lock.ticker.time.in.sec") * time.Second) + + go func() { + for { + select { + case t := <-ticker.C: + { + fmt.Println("Ticking at", t) + shedlockService.Unlock(name, "ticker") + ticker.Stop() + fmt.Println("Ticker stopped") + return + } + } + } + }() + + } + return s +} + +func (l LockerDb) Start() { + l.c.Start() +} + +func (l LockerDb) Stop() { + l.c.Stop() + l.c = nil +} diff --git a/internal/processor/action/incident_assign_action.go b/internal/processor/action/incident_assign_action.go index ee7cd3f..3ff36ff 100644 --- a/internal/processor/action/incident_assign_action.go +++ b/internal/processor/action/incident_assign_action.go @@ -63,7 +63,7 @@ func (iap *AssignIncidentAction) IncidentAssignModalCommandProcessing(callback s return } - msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> is assigned to %s by <@%s>", assignIncidentRoleRequest.UserId, assignIncidentRoleRequest.Role, assignIncidentRoleRequest.CreatedById), false) + msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> is assigned to *%s* by <@%s>", assignIncidentRoleRequest.UserId, assignIncidentRoleRequest.Role, assignIncidentRoleRequest.CreatedById), false) _, _, errMessage := iap.client.PostMessage(callback.View.PrivateMetadata, msgOption) if errMessage != nil { iap.logger.Error("post response failed for IncidentAssignModalCommandProcessing", zap.Error(errMessage)) diff --git a/internal/processor/action/incident_resolve_action.go b/internal/processor/action/incident_resolve_action.go index f8475b6..26f1f5b 100644 --- a/internal/processor/action/incident_resolve_action.go +++ b/internal/processor/action/incident_resolve_action.go @@ -51,7 +51,7 @@ func (irp *ResolveIncidentAction) IncidentResolveProcess(callback slack.Interact zap.String("channel", channelId), zap.String("user_id", callback.User.ID)) - msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> > set status to %s", callback.User.ID, + msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> *>* `houston set status to %s`", callback.User.ID, incident.Resolved), false) _, _, errMessage := irp.client.PostMessage(callback.Channel.ID, msgOption) if errMessage != nil { diff --git a/internal/processor/action/incident_update_description_action.go b/internal/processor/action/incident_update_description_action.go index 02d1371..621d7e0 100644 --- a/internal/processor/action/incident_update_description_action.go +++ b/internal/processor/action/incident_update_description_action.go @@ -74,7 +74,7 @@ func (itp *IncidentUpdateDescriptionAction) IncidentUpdateDescription(callback s zap.String("user_id", user.ID), zap.Error(err)) return } - msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> > set description to %s", user.ID, incidentEntity.Description), false) + msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> *>* `houston set description to %s`", user.ID, incidentEntity.Description), false) _, _, errMessage := itp.client.PostMessage(callback.View.PrivateMetadata, msgOption) if errMessage != nil { itp.logger.Error("post response failed for IncidentUpdateDescription", zap.Error(errMessage)) diff --git a/internal/processor/action/incident_update_severity_action.go b/internal/processor/action/incident_update_severity_action.go index 570ee17..9f02748 100644 --- a/internal/processor/action/incident_update_severity_action.go +++ b/internal/processor/action/incident_update_severity_action.go @@ -5,6 +5,7 @@ import ( "houston/internal/processor/action/view" "houston/model/incident" "houston/model/severity" + "houston/model/team" "houston/pkg/slackbot" "strconv" "time" @@ -19,15 +20,17 @@ type IncidentUpdateSevertityAction struct { logger *zap.Logger severityService *severity.Repository incidentService *incident.Repository + teamService *team.Repository slackbotClient *slackbot.Client } -func NewIncidentUpdateSeverityAction(client *socketmode.Client, logger *zap.Logger, incidentService *incident.Repository, severityService *severity.Repository, slackbotClient *slackbot.Client) *IncidentUpdateSevertityAction { +func NewIncidentUpdateSeverityAction(client *socketmode.Client, logger *zap.Logger, incidentService *incident.Repository, severityService *severity.Repository, teamService *team.Repository, slackbotClient *slackbot.Client) *IncidentUpdateSevertityAction { return &IncidentUpdateSevertityAction{ client: client, logger: logger, severityService: severityService, incidentService: incidentService, + teamService: teamService, slackbotClient: slackbotClient, } } @@ -94,12 +97,37 @@ func (isp *IncidentUpdateSevertityAction) IncidentUpdateSeverity(callback slack. isp.slackbotClient.InviteUsersToConversation(callback.View.PrivateMetadata, o) } - msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> > set severity to %s", user.ID, incidentSeverityEntity.Name), false) + msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> *>* `set severity to %s (%s)`", user.ID, incidentSeverityEntity.Name, incidentSeverityEntity.Description), false) _, _, errMessage := isp.client.PostMessage(callback.View.PrivateMetadata, msgOption) if errMessage != nil { isp.logger.Error("post response failed for IncidentUpdateSeverity", zap.Error(errMessage)) return } + + teamEntity, err := isp.teamService.FindTeamById(incidentEntity.TeamId) + if err != nil { + isp.logger.Error("FindTeamEntityById error", + zap.String("incident_slack_channel_id", channel.ID), zap.String("channel", channel.Name), + zap.String("user_id", user.ID), zap.Error(err)) + return + } else if teamEntity == nil { + isp.logger.Error("Team Not Found", + zap.String("incident_slack_channel_id", channel.ID), zap.String("channel", channel.Name), + zap.String("user_id", user.ID), zap.Error(err)) + return + } + + txt := fmt.Sprintf("set the channel topic: *%s · %s (%s) %s* | %s", teamEntity.Name, incidentSeverityEntity.Name, incidentSeverityEntity.Description, incidentEntity.IncidentName, incidentEntity.Title) + att := slack.Attachment{ + Text: txt, + Color: "#808080", // Grey color code + MarkdownIn: []string{"txt"}, // Define which fields support markdown + } + _, _, errMessage = isp.client.PostMessage(callback.View.PrivateMetadata, slack.MsgOptionAttachments(att)) + if errMessage != nil { + isp.logger.Error("post response failed for IncidentUpdateType", zap.Error(errMessage)) + return + } var payload interface{} isp.client.Ack(*request, payload) } diff --git a/internal/processor/action/incident_update_status_action.go b/internal/processor/action/incident_update_status_action.go index 03f2462..bf8ee57 100644 --- a/internal/processor/action/incident_update_status_action.go +++ b/internal/processor/action/incident_update_status_action.go @@ -86,7 +86,7 @@ func (isp *UpdateIncidentAction) IncidentUpdateStatus(callback slack.Interaction zap.String("incident_slack_channel_id", channel.ID), zap.String("channel", channel.Name), zap.String("user_id", user.ID), zap.Error(err)) } - msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> > set status to %s", user.ID, result.Name), false) + msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> *>* `houston set status to %s`", user.ID, result.Name), false) _, _, errMessage := isp.client.PostMessage(callback.View.PrivateMetadata, msgOption) if errMessage != nil { isp.logger.Error("post response failed for IncidentUpdateStatus", zap.Error(errMessage)) diff --git a/internal/processor/action/incident_update_title_action.go b/internal/processor/action/incident_update_title_action.go index 5259c1d..e200766 100644 --- a/internal/processor/action/incident_update_title_action.go +++ b/internal/processor/action/incident_update_title_action.go @@ -74,7 +74,7 @@ func (itp *IncidentUpdateTitleAction) IncidentUpdateTitle(callback slack.Interac zap.String("user_id", user.ID), zap.Error(err)) return } - msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> > set title to %s", user.ID, incidentEntity.Title), false) + msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> *>* `houston set title to %s`", user.ID, incidentEntity.Title), false) _, _, errMessage := itp.client.PostMessage(callback.View.PrivateMetadata, msgOption) if errMessage != nil { itp.logger.Error("post response failed for IncidentUpdateTitle", zap.Error(errMessage)) diff --git a/internal/processor/action/incident_update_type_action.go b/internal/processor/action/incident_update_type_action.go index 8f39dc6..aa28dc2 100644 --- a/internal/processor/action/incident_update_type_action.go +++ b/internal/processor/action/incident_update_type_action.go @@ -4,6 +4,7 @@ import ( "fmt" "houston/internal/processor/action/view" "houston/model/incident" + "houston/model/severity" "houston/model/team" "houston/pkg/slackbot" "strconv" @@ -19,15 +20,17 @@ type IncidentUpdateTypeAction struct { logger *zap.Logger teamService *team.Repository incidentService *incident.Repository + severityService *severity.Repository slackbotClient *slackbot.Client } -func NewIncidentUpdateTypeAction(client *socketmode.Client, logger *zap.Logger, incidentService *incident.Repository, teamService *team.Repository, slackbotClient *slackbot.Client) *IncidentUpdateTypeAction { +func NewIncidentUpdateTypeAction(client *socketmode.Client, logger *zap.Logger, incidentService *incident.Repository, teamService *team.Repository, severityService *severity.Repository, slackbotClient *slackbot.Client) *IncidentUpdateTypeAction { return &IncidentUpdateTypeAction{ socketModeClient: client, logger: logger, teamService: teamService, incidentService: incidentService, + severityService: severityService, slackbotClient: slackbotClient, } } @@ -91,12 +94,38 @@ func (itp *IncidentUpdateTypeAction) IncidentUpdateType(callback slack.Interacti itp.addDefaultUsersToIncident(callback.View.PrivateMetadata, incidentEntity.TeamId) - msgOption := slack.MsgOptionText(fmt.Sprintf("<@%s> > set Team to %s", user.ID, teamEntity.Name), false) - _, _, errMessage := itp.socketModeClient.PostMessage(callback.View.PrivateMetadata, msgOption) + severityEntity, err := itp.severityService.FindSeverityById(incidentEntity.SeverityId) + if err != nil { + itp.logger.Error("error in fetching severity in incident update type action", zap.String("channel", incidentEntity.SlackChannel), + zap.Uint("incident_id", incidentEntity.ID), zap.Error(err)) + return + } else if severityEntity == nil { + itp.logger.Info("severity not found in incident update type action", zap.String("channel", incidentEntity.SlackChannel), + zap.Uint("incident_id", incidentEntity.ID)) + return + } + + txt := fmt.Sprintf("<@%s> *>* set the channel topic: *%s · %s (%s) %s* | %s", user.ID, teamEntity.Name, severityEntity.Name, severityEntity.Description, incidentEntity.IncidentName, incidentEntity.Title) + att := slack.Attachment{ + Text: txt, + Color: "#808080", // Grey color code + MarkdownIn: []string{"text"}, // Define which fields support markdown + } + _, _, errMessage := itp.socketModeClient.PostMessage(callback.View.PrivateMetadata, slack.MsgOptionAttachments(att)) if errMessage != nil { itp.logger.Error("post response failed for IncidentUpdateType", zap.Error(errMessage)) return } + + if len(teamEntity.ConfluenceLink) > 1 { + textForConfluence := fmt.Sprintf("*%s* -> <%s| Confluence Page>", teamEntity.Name, teamEntity.ConfluenceLink) + _, _, errMessage := itp.socketModeClient.PostMessage(callback.View.PrivateMetadata, slack.MsgOptionText(textForConfluence, false)) + if errMessage != nil { + itp.logger.Error("post response failed for IncidentUpdateType Confluence message", zap.Error(errMessage)) + return + } + } + var payload interface{} itp.socketModeClient.Ack(*request, payload) } diff --git a/internal/processor/action/show_incidents_action.go b/internal/processor/action/show_incidents_action.go index e69bcbf..af9023b 100644 --- a/internal/processor/action/show_incidents_action.go +++ b/internal/processor/action/show_incidents_action.go @@ -26,7 +26,7 @@ func ShowIncidentsProcessor(client *socketmode.Client, logger *zap.Logger, incid func (sip *ShowIncidentsAction) ProcessAction(channel slack.Channel, user slack.User, triggerId string, request *socketmode.Request) { - limit := viper.GetInt("SHOW_INCIDENTS_LIMIT") + limit := viper.GetInt("incidents.show.limit") s, err := sip.incidentService.GetOpenIncidents(limit) if err != nil { sip.logger.Error("GetOpenIncidents query failed.", diff --git a/internal/processor/event_type_interactive_processor.go b/internal/processor/event_type_interactive_processor.go index a69b9ca..116fa09 100644 --- a/internal/processor/event_type_interactive_processor.go +++ b/internal/processor/event_type_interactive_processor.go @@ -46,8 +46,8 @@ func NewBlockActionProcessor(logger *zap.Logger, socketModeClient *socketmode.Cl assignIncidentAction: action.NewAssignIncidentAction(socketModeClient, logger, incidentService), incidentResolveAction: action.NewIncidentResolveProcessor(socketModeClient, logger, incidentService), incidentUpdateAction: action.NewIncidentUpdateAction(socketModeClient, logger, incidentService), - incidentUpdateTypeAction: action.NewIncidentUpdateTypeAction(socketModeClient, logger, incidentService, teamService, slackbotClient), - incidentUpdateSeverityAction: action.NewIncidentUpdateSeverityAction(socketModeClient, logger, incidentService, severityService, slackbotClient), + incidentUpdateTypeAction: action.NewIncidentUpdateTypeAction(socketModeClient, logger, incidentService, teamService, severityService, slackbotClient), + incidentUpdateSeverityAction: action.NewIncidentUpdateSeverityAction(socketModeClient, logger, incidentService, severityService, teamService, slackbotClient), incidentUpdateTitleAction: action.NewIncidentUpdateTitleAction(socketModeClient, logger, incidentService), incidentUpdateDescriptionAction: action.NewIncidentUpdateDescriptionAction(socketModeClient, logger, incidentService), incidentUpdateTagsAction: action.NewIncidentUpdateTagsAction(socketModeClient, logger, incidentService, teamService, tagService), @@ -179,8 +179,8 @@ func NewViewSubmissionProcessor(logger *zap.Logger, socketModeClient *socketmode updateIncidentAction: action.NewIncidentUpdateAction(socketModeClient, logger, incidentService), incidentUpdateTitleAction: action.NewIncidentUpdateTitleAction(socketModeClient, logger, incidentService), incidentUpdateDescriptionAction: action.NewIncidentUpdateDescriptionAction(socketModeClient, logger, incidentService), - incidentUpdateSeverityAction: action.NewIncidentUpdateSeverityAction(socketModeClient, logger, incidentService, severityService, slackbotClient), - incidentUpdateTypeAction: action.NewIncidentUpdateTypeAction(socketModeClient, logger, incidentService, teamService, slackbotClient), + incidentUpdateSeverityAction: action.NewIncidentUpdateSeverityAction(socketModeClient, logger, incidentService, severityService, teamService, slackbotClient), + incidentUpdateTypeAction: action.NewIncidentUpdateTypeAction(socketModeClient, logger, incidentService, teamService, severityService, slackbotClient), incidentUpdateTagsAction: action.NewIncidentUpdateTagsAction(socketModeClient, logger, incidentService, teamService, tagService), } } diff --git a/model/incident/entity.go b/model/incident/entity.go index cf7c714..7951305 100644 --- a/model/incident/entity.go +++ b/model/incident/entity.go @@ -11,11 +11,11 @@ import ( type IncidentStatus string const ( - Investigating IncidentStatus = "Investigating" - Identified = "Identified" - Monitoring = "Monitoring" - Resolved = "Resolved" - Duplicated = "Duplicated" + Investigating IncidentStatus = "INVESTIGATING" + Identified = "IDEWNTIFIED" + Monitoring = "MONITORING" + Resolved = "RESOLVED" + Duplicated = "DUPLICATED" ) const ( diff --git a/model/incident/incident.go b/model/incident/incident.go index 8810fbd..c281e9f 100644 --- a/model/incident/incident.go +++ b/model/incident/incident.go @@ -293,3 +293,25 @@ func (r *Repository) SaveIncidentTag(entity IncidentTagEntity) (*IncidentTagEnti } return &entity, nil } + +func (r *Repository) FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0(db *gorm.DB, resolvedIndex int) ([]IncidentEntity, error) { + var incidentEntity []IncidentEntity + + currentTime := time.Now() + + result := r.gormClient.Table("incident"). + Where("incident_status.name <> ? and incident_status.name <> ? and incident.severity_tat <= ? and severity.name <> ?", Resolved, Duplicated, currentTime, "Sev-0"). + Joins("JOIN severity ON severity.id = incident.severity_id"). + Joins("JOIN incident_status on incident.status = incident_status.id"). + Select("incident.*"). + Scan(&incidentEntity) + + if result.Error != nil { + return nil, result.Error + } + if result.RowsAffected == 0 { + return nil, nil + } + + return incidentEntity, nil +} diff --git a/model/shedlock/entity.go b/model/shedlock/entity.go new file mode 100644 index 0000000..80ad92b --- /dev/null +++ b/model/shedlock/entity.go @@ -0,0 +1,16 @@ +package shedlock + +import "time" + +type ShedlockEntity struct { + Name string `gorm:"name"` + LockUntil time.Time `gorm:"lock_until"` + LockedAt time.Time `gorm:"locked_at"` + LockedBy string `gorm:"locked_by" hostname+name` + LockedValue bool `gorm:"locked_value"` + UnlockedBy string `gorm:"unlocked_by"` +} + +func (ShedlockEntity) TableName() string { + return "shedlock" +} diff --git a/model/shedlock/shedlock.go b/model/shedlock/shedlock.go new file mode 100644 index 0000000..ef424b4 --- /dev/null +++ b/model/shedlock/shedlock.go @@ -0,0 +1,68 @@ +package shedlock + +import ( + "time" + + "go.uber.org/zap" + "gorm.io/gorm" +) + +type Repository struct { + logger *zap.Logger + gormClient *gorm.DB +} + +func NewShedlockRepository(logger *zap.Logger, gormClient *gorm.DB) *Repository { + return &Repository{ + logger: logger, + gormClient: gormClient, + } +} + +func (repo *Repository) Insert(name string, lockedBy string, lockTime int) bool { + var shedlockEntity ShedlockEntity + s := &ShedlockEntity{ + Name: name, + LockUntil: time.Now().Add(time.Duration(lockTime) * time.Second), + LockedAt: time.Now(), + LockedBy: lockedBy, + LockedValue: true, + } + + result := repo.gormClient.Table("shedlock").Where("name=?", name).Where("locked_value=?", false).Delete(&shedlockEntity) + + create := repo.gormClient.Create(&s) + if create.Error == nil && create.RowsAffected > 0 { + return true + } + + if result.Error != nil { + repo.logger.Error("Error While locking: ", zap.Error(result.Error)) + } + return false +} + +func (repo *Repository) Unlock(name string, unlockedBy string) bool { + now := time.Now() + + updatedData := map[string]interface{}{ + "locked_value": false, + "lock_until": now, + "unlocked_by": unlockedBy, + } + + result := repo.gormClient.Table("shedlock").Where("name=?", name).Where("locked_value=?", true).UpdateColumns(updatedData) + + if result.Error != nil { + repo.logger.Error("Failed to unlock ShedLock", zap.Error(result.Error)) + + return false + } + + // Check if any rows were affected + if result.RowsAffected == 0 { + repo.logger.Info("No ShedLock found with name: " + name + " for unlocking by: " + unlockedBy) + return false + } + return true +} diff --git a/model/team/entity.go b/model/team/entity.go index 14cef70..db89b38 100644 --- a/model/team/entity.go +++ b/model/team/entity.go @@ -7,9 +7,10 @@ import ( type TeamEntity struct { gorm.Model - Name string `gorm:"column:name"` - SlackUserIds pq.StringArray `gorm:"column:slack_user_ids;type:string[]"` - Active bool `gorm:"column:active"` + Name string `gorm:"column:name"` + SlackUserIds pq.StringArray `gorm:"column:slack_user_ids;type:string[]"` + ConfluenceLink string `gorm:"column:confluence_link"` + Active bool `gorm:"column:active"` } func (TeamEntity) TableName() string { diff --git a/pkg/postgres/config.go b/pkg/postgres/config.go index 66b083e..f4f7214 100644 --- a/pkg/postgres/config.go +++ b/pkg/postgres/config.go @@ -2,6 +2,7 @@ package postgres import ( "os" + "time" "go.uber.org/zap" "gorm.io/driver/postgres" @@ -13,26 +14,37 @@ type Client struct { gormClient *gorm.DB } -func NewGormClient(dsn string, logger *zap.Logger) *gorm.DB { - // todo: set the connection configs +func NewGormClient(dsn string, connMaxIdleTime string, connMaxLifetime string, setMaxIdleConns int, setMaxOpenConns int, logger *zap.Logger) *gorm.DB { db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { logger.Error("database connection failed", zap.Error(err)) os.Exit(1) } + sqlDB, err := db.DB() + if err != nil { + logger.Fatal(err.Error()) + } + + connMaxIdleDuration, err := time.ParseDuration(connMaxIdleTime) + if err != nil { + logger.Fatal(err.Error()) + } + + connMaxLifetimeDuration, err := time.ParseDuration(connMaxLifetime) + if err != nil { + logger.Fatal(err.Error()) + } + + sqlDB.SetConnMaxIdleTime(time.Duration(connMaxIdleDuration.Seconds())) + sqlDB.SetConnMaxLifetime(time.Duration(connMaxLifetimeDuration.Seconds())) + sqlDB.SetMaxIdleConns(setMaxIdleConns) + sqlDB.SetMaxOpenConns(setMaxOpenConns) + + err = sqlDB.Ping() + if err != nil { + logger.Fatal(err.Error()) + } logger.Info("database connection established") return db } - -func PQConnection(logger *zap.Logger) *gorm.DB { - dsn := "" - db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) - if err != nil { - logger.Error("database connection failed", zap.Error(err)) - os.Exit(1) - } - - logger.Info("database connection successful") - return db -} diff --git a/schema.sql b/schema.sql deleted file mode 100644 index 3a72e13..0000000 --- a/schema.sql +++ /dev/null @@ -1,186 +0,0 @@ --- CREATE TABLE tenant ( --- id SERIAL PRIMARY KEY, --- name text, --- key text, --- url text, --- vertical text, --- version bigint default 0, --- created_at timestamp without time zone, --- updated_at timestamp without time zone, --- deleted_at timestamp without time zone --- ); --- --- CREATE TABLE grafana ( --- id SERIAL PRIMARY KEY, --- name text, --- url text, --- tenant_id bigint, --- status boolean DEFAULT false, --- alert_id bigint, --- version bigint default 0, --- created_at timestamp without time zone, --- updated_at timestamp without time zone, --- deleted_at timestamp without time zone --- ); --- --- CREATE TABLE alerts ( --- id SERIAL PRIMARY KEY, --- name text, --- team_name text, --- service_name text, --- status boolean DEFAULT false, --- version bigint default 0, --- created_at timestamp without time zone, --- updated_at timestamp without time zone, --- deleted_at timestamp without time zone --- ); - - -CREATE TABLE incident ( - id SERIAL PRIMARY KEY, - title text, - description text, - status integer not null, - severity_id integer not null, - incident_name text, - slack_channel varchar(100), - detection_time timestamp without time zone, - start_time timestamp without time zone, - end_time timestamp without time zone, - team_id int not null, - jira_id varchar(100), - confluence_id varchar(100), - created_by varchar(100), - updated_by varchar(100), - severity_tat timestamp without time zone, - remind_me_at timestamp without time zone, - enable_reminder boolean DEFAULT false, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -CREATE TABLE team ( - id SERIAL PRIMARY KEY, - name varchar(50) unique not null, - slack_user_ids varchar[] default '{}', - active boolean DEFAULT false, - version bigint default 0, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -create table team_tag ( - id serial primary key, - team_id int not null, - tag_id int not null, - optional boolean default false, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -CREATE TABLE houston_user ( - id SERIAL PRIMARY KEY, - name varchar(50), - slack_user_id varchar(100), - active boolean DEFAULT true -); - -CREATE TABLE severity ( - id SERIAL PRIMARY KEY, - name varchar(50), - description text, - version bigint default 0, - sla int, - slack_user_ids varchar[] default '{}', - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -CREATE TABLE tag ( - id SERIAL PRIMARY KEY, - name varchar not null, - label text not null, - place_holder text, - action_id varchar(100) not null, - type varchar(100) not null, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -create table tag_value ( - id serial primary key, - tag_id int not null, - value varchar not null, - create_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -CREATE TABLE incident_status ( - id SERIAL PRIMARY KEY, - name varchar(50), - description text, - is_terminal_status boolean default false, - version bigint default 0, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -create table role ( - id serial primary key, - name varchar(100) not null, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -CREATE TABLE incident_role ( - id SERIAL PRIMARY KEY, - incident_id integer not null, - role varchar(100), - assigned_to varchar(100), - assigned_by varchar(100), - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -CREATE TABLE incident_channel ( - id SERIAL PRIMARY KEY, - slack_channel varchar(100), - incident_id int not null, - message_timestamp varchar(100), - version bigint default 0, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - -create table incident_tag ( - id serial primary key, - incident_id int not null, - tag_id int not null, - tag_value_ids int[] default '{}', - free_text_value text, - created_at timestamp without time zone, - updated_at timestamp without time zone, - deleted_at timestamp without time zone -); - --- CREATE TABLE audit ( --- id SERIAL PRIMARY KEY, --- incident_id bigint, --- event text, --- user_name varchar(100), --- user_id varchar(100), --- version bigint default 0, --- created_at timestamp without time zone, --- updated_at timestamp without time zone, --- deleted_at timestamp without time zone --- ); \ No newline at end of file