varnitgoyal95/janus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DE-3551 | segregating app events to different topic

Browse Source
This commit is contained in:
puru
2025-01-16 21:04:39 +05:30
parent abb3981b30
commit 2d5cdfaee7
5 changed files with 38 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
lib/RequestHandler.go
View File

@@ -78,26 +78,41 @@ func (wp *WorkerPool) processRequest(request RequestObject) {
source := getSource(result)
recordValue := messageBytes
topicFromSource := getTopicFromSource(source)
topics := getTopicsFromSource(source)
message := &sarama.ProducerMessage{
Topic: topicFromSource,
Value: sarama.ByteEncoder(recordValue),
}
// Send to all designated topics
for _, topic := range topics {
message := &sarama.ProducerMessage{
Topic: topic,
Value: sarama.ByteEncoder(recordValue),
}
metrics.RequestCounter.WithLabelValues(source).Inc()
metrics.EventProcessingTimeHist.WithLabelValues(topicFromSource, source).Observe(float64(utils.NanosToMillis(time.Now().UnixNano()) - eventProcessingStartTime))
metrics.RequestCounter.WithLabelValues(source).Inc()
metrics.EventProcessingTimeHist.WithLabelValues(topic, source).Observe(float64(utils.NanosToMillis(time.Now().UnixNano()) - eventProcessingStartTime))
if err := producer_module.WriteMessageToKafkaAsync(message, source); err != nil {
wp.logger.Error("Failed to write message to Kafka", zap.Error(err), zap.String("source", source))
if err := producer_module.WriteMessageToKafkaAsync(message, source); err != nil {
wp.logger.Error("Failed to write message to Kafka",
zap.Error(err),
zap.String("source", source),
zap.String("topic", topic))
}
}
}
func getTopicFromSource(source string) string {
if strings.ToUpper(source) == "LITMUS" {
return producer_module.GetLitmusClickstreamTopic()
func getTopicsFromSource(source string) []string {
switch strings.ToUpper(source) {
case "LITMUS":
return []string{producer_module.GetLitmusClickstreamTopic()}
case "SYNCTIMER", "SYNCEVENTTASK", "IOSPULSETIMER":
// TODO : remove default topic for events from app source once users
// have migrated their pipelines to new topic
return []string{
producer_module.GetAppClickstreamTopic(),
producer_module.GetDefaultClickstreamTopic(),
}
default:
return []string{producer_module.GetDefaultClickstreamTopic()}
}
return producer_module.GetDefaultClickstreamTopic()
}
func getSource(event map[string]interface{}) string {