package config import ( "alfred/pkg/log" "encoding/json" "os" "strings" "time" "github.com/spf13/viper" "go.uber.org/zap" ) type baseConfig struct { Env string Port int MetricPort int GinMode string } type IngesterConfig struct { BaseConfig baseConfig KafkaConfig IngesterKafkaConfig ElasticSearchConfig IngesterElasticSearchConfig S3Config IngesterS3Config HttpConfig HttpConfig CorsConfig CorsConfig OutboundServiceConfig IngestorOutboundServiceConfig ApiKeyConfig map[string]string DefaultFactoryWebClientList []string DefaultFactoryWebClientListWithCookieData []string WhiteListedTags WhiteListedTagsConfig WebWhitelistedClientsConfig []string CruiseCacheTtl time.Duration RateLimiterConfig IngesterRateLimitConfig FpsThresholdScoreMap map[string]interface{} FpsWeightMapNavi map[string]interface{} FpsDisableFeatureMap map[string]interface{} FpsBatteryMapNavi map[string]interface{} FpsMemoryMapNavi map[string]interface{} FpsWeightMapCosmos map[string]interface{} FpsBatteryMapCosmos map[string]interface{} FpsMemoryMapCosmos map[string]interface{} CruiseLogsDeviceIds map[string]interface{} PastTimestampValidationDiffInHours time.Duration FutureTimestampValidationDiffInHours time.Duration WebSessionDataSliceSize int IsWebSessionDataSliceEnabledClientMap map[string]string TracerConfig TracerConfig } type VideoProcessingConfig struct { MaxConcurrentJobs int `json:"max_concurrent_jobs"` DefaultTimeoutSeconds int `json:"default_timeout_seconds"` } type CoreConfig struct { BaseConfig baseConfig AuthEnabled bool ByPassAuthSources []string ScreenTagFilters map[string][]string VideoProcessingConfig VideoProcessingConfig ElasticSearchConfig CoreElasticSearchConfig OutboundServiceConfig CoreOutboundServiceConfig FfmpegConfig FfmpegConfig HttpConfig HttpConfig CorsConfig CorsConfig S3Config CoreS3Config CruiseDropdowns string ZipUploadTimeConfig float64 ApiKeyClientMap map[string]string DefaultSessionTime time.Duration MaskingConfig MaskingConfig DeviceMonitoringConfig CoreDeviceMonitoringConfig ClientsWithoutDurationResponse []string MinAppVersionForScreenNameFilter string ZipsWithEventsAppVersions string MaxFetchWebVideoGofuncConcurrency int64 MaxBlurringVideoGofuncConcurrency int64 SessionErrorEventsFilter []string TouchPointsConfig TouchPointsConfig ShedlockConfig CoreShedlockConfig MaxUpdateSessionErrorEventsGofuncConcurrency int64 ClientAppOsMap map[string]string ClientProjectNameMap map[string]string AuthKeyServiceMap map[string]string } type CollectorConfig struct { BaseConfig baseConfig KafkaConfig CollectorKafkaConfig ElasticSearchConfig CollectorElasticSearchConfig S3Config CollectorS3Config CorsConfig CorsConfig CacheTimeForFragmentsIngestion time.Duration MaxRetry int InitialDelayInSeconds time.Duration ErrorEventsSlackTemplateId string SessionErrorEventsFilter []string IngestErrorEventsFilter []string PastTimestampValidationDiffInHours time.Duration FutureTimestampValidationDiffInHours time.Duration IgnoredEventTypes []string EventListenerGoroutineGroupLimit int WebSessionUploadListenerGoroutineGroupLimit int SessionUploadListenerGoroutineGroupLimit int } var ingesterConfig IngesterConfig var coreConfig CoreConfig var collectorConfig CollectorConfig func GetIngesterConfig() IngesterConfig { return ingesterConfig } func GetCoreConfig() CoreConfig { return coreConfig } func GetCollectorConfig() CollectorConfig { return collectorConfig } func InitIngesterConfig() { ingesterConfig = IngesterConfig{ BaseConfig: baseConfig{ Env: viper.GetString("env"), Port: viper.GetInt("port"), MetricPort: viper.GetInt("metrics.port"), GinMode: viper.GetString("gin.mode"), }, KafkaConfig: *NewIngesterKafkaConfig(), ElasticSearchConfig: *NewIngesterElasticSearchConfig(), S3Config: *NewIngesterS3Config(), HttpConfig: *NewIngesterHttpConfig(), CorsConfig: *NewCorsConfig(), TracerConfig: NewTracerConfig(), ApiKeyConfig: viper.GetStringMapString("api.key.client.map"), DefaultFactoryWebClientList: strings.Split(viper.GetString("default.factory.web.client.list"), ","), DefaultFactoryWebClientListWithCookieData: strings.Split(viper.GetString("default.factory.web.client.list.with.cookie.data"), ","), WebWhitelistedClientsConfig: strings.Split(viper.GetString("whitelisted.web.clients"), ","), WhiteListedTags: WhiteListedTagsConfig{ CosmosTags: strings.Split(viper.GetString("whitelisted.cosmos.meta.data.tags"), ","), NaviAppTags: strings.Split(viper.GetString("whitelisted.navi.app.meta.data.tags"), ","), }, CruiseCacheTtl: viper.GetDuration("cruise.cache.ttl.minutes"), RateLimiterConfig: *NewIngesterRateLimitConfig(), FpsThresholdScoreMap: viper.GetStringMap("frames.per.second.threshold.score.map"), FpsWeightMapNavi: viper.GetStringMap("frames.per.second.weight.map.navi"), FpsDisableFeatureMap: viper.GetStringMap("frames.per.second.disable.feature.map"), FpsBatteryMapNavi: viper.GetStringMap("frames.per.second.battery.map.navi"), FpsMemoryMapNavi: viper.GetStringMap("frames.per.second.memory.map.navi"), FpsBatteryMapCosmos: viper.GetStringMap("frames.per.second.battery.cosmos.map"), FpsMemoryMapCosmos: viper.GetStringMap("frames.per.second.memory.cosmos.map"), FpsWeightMapCosmos: viper.GetStringMap("frames.per.second.weight.cosmos.map"), FutureTimestampValidationDiffInHours: viper.GetDuration("request.body.future.timestamp.validation.difference.in.hours"), PastTimestampValidationDiffInHours: viper.GetDuration("request.body.past.timestamp.validation.difference.in.hours"), OutboundServiceConfig: *NewIngestorOutboundServiceConfig(), CruiseLogsDeviceIds: viper.GetStringMap("logs.device.ids"), WebSessionDataSliceSize: viper.GetInt("web.session.data.slice.size"), IsWebSessionDataSliceEnabledClientMap: viper.GetStringMapString("is.web.session.data.slice.enabled.client.map"), } } func initCoreConfig() { config := viper.GetString("filter.screen.tag.config") filterScreenTagConfig := make(map[string][]string) err := json.Unmarshal([]byte(config), &filterScreenTagConfig) if err != nil { log.Error("failed to unmarshal screen tag config", zap.Error(err)) os.Exit(1) } coreConfig = CoreConfig{ VideoProcessingConfig: VideoProcessingConfig{ MaxConcurrentJobs: viper.GetInt("video.processing.max.concurrent.jobs"), DefaultTimeoutSeconds: viper.GetInt("video.processing.default.timeout.seconds"), }, BaseConfig: baseConfig{ Env: viper.GetString("env"), Port: viper.GetInt("port"), MetricPort: viper.GetInt("metrics.port"), GinMode: viper.GetString("gin.mode"), }, AuthEnabled: viper.GetBool("auth.enabled"), ByPassAuthSources: strings.Split(viper.GetString("source.to.bypass"), ","), ScreenTagFilters: filterScreenTagConfig, ElasticSearchConfig: *NewCoreElasticSearchConfig(), OutboundServiceConfig: *NewCoreOutboundServiceConfig(), FfmpegConfig: *NewFfmpegConfig(), HttpConfig: *NewCoreHttpConfig(), CorsConfig: *NewCorsConfig(), S3Config: *NewCoreS3Config(), CruiseDropdowns: viper.GetString("cruise.dropdowns"), ZipUploadTimeConfig: viper.GetFloat64("zip.upload.frequency.milliseconds"), ApiKeyClientMap: viper.GetStringMapString("api.key.client.map"), DefaultSessionTime: viper.GetDuration("default.app.sessions.time.hours"), DeviceMonitoringConfig: *NewCoreDeviceMonitoringConfig(), ClientsWithoutDurationResponse: strings.Split(viper.GetString("clients.without.duration.response"), ","), ZipsWithEventsAppVersions: viper.GetString("video.generation.events.app.versions"), MinAppVersionForScreenNameFilter: viper.GetString("min.app.version.for.screenname.filter"), MaxFetchWebVideoGofuncConcurrency: viper.GetInt64("max.fetch.web.video.gofunc.concurrency"), MaxBlurringVideoGofuncConcurrency: viper.GetInt64("max.blurring.video.gofunc.concurrency"), SessionErrorEventsFilter: strings.Split(viper.GetString("session.error.events.filter"), ","), TouchPointsConfig: *NewTouchPointsConfig(), ShedlockConfig: *NewCoreShedlockConfig(), MaskingConfig: *NewMaskingConfig(), MaxUpdateSessionErrorEventsGofuncConcurrency: viper.GetInt64("max.update.session.error.events.gofunc.concurrency"), ClientAppOsMap: viper.GetStringMapString("client.app.os.map"), ClientProjectNameMap: viper.GetStringMapString("client.project.name.map"), AuthKeyServiceMap: viper.GetStringMapString("auth.key.service.map"), } } func initCollectorConfig() { collectorConfig = CollectorConfig{ BaseConfig: baseConfig{ Env: viper.GetString("env"), Port: viper.GetInt("port"), MetricPort: viper.GetInt("metrics.port"), GinMode: viper.GetString("gin.mode"), }, KafkaConfig: *NewCollectorKafkaConfig(), ElasticSearchConfig: *NewCollectorElasticSearchConfig(), S3Config: *NewCollectorS3Config(), CacheTimeForFragmentsIngestion: viper.GetDuration("cache.time.fragments.ingestion"), MaxRetry: viper.GetInt("ingest.elasticsearch.max.retry"), InitialDelayInSeconds: viper.GetDuration("ingest.elasticsearch.initial.delay.in.seconds"), ErrorEventsSlackTemplateId: viper.GetString("error.events.slack.template.id"), SessionErrorEventsFilter: strings.Split(viper.GetString("session.error.events.filter"), ","), IngestErrorEventsFilter: strings.Split(viper.GetString("ingest.error.events.filter"), ","), FutureTimestampValidationDiffInHours: viper.GetDuration("request.body.future.timestamp.validation.difference.in.hours"), PastTimestampValidationDiffInHours: viper.GetDuration("request.body.past.timestamp.validation.difference.in.hours"), IgnoredEventTypes: strings.Split(viper.GetString("ignored.event.types"), ","), EventListenerGoroutineGroupLimit: viper.GetInt("event.listener.goroutine.group.limit"), WebSessionUploadListenerGoroutineGroupLimit: viper.GetInt("web.session.upload.listener.goroutine.group.limit"), SessionUploadListenerGoroutineGroupLimit: viper.GetInt("session.upload.listener.goroutine.group.limit"), } } func LoadIngesterConfig() { viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.SetConfigName("ingester-config") viper.SetConfigType("properties") viper.SetConfigFile("./config/application-ingester.properties") err := viper.ReadInConfig() if err != nil { log.Error("Error while loading ingester configuration", zap.Error(err)) os.Exit(1) } InitIngesterConfig() } func LoadCollectorConfig() { viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.SetConfigName("collector-config") viper.SetConfigType("properties") viper.SetConfigFile("./config/application-collector.properties") err := viper.ReadInConfig() if err != nil { log.Error("Error while loading collector configuration", zap.Error(err)) os.Exit(1) } initCollectorConfig() } func LoadCoreConfig() { viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.SetConfigName("core-config") viper.SetConfigType("properties") viper.SetConfigFile("./config/application-core.properties") err := viper.ReadInConfig() if err != nil { log.Error("Error while loading core configuration", zap.Error(err)) os.Exit(1) } initCoreConfig() } type FerretConfig struct { BaseConfig baseConfig KafkaConfig FerretKafkaConfig ApiKeyConfig map[string]string } var ferretConfig FerretConfig func GetFerretConfig() FerretConfig { return ferretConfig } func initFerretConfig() { ferretConfig = FerretConfig{ BaseConfig: baseConfig{ Env: viper.GetString("env"), Port: viper.GetInt("port"), MetricPort: viper.GetInt("metrics.port"), GinMode: viper.GetString("gin.mode"), }, KafkaConfig: *NewFerretKafkaConfig(), ApiKeyConfig: viper.GetStringMapString("api.key.client.map"), } } func LoadFerretConfig() { viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.SetConfigName("ferret-config") viper.SetConfigType("properties") viper.SetConfigFile("./config/application-ferret.properties") err := viper.ReadInConfig() if err != nil { log.Error("Error while loading ferret configuration", zap.Error(err)) os.Exit(1) } initFerretConfig() }