varnitgoyal95/deployment-portal-be
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

INFRA-1890 | Ashvin | Fix approval request 403

Browse Source
This commit is contained in:
Ashvin Sharma
2023-08-08 13:32:25 +05:30
parent 1b291e2ee8
commit 7c231d5bab
7 changed files with 92 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
scripts/prometheus_rule_oom.py
View File

@@ -43,8 +43,8 @@ def modify_prometheus_rules(api_instance, namespaces):
"expr": f'increase(kube_pod_container_status_restarts_total{{namespace="{namespace}", container="{app_name}"}}[10m]) >= 2 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{{namespace="{namespace}", container="{app_name}", reason="OOMKilled"}} > 0',
"labels": {
"severity": 'critical',
"alertTeam": f'"{app_team}"',
"appName": f'"{app_name}"',
"alertTeam": f'{app_team}',
"appName": f'{app_name}',
},
}
pod_oom_killed_alert = {
@@ -57,8 +57,8 @@ def modify_prometheus_rules(api_instance, namespaces):
"expr": f'kube_pod_container_status_restarts_total{{namespace="{namespace}", container="{app_name}"}} - kube_pod_container_status_restarts_total{{namespace="{namespace}", container="{app_name}"}} offset 5m >= 1 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{{namespace="{namespace}", container="{app_name}", reason="OOMKilled"}} > 0',
"labels": {
"severity": 'warning',
"alertTeam": f'"{app_team}"',
"appName": f'"{app_name}',
"alertTeam": f'{app_team}',
"appName": f'{app_name}',
},
}
@@ -78,15 +78,32 @@ def modify_prometheus_rules(api_instance, namespaces):
if __name__ == "__main__":
# Load the Kubernetes configuration from default location
config.load_kube_config()
kubernetes_contexts = [
"nonprod.np.navi-tech.in",
"aps1.prod.navi-tech.in",
"aps1.np.navi-gi.in",
"aps1.prod.navi-gi.in",
"aps1.np.navi-sa.in",
"aps1.prod.navi-sa.in",
"aps1.np.navi-pay.in",
"aps1.prod.navi-pay.in",
]
# Create the API client
api_inst = client.CustomObjectsApi()
for context in kubernetes_contexts:
print(f"Modifying PrometheusRule resources for Kubernetes context: {context}")
try:
# Load the Kubernetes configuration for the current context
config.load_kube_config(context=context)
# Call the function to modify the PrometheusRule resources
v1 = client.CoreV1Api()
ns = [ns.metadata.name for ns in v1.list_namespace().items]
modify_prometheus_rules(api_inst, ns)
# Create the API client
api_inst = client.CustomObjectsApi()
print("PrometheusRule modifications completed.")
# Call the function to modify the PrometheusRule resources
v1 = client.CoreV1Api()
ns = [ns.metadata.name for ns in v1.list_namespace().items]
modify_prometheus_rules(api_inst, ns)
print(f"PrometheusRule modifications completed for Kubernetes context: {context}")
print("--------------------------------------------------------------")
except Exception as e:
print(f"Error occurred for Kubernetes context {context}: {str(e)}")