5d17f96135
* INFRA-3438 | Saurabh | Added Client TLS Negotiation Error in json schema and script * INFRA-3438 | Saurabh | Added Client TLS Negotiation Error in json schema and script
99 lines
4.3 KiB
Python
99 lines
4.3 KiB
Python
from kubernetes import client, config
|
|
|
|
|
|
def modify_prometheus_rules(api_instance, namespaces):
|
|
# Retrieve the existing PrometheusRule resources in the specified namespace
|
|
for namespace in namespaces:
|
|
# skip if the namespace name is either monitoring, logging, kube-system, istio-system, kube-node-lease, kube-public, portieris, telepresence, twistlock, velero, wiz
|
|
if namespace in ["monitoring", "logging", "kube-system", "istio-system", "kube-node-lease",
|
|
"kube-public", "portieris", "telepresence", "twistlock", "velero", "wiz"]:
|
|
print(f"Skipping namespace: {namespace}")
|
|
continue
|
|
|
|
rules = api_instance.list_namespaced_custom_object(
|
|
group="monitoring.coreos.com",
|
|
version="v1",
|
|
plural="prometheusrules",
|
|
namespace=namespace
|
|
)
|
|
|
|
for rule in rules.get("items", []):
|
|
print(f"Modifying PrometheusRule: {namespace} {rule['metadata']['name']}")
|
|
if len(rule["spec"]["groups"]) == 0:
|
|
print(f"PrometheusRule: {namespace} {rule['metadata']['name']} has no rules")
|
|
continue
|
|
|
|
if 'labels' not in rule["metadata"] or 'Team' not in rule["metadata"]["labels"]:
|
|
print(f"PrometheusRule: {namespace} {rule['metadata']['name']} has no Team label")
|
|
continue
|
|
|
|
elb5xx_exists = any(r["alert"] == "HighELB5xx" for r in rule["spec"]["groups"][0]["rules"])
|
|
if elb5xx_exists:
|
|
app_name = rule["metadata"]["name"]
|
|
app_team = rule["metadata"]["labels"]['Team']
|
|
|
|
|
|
high_tls_negotiation_errors_alert = {
|
|
"alert": "HighTLSNegotiationErrors",
|
|
"annotations": {
|
|
"description":f'Namespace: {namespace}, AppName: {app_name}; has more client TLS negotiation errors in the last 1m',
|
|
"runbook": 'https://navihq.atlassian.net/wiki/spaces/IN/pages/509936863/Runbook',
|
|
"summary": 'Service is facing a lot of TLS negotiation errors'
|
|
},
|
|
"expr": f"sum by (tag_Ingress) (aws_alb_client_tlsnegotiation_error_count_sum{{tag_Ingress=\"{app_name}\"}}) > 1",
|
|
"for": "1m",
|
|
"labels": {
|
|
"alertTeam": f'{app_team}',
|
|
"appName": f'{app_name}',
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
|
|
# Add the HighTLSNegotiationErrors alert to the PrometheusRule
|
|
rule["spec"]["groups"][0]["rules"].append(high_tls_negotiation_errors_alert)
|
|
|
|
# Update the PrometheusRule resource with the modified rules
|
|
api_instance.replace_namespaced_custom_object(
|
|
group="monitoring.coreos.com",
|
|
version="v1",
|
|
plural="prometheusrules",
|
|
namespace=namespace,
|
|
name=app_name,
|
|
body=rule
|
|
)
|
|
|
|
print(f"Added HighTLSNegotiationErrors alert to PrometheusRule: {namespace} {rule['metadata']['name']}")
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
kubernetes_contexts = [
|
|
"nonprod.np.navi-tech.in",
|
|
"aps1.prod.navi-tech.in",
|
|
"aps1.np.navi-gi.in",
|
|
"aps1.prod.navi-gi.in",
|
|
"aps1.np.navi-sa.in",
|
|
"aps1.prod.navi-sa.in",
|
|
"aps1.np.navi-pay.in",
|
|
"aps1.prod.navi-pay.in",
|
|
]
|
|
|
|
for context in kubernetes_contexts:
|
|
print(f"Modifying PrometheusRule resources for Kubernetes context: {context}")
|
|
try:
|
|
# Load the Kubernetes configuration for the current context
|
|
config.load_kube_config(context=context)
|
|
|
|
# Create the API client
|
|
api_inst = client.CustomObjectsApi()
|
|
|
|
# Call the function to modify the PrometheusRule resources
|
|
v1 = client.CoreV1Api()
|
|
ns = [ns.metadata.name for ns in v1.list_namespace().items]
|
|
modify_prometheus_rules(api_inst, ns)
|
|
|
|
print(f"PrometheusRule modifications completed for Kubernetes context: {context}")
|
|
print("--------------------------------------------------------------")
|
|
except Exception as e:
|
|
print(f"Error occurred for Kubernetes context {context}: {str(e)}")
|