Skip to content
Snippets Groups Projects
Commit f778ba4d authored by Robotka István Adrián's avatar Robotka István Adrián
Browse files

alerting poc

parent e0f3f213
No related branches found
No related tags found
No related merge requests found
......@@ -31,5 +31,7 @@ docker build:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
script:
- echo "{\"auths\":{\"registry.kszk.bme.hu\":{\"username\":\"$CI_REG_USER\",\"password\":\"$CI_REG_PASS\"}}}" > /kaniko/.docker/config.json
- echo "{\"auths\":{\"registry.kszk.bme.hu\":{\"username\":\"$KSZK_NEXUS_USERNAME\",\"password\":\"$KSZK_NEXUS_PASSWORD\"}}}" > /kaniko/.docker/config.json
- /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination "$CONTAINER_IMAGE"
only:
- master
"""Fill YAML-jinja2 templates, prepare the final data-model"""
import logging
import sys
import jinja2
import yaml
from yaml.scanner import ScannerError
from config import Config
from constants import APP_LOG_TAG, TEMPLATE_PREFIX
from helper import collect_snippets, check_true
from constants import APP_LOG_TAG, EXPORTER_PREFIX, ALERT_GROUP_PREFIX
from transformation import Transformation
class ConfigBuilder:
......@@ -15,32 +17,85 @@ class ConfigBuilder:
def __init__(self, cfg: Config):
self.logger = logging.getLogger(APP_LOG_TAG)
self.config = cfg
self.config.scrape_configs = []
self.config.alert_groups = []
def build(self):
"""Fill YAML-jinja2 templates, prepare the final data-model"""
for service in self.config.service_definitions:
for job in service['scraping']:
self.logger.debug("Templating: " + service['service_name'] + ' -> ' + job['template'])
self.load_template(job)
output = self.substitute_template_field(job, service)
self.logger.info("Build config structures")
for service_def in self.config.service_definitions:
self.logger.debug("Templating service=" + service_def['service_name'])
self.build_scraping_config(service_def)
self.build_alerting_config(service_def)
def build_scraping_config(self, service_def):
"""Prepare the final data-model for scraping"""
for exporter in service_def['scraping']:
self.logger.debug("Templating scraping service=" + service_def['service_name'] +
', exporter=' + exporter['exporter'])
output_yaml = self.substitute_exporter_field(exporter, service_def)
transform = Transformation(self.config)
transform.transformate_scraping_job(exporter, service_def, output_yaml)
if transform.is_ignored_job(exporter):
self.logger.info("Ignore job: " + exporter['name'] + ' [' + exporter['ignore'] + ']')
continue
# store generated prometheus job config
if 'ignore' in job and check_true(job['ignore']):
job['output_yaml'] = None
self.config.scrape_configs.append(output_yaml)
def build_alerting_config(self, service_def):
"""Prepare the final data-model for alerting"""
if 'alerting' not in service_def or 'group' not in service_def['alerting']:
self.logger.warning("No alerting definition for service=" + service_def['service_name'])
return
else:
job['output_yaml'] = yaml.safe_load(output)
self.logger.debug("Templating alerting for service=" + service_def['service_name'])
def load_template(self, job):
"""Substitute 'template' field with its content"""
template_identifier = TEMPLATE_PREFIX + job['template']
job['template'] = self.config.templates[template_identifier]
for alert_group in service_def['alerting']['group']:
self.substitute_alert_group(alert_group, service_def)
transform = Transformation(self.config)
transform.transformate_alerting_group(alert_group, service_def)
self.config.alert_groups.append(alert_group['alerts'])
def substitute_exporter_field(self, job, service):
"""Substitute 'exporter' field with its content"""
include_field_name = 'exporter'
template_identifier = EXPORTER_PREFIX + job[include_field_name]
job['exporter_name'] = job[include_field_name]
def substitute_template_field(self, job, service):
"""Fill template field with data"""
env = jinja2.Environment(undefined=jinja2.DebugUndefined)
template = env.from_string(job['template'])
data = service.copy()
data['job'] = job
data['snippet'] = collect_snippets(self.config.templates)
data['snippets'] = self.config.snippets
yaml_data = self.substitute_field(template_identifier, include_field_name, job, data)
return yaml_data
def substitute_alert_group(self, alert_group, service):
"""Substitute 'alerts' field with its content"""
include_field_name = 'alerts'
template_identifier = ALERT_GROUP_PREFIX + alert_group[include_field_name]
alert_group['alert_name'] = alert_group[include_field_name]
data = service.copy()
data['alert_group'] = alert_group
data['snippets'] = self.config.snippets
yaml_data = self.substitute_field(template_identifier, include_field_name, alert_group, data)
alert_group[include_field_name] = yaml_data['group']
def substitute_field(self, template_identifier, field_name, parent_object, data):
"""Substitute field with jinja2 content"""
parent_object[field_name] = self.config.templates[template_identifier]
"""Fill template field with data"""
env = jinja2.Environment(undefined=jinja2.DebugUndefined)
template = env.from_string(parent_object[field_name])
yaml_as_text = template.render(data)
return template.render(data)
service_name = data['service_name']
try:
yaml_data = yaml.safe_load(yaml_as_text)
return yaml_data
except ScannerError as e:
self.logger.fatal("Cannot parse intermediate YAML data (service_name=" + service_name + "):")
self.logger.fatal(yaml_as_text)
self.logger.fatal(e)
sys.exit(42)
"""Read service YAML files and preload templates"""
"""Read service, template and snippet files (without templating)"""
import logging
from pathlib import Path
import yaml
from constants import APP_LOG_TAG
from helper import is_service_file, is_template_file
from constants import APP_LOG_TAG, SERVICE_PREFIX, TEMPLATE_PREFIX, SNIPPET_PREFIX
class Config:
"""Read service YAML files and preload templates"""
"""Read service, template and snippet files (without templating)"""
service_definitions = []
templates = {}
snippets = {}
def __init__(self, path: Path):
self.logger = logging.getLogger(APP_LOG_TAG)
# YAML files in tha data directory recursively
base_path_len = len(str(path.absolute())) + 1
self.preload_service_files(path.rglob('*.yaml'), base_path_len)
self.preload_template_files(path.rglob('*.yaml.j2'), base_path_len)
self.read_service_files(path.rglob('*.yaml'), base_path_len)
self.read_template_files(path.rglob('*.yaml.j2'), base_path_len)
self.read_snippet_files(path.rglob('*.yaml.j2'), base_path_len)
def preload_service_files(self, path_glob, base_path_len: int):
"""Read all (service)YAML files in tha data directory recursively"""
def read_service_files(self, path_glob, base_path_len: int):
"""Read all service YAML files in tha data directory recursively"""
self.logger.info("Read service YAML files")
for yaml_path in path_glob:
file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml")]
if is_service_file(file_identifier):
self.service_definitions.append(self.read_yaml_file(yaml_path))
if file_identifier.startswith(SERVICE_PREFIX):
self.logger.debug("Load service file: " + str(yaml_path.absolute()))
with yaml_path.open('r') as stream:
try:
service_yaml = yaml.safe_load(stream)
self.service_definitions.append(service_yaml)
except yaml.YAMLError as exc:
self.logger.error("Cannot load service YAML file.")
self.logger.error(exc)
def preload_template_files(self, path_glob, base_path_len: int):
"""Read all (template)YAML files in tha data directory recursively"""
def read_template_files(self, path_glob, base_path_len: int):
"""Read all template files in tha data directory recursively"""
self.logger.info("Read template files (without parsing)")
for yaml_path in path_glob:
file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml.j2")]
if is_template_file(file_identifier):
if file_identifier.startswith(TEMPLATE_PREFIX):
data = open(str(yaml_path.absolute()), "r")
self.logger.debug("Load template file: " + str(yaml_path.absolute()))
self.templates[file_identifier] = data.read()
def read_yaml_file(self, yaml_path: Path):
"""Reads a YAML file"""
self.logger.debug("Load YAML file: " + str(yaml_path.absolute()))
with yaml_path.open('r') as stream:
try:
return yaml.safe_load(stream)
except yaml.YAMLError as exc:
self.logger.error("Cannot load YAML file.")
self.logger.error(exc)
def read_snippet_files(self, path_glob, base_path_len: int):
"""Read all snippet files in tha data directory recursively"""
self.logger.info("Read snippet files (without parsing)")
for yaml_path in path_glob:
file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml.j2")]
if file_identifier.startswith(SNIPPET_PREFIX):
data = open(str(yaml_path.absolute()), "r")
self.logger.debug("Load snippet file: " + str(yaml_path.absolute()))
snippet_name = file_identifier[len(SNIPPET_PREFIX):]
self.snippets[snippet_name] = data.read()
......@@ -4,4 +4,7 @@ GENERATOR_OUTPUT_FOLDER = 'generated/'
OUTPUT_TEMPLATE_FOLDER = 'output-templates/'
SERVICE_PREFIX = 'services/'
TEMPLATE_PREFIX = 'service-templates/'
SNIPPET_PREFIX = 'service-templates/snippet/'
EXPORTER_PREFIX = 'service-templates/exporters/'
INDIVIDUAL_ALERT_PREFIX = 'service-templates/alerting/individual/'
ALERT_GROUP_PREFIX = 'service-templates/alerting/group/'
SNIPPET_PREFIX = 'service-templates/snippets/'
......@@ -10,7 +10,6 @@ import yaml
from config import Config
from constants import APP_LOG_TAG, OUTPUT_TEMPLATE_FOLDER, GENERATOR_OUTPUT_FOLDER
from helper import collect_snippets
def autogen_warning():
......@@ -35,9 +34,11 @@ class Generator:
data = {
'autogen_warning': autogen_warning(),
'generation_info': self.generation_info(),
'snippet': collect_snippets(self.config.templates),
'snippets': self.config.snippets,
}
self.collect_scrape_configs(data)
self.collect_alert_groups(data)
self.collect_alertmanager_receivers(data)
self.generate_files(data)
def generation_info(self):
......@@ -56,13 +57,28 @@ class Generator:
def collect_scrape_configs(self, output):
"""Build data model to the generation"""
data = []
for service in self.config.service_definitions:
for job in service['scraping']:
if job['output_yaml'] is not None:
data.append(job['output_yaml'])
output['scrape_configs'] = yaml.dump(
{'scrape_configs': data},
{'scrape_configs': self.config.scrape_configs},
explicit_start=False,
default_flow_style=False
)
def collect_alert_groups(self, output):
"""Build data model to the generation"""
alert_groups = self.config.alert_groups
output['alert_groups'] = yaml.dump(
{'groups': alert_groups},
explicit_start=False,
default_flow_style=False
)
def collect_alertmanager_receivers(self, output):
"""Build data model to the generation"""
alertmanager_receivers = 'alertmanager_receivers'
output['alertmanager_receivers'] = yaml.dump(
{'receivers': alertmanager_receivers},
explicit_start=False,
default_flow_style=False
)
......
"""helper methods"""
from constants import SERVICE_PREFIX, TEMPLATE_PREFIX, SNIPPET_PREFIX
def is_service_file(file_identifier: str) -> bool:
"""Hmm, is it a service definition file or not"""
return file_identifier.startswith(SERVICE_PREFIX)
def is_template_file(file_identifier: str) -> bool:
"""Hmm, is it a service template file or not"""
return file_identifier.startswith(TEMPLATE_PREFIX)
def is_snippet_file(file_identifier: str) -> bool:
"""Hmm, is it a snippet template file or not"""
return file_identifier.startswith(SNIPPET_PREFIX)
def check_true(s: str) -> bool:
"""Hmm, is it a true-ish string"""
......@@ -25,13 +8,3 @@ def check_true(s: str) -> bool:
'i', 'igen', 'aha', 'ja', 'jaja', 'nosza', 'mibajlehet', 'miért ne', # Hungarian extension
'temp', 'temporary', 'ideiglenesen' # other extension
]
def collect_snippets(templates):
"""Get snippets from templates"""
snippets = {}
for identifier in templates:
if is_snippet_file(identifier):
snippet_name = identifier[len(SNIPPET_PREFIX):]
snippets[snippet_name] = templates[identifier]
return snippets
......@@ -21,9 +21,10 @@ if __name__ == "__main__":
logger.error("Usage: data_folder")
sys.exit(1)
# the base path (read YAML files from)
data_folder = Path(sys.argv[1])
# Read service YAML files and preload templates
# Read service, template and snippet files (without templating)
config = Config(data_folder)
# Fill YAML-jinja2 templates,
......
"""Custom transformation of data-model"""
import logging
from config import Config
from constants import APP_LOG_TAG
from helper import check_true
class Transformation:
"""Custom transformation of data-model"""
def __init__(self, cfg: Config):
self.logger = logging.getLogger(APP_LOG_TAG)
self.config = cfg
def transformate_scraping_job(self, job, service, output_yaml):
"""Modify job's data to be suitable for templating"""
self.add_job_name(job, service, output_yaml)
self.add_labels(job, service, output_yaml)
def add_job_name(self, job, service, output_yaml):
"""Add job_name property to prometheus job definitions"""
if 'job_name' not in job:
module_name = ''
if 'params' in output_yaml and 'module' in output_yaml['params']:
module_name = '#' + output_yaml['params']['module'][0]
job['name'] = service['service_name'] + '@' + job['exporter_name'] + module_name
output_yaml['job_name'] = job['name']
def add_labels(self, job, service, output_yaml):
"""Add labels e.g. job_name to prometheus job definitions"""
for static_configs in output_yaml['static_configs']:
if 'labels' not in static_configs:
static_configs['labels'] = {}
static_configs['labels']['job'] = job['exporter_name'] # TODO check is this intended<
# static_configs['labels']['exporter'] = job['exporter_name']
static_configs['labels']['service_name'] = service['service_name']
def is_ignored_job(self, job):
"""Check ignored property"""
return 'ignore' in job and check_true(job['ignore'])
def transformate_alerting_group(self, alert_group, service_def):
"""Modify alert def's data to be suitable for templating"""
self.add_alert_group_name(alert_group, service_def)
def add_alert_group_name(self, alert_group, service_def):
"""Add name property to prometheus alert definition"""
if 'name' not in alert_group['alerts']:
name = service_def["service_name"]
name += "@"
name += alert_group["alert_name"].replace("/", "-")
alert_group['alerts']['name'] = name
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment