diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 70bae43b8d9a925bcbc011674e6001b545bdcc68..fa59dd27f480ba5ddf64cb5d7995eadc4b67c194 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,5 +31,7 @@ docker build:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
script:
- - echo "{\"auths\":{\"registry.kszk.bme.hu\":{\"username\":\"$CI_REG_USER\",\"password\":\"$CI_REG_PASS\"}}}" > /kaniko/.docker/config.json
+ - echo "{\"auths\":{\"registry.kszk.bme.hu\":{\"username\":\"$KSZK_NEXUS_USERNAME\",\"password\":\"$KSZK_NEXUS_PASSWORD\"}}}" > /kaniko/.docker/config.json
- /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination "$CONTAINER_IMAGE"
+ only:
+ - master
diff --git a/build.py b/build.py
index eca9a80fe89cddb620cc880e95f129be78f3c4ba..ef2e1e72e1de7869b49a30b53ff2a0cc388ad42c 100644
--- a/build.py
+++ b/build.py
@@ -1,12 +1,14 @@
"""Fill YAML-jinja2 templates, prepare the final data-model"""
import logging
+import sys
import jinja2
import yaml
+from yaml.scanner import ScannerError
from config import Config
-from constants import APP_LOG_TAG, TEMPLATE_PREFIX
-from helper import collect_snippets, check_true
+from constants import APP_LOG_TAG, EXPORTER_PREFIX, ALERT_GROUP_PREFIX
+from transformation import Transformation
class ConfigBuilder:
@@ -15,32 +17,85 @@ class ConfigBuilder:
def __init__(self, cfg: Config):
self.logger = logging.getLogger(APP_LOG_TAG)
self.config = cfg
+ self.config.scrape_configs = []
+ self.config.alert_groups = []
def build(self):
"""Fill YAML-jinja2 templates, prepare the final data-model"""
- for service in self.config.service_definitions:
- for job in service['scraping']:
- self.logger.debug("Templating: " + service['service_name'] + ' -> ' + job['template'])
- self.load_template(job)
- output = self.substitute_template_field(job, service)
-
- # store generated prometheus job config
- if 'ignore' in job and check_true(job['ignore']):
- job['output_yaml'] = None
- else:
- job['output_yaml'] = yaml.safe_load(output)
-
- def load_template(self, job):
- """Substitute 'template' field with its content"""
- template_identifier = TEMPLATE_PREFIX + job['template']
- job['template'] = self.config.templates[template_identifier]
-
- def substitute_template_field(self, job, service):
- """Fill template field with data"""
- env = jinja2.Environment(undefined=jinja2.DebugUndefined)
- template = env.from_string(job['template'])
+ self.logger.info("Build config structures")
+ for service_def in self.config.service_definitions:
+ self.logger.debug("Templating service=" + service_def['service_name'])
+ self.build_scraping_config(service_def)
+ self.build_alerting_config(service_def)
+
+ def build_scraping_config(self, service_def):
+ """Prepare the final data-model for scraping"""
+ for exporter in service_def['scraping']:
+ self.logger.debug("Templating scraping service=" + service_def['service_name'] +
+ ', exporter=' + exporter['exporter'])
+ output_yaml = self.substitute_exporter_field(exporter, service_def)
+ transform = Transformation(self.config)
+ transform.transformate_scraping_job(exporter, service_def, output_yaml)
+ if transform.is_ignored_job(exporter):
+ self.logger.info("Ignore job: " + exporter['name'] + ' [' + exporter['ignore'] + ']')
+ continue
+ # store generated prometheus job config
+ self.config.scrape_configs.append(output_yaml)
+
+ def build_alerting_config(self, service_def):
+ """Prepare the final data-model for alerting"""
+ if 'alerting' not in service_def or 'group' not in service_def['alerting']:
+ self.logger.warning("No alerting definition for service=" + service_def['service_name'])
+ return
+ else:
+ self.logger.debug("Templating alerting for service=" + service_def['service_name'])
+
+ for alert_group in service_def['alerting']['group']:
+ self.substitute_alert_group(alert_group, service_def)
+ transform = Transformation(self.config)
+ transform.transformate_alerting_group(alert_group, service_def)
+ self.config.alert_groups.append(alert_group['alerts'])
+
+ def substitute_exporter_field(self, job, service):
+ """Substitute 'exporter' field with its content"""
+ include_field_name = 'exporter'
+ template_identifier = EXPORTER_PREFIX + job[include_field_name]
+ job['exporter_name'] = job[include_field_name]
+
data = service.copy()
data['job'] = job
- data['snippet'] = collect_snippets(self.config.templates)
+ data['snippets'] = self.config.snippets
+
+ yaml_data = self.substitute_field(template_identifier, include_field_name, job, data)
+ return yaml_data
+
+ def substitute_alert_group(self, alert_group, service):
+ """Substitute 'alerts' field with its content"""
+ include_field_name = 'alerts'
+ template_identifier = ALERT_GROUP_PREFIX + alert_group[include_field_name]
+ alert_group['alert_name'] = alert_group[include_field_name]
+
+ data = service.copy()
+ data['alert_group'] = alert_group
+ data['snippets'] = self.config.snippets
+ yaml_data = self.substitute_field(template_identifier, include_field_name, alert_group, data)
+ alert_group[include_field_name] = yaml_data['group']
+
+ def substitute_field(self, template_identifier, field_name, parent_object, data):
+ """Substitute field with jinja2 content"""
+ parent_object[field_name] = self.config.templates[template_identifier]
+
+ """Fill template field with data"""
+ env = jinja2.Environment(undefined=jinja2.DebugUndefined)
+ template = env.from_string(parent_object[field_name])
+ yaml_as_text = template.render(data)
- return template.render(data)
+ service_name = data['service_name']
+ try:
+ yaml_data = yaml.safe_load(yaml_as_text)
+ return yaml_data
+ except ScannerError as e:
+ self.logger.fatal("Cannot parse intermediate YAML data (service_name=" + service_name + "):")
+ self.logger.fatal(yaml_as_text)
+ self.logger.fatal(e)
+ sys.exit(42)
diff --git a/config.py b/config.py
index 12370595c7e23636ceefeb6f6f7aa7689c216934..e6e28f97d3ce10368ba1f7a58bea7391fc07fa3c 100644
--- a/config.py
+++ b/config.py
@@ -1,47 +1,58 @@
-"""Read service YAML files and preload templates"""
+"""Read service, template and snippet files (without templating)"""
import logging
from pathlib import Path
import yaml
-from constants import APP_LOG_TAG
-from helper import is_service_file, is_template_file
+from constants import APP_LOG_TAG, SERVICE_PREFIX, TEMPLATE_PREFIX, SNIPPET_PREFIX
class Config:
- """Read service YAML files and preload templates"""
+ """Read service, template and snippet files (without templating)"""
service_definitions = []
templates = {}
+ snippets = {}
def __init__(self, path: Path):
self.logger = logging.getLogger(APP_LOG_TAG)
# YAML files in tha data directory recursively
base_path_len = len(str(path.absolute())) + 1
- self.preload_service_files(path.rglob('*.yaml'), base_path_len)
- self.preload_template_files(path.rglob('*.yaml.j2'), base_path_len)
+ self.read_service_files(path.rglob('*.yaml'), base_path_len)
+ self.read_template_files(path.rglob('*.yaml.j2'), base_path_len)
+ self.read_snippet_files(path.rglob('*.yaml.j2'), base_path_len)
- def preload_service_files(self, path_glob, base_path_len: int):
- """Read all (service)YAML files in tha data directory recursively"""
+ def read_service_files(self, path_glob, base_path_len: int):
+ """Read all service YAML files in tha data directory recursively"""
+ self.logger.info("Read service YAML files")
for yaml_path in path_glob:
file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml")]
- if is_service_file(file_identifier):
- self.service_definitions.append(self.read_yaml_file(yaml_path))
-
- def preload_template_files(self, path_glob, base_path_len: int):
- """Read all (template)YAML files in tha data directory recursively"""
+ if file_identifier.startswith(SERVICE_PREFIX):
+ self.logger.debug("Load service file: " + str(yaml_path.absolute()))
+ with yaml_path.open('r') as stream:
+ try:
+ service_yaml = yaml.safe_load(stream)
+ self.service_definitions.append(service_yaml)
+ except yaml.YAMLError as exc:
+ self.logger.error("Cannot load service YAML file.")
+ self.logger.error(exc)
+
+ def read_template_files(self, path_glob, base_path_len: int):
+ """Read all template files in tha data directory recursively"""
+ self.logger.info("Read template files (without parsing)")
for yaml_path in path_glob:
file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml.j2")]
- if is_template_file(file_identifier):
+ if file_identifier.startswith(TEMPLATE_PREFIX):
data = open(str(yaml_path.absolute()), "r")
self.logger.debug("Load template file: " + str(yaml_path.absolute()))
self.templates[file_identifier] = data.read()
- def read_yaml_file(self, yaml_path: Path):
- """Reads a YAML file"""
- self.logger.debug("Load YAML file: " + str(yaml_path.absolute()))
- with yaml_path.open('r') as stream:
- try:
- return yaml.safe_load(stream)
- except yaml.YAMLError as exc:
- self.logger.error("Cannot load YAML file.")
- self.logger.error(exc)
+ def read_snippet_files(self, path_glob, base_path_len: int):
+ """Read all snippet files in tha data directory recursively"""
+ self.logger.info("Read snippet files (without parsing)")
+ for yaml_path in path_glob:
+ file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml.j2")]
+ if file_identifier.startswith(SNIPPET_PREFIX):
+ data = open(str(yaml_path.absolute()), "r")
+ self.logger.debug("Load snippet file: " + str(yaml_path.absolute()))
+ snippet_name = file_identifier[len(SNIPPET_PREFIX):]
+ self.snippets[snippet_name] = data.read()
diff --git a/constants.py b/constants.py
index 4b59614e2518ef7602eebbbd9ba00d9c8c5630af..906ac96929df53e5da5ce59a8d1cda55a3219865 100644
--- a/constants.py
+++ b/constants.py
@@ -4,4 +4,7 @@ GENERATOR_OUTPUT_FOLDER = 'generated/'
OUTPUT_TEMPLATE_FOLDER = 'output-templates/'
SERVICE_PREFIX = 'services/'
TEMPLATE_PREFIX = 'service-templates/'
-SNIPPET_PREFIX = 'service-templates/snippet/'
+EXPORTER_PREFIX = 'service-templates/exporters/'
+INDIVIDUAL_ALERT_PREFIX = 'service-templates/alerting/individual/'
+ALERT_GROUP_PREFIX = 'service-templates/alerting/group/'
+SNIPPET_PREFIX = 'service-templates/snippets/'
diff --git a/generator.py b/generator.py
index 873e12e9a5c5bc46a5adacde122f9836078034bc..cadd9aae51490d0fc7c77726ed8771fe06ae4363 100644
--- a/generator.py
+++ b/generator.py
@@ -10,7 +10,6 @@ import yaml
from config import Config
from constants import APP_LOG_TAG, OUTPUT_TEMPLATE_FOLDER, GENERATOR_OUTPUT_FOLDER
-from helper import collect_snippets
def autogen_warning():
@@ -35,9 +34,11 @@ class Generator:
data = {
'autogen_warning': autogen_warning(),
'generation_info': self.generation_info(),
- 'snippet': collect_snippets(self.config.templates),
+ 'snippets': self.config.snippets,
}
self.collect_scrape_configs(data)
+ self.collect_alert_groups(data)
+ self.collect_alertmanager_receivers(data)
self.generate_files(data)
def generation_info(self):
@@ -56,13 +57,28 @@ class Generator:
def collect_scrape_configs(self, output):
"""Build data model to the generation"""
- data = []
- for service in self.config.service_definitions:
- for job in service['scraping']:
- if job['output_yaml'] is not None:
- data.append(job['output_yaml'])
output['scrape_configs'] = yaml.dump(
- {'scrape_configs': data},
+ {'scrape_configs': self.config.scrape_configs},
+ explicit_start=False,
+ default_flow_style=False
+ )
+
+ def collect_alert_groups(self, output):
+ """Build data model to the generation"""
+ alert_groups = self.config.alert_groups
+
+ output['alert_groups'] = yaml.dump(
+ {'groups': alert_groups},
+ explicit_start=False,
+ default_flow_style=False
+ )
+
+ def collect_alertmanager_receivers(self, output):
+ """Build data model to the generation"""
+ alertmanager_receivers = 'alertmanager_receivers'
+
+ output['alertmanager_receivers'] = yaml.dump(
+ {'receivers': alertmanager_receivers},
explicit_start=False,
default_flow_style=False
)
diff --git a/helper.py b/helper.py
index 48c3064feb007488f8c05e76ffdf57276ae849aa..c7a9b2f83f790955af0f282cb3a2f8ef3f63cda9 100644
--- a/helper.py
+++ b/helper.py
@@ -1,22 +1,5 @@
"""helper methods"""
-from constants import SERVICE_PREFIX, TEMPLATE_PREFIX, SNIPPET_PREFIX
-
-
-def is_service_file(file_identifier: str) -> bool:
- """Hmm, is it a service definition file or not"""
- return file_identifier.startswith(SERVICE_PREFIX)
-
-
-def is_template_file(file_identifier: str) -> bool:
- """Hmm, is it a service template file or not"""
- return file_identifier.startswith(TEMPLATE_PREFIX)
-
-
-def is_snippet_file(file_identifier: str) -> bool:
- """Hmm, is it a snippet template file or not"""
- return file_identifier.startswith(SNIPPET_PREFIX)
-
def check_true(s: str) -> bool:
"""Hmm, is it a true-ish string"""
@@ -25,13 +8,3 @@ def check_true(s: str) -> bool:
'i', 'igen', 'aha', 'ja', 'jaja', 'nosza', 'mibajlehet', 'miért ne', # Hungarian extension
'temp', 'temporary', 'ideiglenesen' # other extension
]
-
-
-def collect_snippets(templates):
- """Get snippets from templates"""
- snippets = {}
- for identifier in templates:
- if is_snippet_file(identifier):
- snippet_name = identifier[len(SNIPPET_PREFIX):]
- snippets[snippet_name] = templates[identifier]
- return snippets
diff --git a/pupak.py b/pupak.py
index c53c4833339256a54bd3ef4d723fc22ccbfdb441..16d5f80353819afcd298784fa8bb14db0df20064 100644
--- a/pupak.py
+++ b/pupak.py
@@ -21,9 +21,10 @@ if __name__ == "__main__":
logger.error("Usage: data_folder")
sys.exit(1)
+ # the base path (read YAML files from)
data_folder = Path(sys.argv[1])
- # Read service YAML files and preload templates
+ # Read service, template and snippet files (without templating)
config = Config(data_folder)
# Fill YAML-jinja2 templates,
diff --git a/transformation.py b/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..48f758eb0df9c1667c2d708cf4ab36258716996b
--- /dev/null
+++ b/transformation.py
@@ -0,0 +1,53 @@
+"""Custom transformation of data-model"""
+import logging
+
+from config import Config
+from constants import APP_LOG_TAG
+from helper import check_true
+
+
+class Transformation:
+ """Custom transformation of data-model"""
+
+ def __init__(self, cfg: Config):
+ self.logger = logging.getLogger(APP_LOG_TAG)
+ self.config = cfg
+
+ def transformate_scraping_job(self, job, service, output_yaml):
+ """Modify job's data to be suitable for templating"""
+ self.add_job_name(job, service, output_yaml)
+ self.add_labels(job, service, output_yaml)
+
+ def add_job_name(self, job, service, output_yaml):
+ """Add job_name property to prometheus job definitions"""
+ if 'job_name' not in job:
+ module_name = ''
+ if 'params' in output_yaml and 'module' in output_yaml['params']:
+ module_name = '#' + output_yaml['params']['module'][0]
+ job['name'] = service['service_name'] + '@' + job['exporter_name'] + module_name
+ output_yaml['job_name'] = job['name']
+
+ def add_labels(self, job, service, output_yaml):
+ """Add labels e.g. job_name to prometheus job definitions"""
+ for static_configs in output_yaml['static_configs']:
+ if 'labels' not in static_configs:
+ static_configs['labels'] = {}
+ static_configs['labels']['job'] = job['exporter_name'] # TODO check is this intended<
+ # static_configs['labels']['exporter'] = job['exporter_name']
+ static_configs['labels']['service_name'] = service['service_name']
+
+ def is_ignored_job(self, job):
+ """Check ignored property"""
+ return 'ignore' in job and check_true(job['ignore'])
+
+ def transformate_alerting_group(self, alert_group, service_def):
+ """Modify alert def's data to be suitable for templating"""
+ self.add_alert_group_name(alert_group, service_def)
+
+ def add_alert_group_name(self, alert_group, service_def):
+ """Add name property to prometheus alert definition"""
+ if 'name' not in alert_group['alerts']:
+ name = service_def["service_name"]
+ name += "@"
+ name += alert_group["alert_name"].replace("/", "-")
+ alert_group['alerts']['name'] = name