add project

.editorconfig

+# EditorConfig is awesome: https://EditorConfig.org
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+max_line_length = 120
+tab_width = 4
+
+[*.yml]
+indent_size = 2
+
+[*.json]
+indent_size = 2
+
+[*.sh]
+max_line_length = 80
+indent_size = 2

.gitignore

+/.idea/
+/venv/
+/data/

.gitlab-ci.yml

+image: "python:3.7"
+
+stages:
+  - Static Analysis
+  - Docker build
+
+variables:
+  CONTAINER_IMAGE: "registry.kszk.bme.hu/kszk/monitoring/generator:$CI_COMMIT_REF_NAME"
+
+before_script:
+  - python --version
+  - pip install -r requirements.txt
+
+flake8:
+  stage: Static Analysis
+  script:
+    - flake8 --max-line-length=120 *.py
+
+pylint:
+  stage: Static Analysis
+  allow_failure: true
+  script:
+    - pylint -d C0301 *.py
+
+docker build:
+  stage: Docker build
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  script:
+    - echo "{\"auths\":{\"registry.kszk.bme.hu\":{\"username\":\"$CI_REG_USER\",\"password\":\"$CI_REG_PASS\"}}}" > /kaniko/.docker/services.json
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CONTAINER_IMAGE

README.md

+# Pupák Config generator
+Generates config files from custom data scheme and jinja2 templates.
+Uses yaml as data source.
+
+## Usage
+You do not want to use this.
+It's under development.

config.py

+import yaml
+from validation import validate_config
+from transformation import setup_host_fullnames
+from transformation import setup_static_dns
+from transformation import setup_node_exporters
+from transformation import setup_bind9_exporters
+from transformation import setup_ping_exporters
+from transformation import setup_wmi_exporters
+from transformation import setup_exporters
+from transformation import setup_autogen_warning
+
+
+class config():
+    def __init__(self, data_root):
+        self.raw_config = yaml.safe_load(open(data_root))
+        print("YAML data file has loaded: " + data_root)
+
+    def validate(self):
+        self.validated_config = validate_config(self)
+
+    def transformate(self):
+        setup_host_fullnames(self)  # should be the first extender
+        setup_static_dns(self)
+        setup_node_exporters(self)
+        setup_bind9_exporters(self)
+        setup_ping_exporters(self)
+        setup_wmi_exporters(self)
+        self.validated_config['snmp_exporters'] = setup_exporters(self, 'snmp')
+        setup_autogen_warning(self)

generator.py

+#!/usr/bin/env python3
+import os
+import sys
+from config import config
+
+from schema import Schema, And, Optional
+import jinja2
+import yaml
+
+
+def generate_config_files(data, template_folder, out_folder):
+    template_loader = jinja2.FileSystemLoader(searchpath=template_folder)
+    template_env = jinja2.Environment(loader=template_loader)
+    print(data)
+
+    for filename in os.listdir(template_folder):
+        output = template_env.get_template(filename).render(data)
+
+        out_name = out_folder + '/' + filename
+        out_name = out_name.replace(".j2", "")
+        f = open(out_name, "w+")
+        f.write(output)
+        f.close()
+
+
+if __name__ == "__main__":
+    print("Starting generator script.")
+
+    if len(sys.argv) < 3 + 1:
+        print("Usage: data_file template_folder out_folder")
+        exit(1)
+
+    config_filename = sys.argv[1]
+    template_folder = sys.argv[2]
+    out_folder = sys.argv[3]
+
+    cfg = config(config_filename)
+    print("YAML data file has loaded: " + config_filename)
+
+    print("Data scheme validation:")
+    cfg.validate()
+    print("Data scheme is VALID.")
+
+    cfg.transformate()
+    print("Successful data scheme extension.")
+
+    generate_config_files(cfg.validated_config, template_folder, out_folder)
+    print("Config files has been generated.")

requirements.txt

+PyYAML
+jinja2
+schema

transformation.py

+def setup_host_fullnames(config):
+    for host in config.validated_config['hosts']:
+        if 'fullname' not in host:
+            host['fullname'] = host['name']
+
+
+# Mine hosts out of the services like this:
+# 'static_dns': [{
+#   'fullname': 'avian.sch.bme.hu',
+#   'address': '10.0.209.160'
+# }]
+# Note: only first exporter's address!
+def setup_static_dns(config):
+    config.validated_config['static_dns'] = []
+    for host in config.validated_config['hosts']:
+        if len(host['exporters']) == 0:
+            continue
+        key = next(iter(host['exporters']))
+        exporter = host['exporters'][key]
+        config.validated_config['static_dns'].append({
+            'fullname': host['fullname'],
+            'address': exporter['addresses'][0]  # TODO support multiple addresses
+        })
+
+
+def setup_exporters(config, type):
+    exporters = []
+    for host in config.validated_config['hosts']:
+        if type not in host['exporters']:
+            continue
+
+        metrics_labels = [{
+            'name': 'facility',
+            'value': host['facility']
+        },
+            # {
+            #    'name': 'owner',
+            #    'value': host['owner']
+            # }
+        ]
+
+        if 'metrics_labels' in host:
+            for label in host['metrics_labels']:
+                key = next(iter(label))
+                metrics_labels.append({
+                    'name': key,
+                    'value': label[key]
+                })
+
+        exporters.append({
+            'target': host['fullname'],
+            'metrics_labels': metrics_labels
+        })
+    return exporters
+
+
+def search_config_by_host_fullname(fullname, config):
+    for host in config.validated_config['hosts']:
+        if host['fullname'] == fullname:
+            return host
+
+
+def setup_node_exporters(config):
+    config.validated_config['node_exporters'] = setup_exporters(config, 'node')
+    for exporter in config.validated_config['node_exporters']:
+        host = search_config_by_host_fullname(exporter['target'], config)
+        exporter['target'] += ':' + host['exporters']['node']['port']
+
+
+def setup_bind9_exporters(config):
+    config.validated_config['bind9_exporters'] = setup_exporters(config, 'bind9')
+    for exporter in config.validated_config['bind9_exporters']:
+        host = search_config_by_host_fullname(exporter['target'], config)
+        exporter['target'] += ':' + host['exporters']['bind9']['port']
+
+
+def setup_wmi_exporters(config):
+    config.validated_config['wmi_exporters'] = setup_exporters(config, 'wmi')
+    for exporter in config.validated_config['wmi_exporters']:
+        host = search_config_by_host_fullname(exporter['target'], config)
+        exporter['target'] += ':' + host['exporters']['wmi']['port']
+
+
+def setup_ping_exporters(config):
+    config.validated_config['ping_exporters'] = setup_exporters(config, 'ping')
+    # for exporter in services.validated_config['ping_exporters']:
+    #     host = search_config_by_host_fullname(exporter['target'])
+
+
+def setup_autogen_warning(config):
+    config.validated_config['autogen_warning'] = ""
+    for x in range(30):
+        config.validated_config['autogen_warning'] += "# !!! AUTOGENERATED ; DO NOT EDIT\n"

validation.py

+from schema import Schema, And, Optional
+
+
+# Returns the validated services (completed with default values)
+def validate_config(config):
+    # Schema validation rules
+    # source: https://pypi.org/project/schema/
+    schema = Schema({
+        'hosts': [{
+            'name': And(str, lambda str: 2 <= len(str) <= 30),
+            Optional('fullname'): And(str, lambda str: 2 <= len(str) <= 30),
+            'type': And(str, lambda str: str in ['physical-server', 'virtual-server', 'network-device']),
+            'owner': And(str,
+                         lambda str: str in ['kszk', 'sysadmin', 'neteam', 'hat', 'devteam', 'securiteam', 'misc']),
+            'facility': And(str, lambda str: str in ['web', 'auth', 'db', 'mail', 'dns', 'virtualization', 'storage',
+                                                     'monitoring', 'misc', 'dontcare',
+                                                     'core-net', 'server-net', 'endpoint-net']),
+            Optional('metrics_labels'): [{str: str}],
+            Optional('dependencies'): [str],
+            'exporters': {
+                Optional('ping'): {
+                    'addresses': [And(str, lambda str: 2 <= len(str) <= 50)]
+                },
+                Optional('node'): {
+                    'addresses': [And(str, lambda str: 2 <= len(str) <= 50)],
+                    Optional('port', default='9100'): str
+                },
+                Optional('bind9'): {
+                    'addresses': [And(str, lambda str: 2 <= len(str) <= 50)],
+                    Optional('port', default='9119'): str
+                },
+                Optional('wmi'): {
+                    'addresses': [And(str, lambda str: 2 <= len(str) <= 50)],
+                    Optional('port', default='9182'): str
+                },
+                Optional('snmp'): {
+                    'addresses': [And(str, lambda str: 2 <= len(str) <= 50)],
+                    'mib': And(str, lambda str: 2 <= len(str) <= 50),
+                }
+            },
+        }]
+    })
+
+    return schema.validate(config.raw_config)