add snippets functionality

build.py

 """Fill YAML-jinja2 templates, prepare the final data-model"""
 import logging
 
+import jinja2
 import yaml
-from jinja2 import Template
 
-from constants import APP_LOG_TAG
-from rawconfig import RawConfig
+from config import Config
+from constants import APP_LOG_TAG, TEMPLATE_PREFIX
+from helper import collect_snippets
 
 
 class ConfigBuilder:
     """Fill YAML-jinja2 templates, prepare the final data-model"""
-    service_definitions = []
-    templates = {}
 
-    def __init__(self, cfg: RawConfig):
+    def __init__(self, cfg: Config):
         """Instantiate ConfigBuilder"""
         self.logger = logging.getLogger(APP_LOG_TAG)
-        self.service_definitions = cfg.service_definitions
-        self.templates = cfg.templates
+        self.config = cfg
 
     def build(self):
         """Fill YAML-jinja2 templates, prepare the final data-model"""
-        for service in self.service_definitions:
+        for service in self.config.service_definitions:
             for job in service['scraping']:
-                template = Template(job['template'])
+                self.logger.debug("Templating: " + service['service_name'] + ' -> ' + job['template'])
+                self.load_template(job)
+
+                env = jinja2.Environment(undefined=jinja2.DebugUndefined)
+                template = env.from_string(job['template'])
                 data = service.copy()
                 data['targets'] = job['targets']
+                data['snippet'] = collect_snippets(self.config.templates)
+
                 output = template.render(data)
                 # store generated prometheus job config
                 job['output_yaml'] = yaml.safe_load(output)
+
+    def load_template(self, job):
+        template_identifier = TEMPLATE_PREFIX + job['template']
+        job['template'] = self.config.templates[template_identifier]

rawconfig.py → config.py

@@ -4,20 +4,11 @@ from pathlib import Path
 
 import yaml
 
-from constants import APP_LOG_TAG, SERVICE_PREFIX, TEMPLATE_PREFIX
+from constants import APP_LOG_TAG
+from helper import is_service_file, is_template_file
 
 
-def is_service_file(file_identifier: str) -> bool:
-    """Hmm, is it a service definition file or not"""
-    return file_identifier.startswith(SERVICE_PREFIX)
-
-
-def is_template_file(file_identifier: str) -> bool:
-    """Hmm, is it a service template file or not"""
-    return file_identifier.startswith(TEMPLATE_PREFIX)
-
-
-class RawConfig:
+class Config:
     """Read service YAML files and preload templates"""
     service_definitions = []
     templates = {}
@@ -29,7 +20,6 @@ class RawConfig:
         base_path_len = len(str(path.absolute())) + 1
         self.preload_service_files(path.rglob('*.yaml'), base_path_len)
         self.preload_template_files(path.rglob('*.yaml.j2'), base_path_len)
-        self.fill_templates()
 
     def preload_service_files(self, path_glob, base_path_len: int):
         """Read all (service)YAML files in tha data directory recursively"""
@@ -44,6 +34,7 @@ class RawConfig:
             file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml.j2")]
             if is_template_file(file_identifier):
                 data = open(str(yaml_path.absolute()), "r")
+                self.logger.debug("Load template file: " + str(yaml_path.absolute()))
                 self.templates[file_identifier] = data.read()
 
     def read_yaml_file(self, yaml_path: Path):
@@ -55,10 +46,3 @@ class RawConfig:
             except yaml.YAMLError as exc:
                 self.logger.error("Cannot load YAML file.")
                 self.logger.error(exc)
-
-    def fill_templates(self):
-        """Fill a service definition template field with its content"""
-        for service in self.service_definitions:
-            for job in service['scraping']:
-                template_identifier = TEMPLATE_PREFIX + job['template']
-                job['template'] = self.templates[template_identifier]

constants.py

@@ -4,3 +4,4 @@ GENERATOR_OUTPUT_FOLDER = 'generated/'
 OUTPUT_TEMPLATE_FOLDER = 'output-templates/'
 SERVICE_PREFIX = 'services/'
 TEMPLATE_PREFIX = 'service-templates/'
+SNIPPET_PREFIX = 'service-templates/snippet/'

generator.py

@@ -8,8 +8,9 @@ from pathlib import Path
 import jinja2
 import yaml
 
-from build import ConfigBuilder
+from config import Config
 from constants import APP_LOG_TAG, OUTPUT_TEMPLATE_FOLDER, GENERATOR_OUTPUT_FOLDER
+from helper import collect_snippets
 
 
 def autogen_warning():
@@ -23,7 +24,7 @@ def autogen_warning():
 class Generator:
     """Generates output from templates and some data"""
 
-    def __init__(self, data_folder: Path, cfg: ConfigBuilder):
+    def __init__(self, data_folder: Path, cfg: Config):
         self.logger = logging.getLogger(APP_LOG_TAG)
         self.data_folder = data_folder
         self.config = cfg
@@ -34,6 +35,7 @@ class Generator:
         data = {
             'autogen_warning': autogen_warning(),
             'generation_info': self.generation_info(),
+            'snippet': collect_snippets(self.config.templates),
         }
         self.collect_scrape_configs(data)
         self.generate_files(data)
@@ -65,12 +67,16 @@ class Generator:
         )
 
     def generate_files(self, data):
+        """Fill jinja2 template files with data"""
         base = str(self.data_folder.absolute())
         template_folder = base + '/' + OUTPUT_TEMPLATE_FOLDER
         out_folder = base + '/' + GENERATOR_OUTPUT_FOLDER
 
         template_loader = jinja2.FileSystemLoader(searchpath=template_folder)
-        template_env = jinja2.Environment(loader=template_loader)
+        template_env = jinja2.Environment(
+            loader=template_loader,
+            undefined=jinja2.StrictUndefined
+        )
 
         for filename in os.listdir(template_folder):
             output = template_env.get_template(filename).render(data)

helper.py

+"""helper methods"""
+
+from constants import SERVICE_PREFIX, TEMPLATE_PREFIX, SNIPPET_PREFIX
+
+
+def is_service_file(file_identifier: str) -> bool:
+    """Hmm, is it a service definition file or not"""
+    return file_identifier.startswith(SERVICE_PREFIX)
+
+
+def is_template_file(file_identifier: str) -> bool:
+    """Hmm, is it a service template file or not"""
+    return file_identifier.startswith(TEMPLATE_PREFIX)
+
+
+def is_snippet_file(file_identifier: str) -> bool:
+    """Hmm, is it a snippet template file or not"""
+    return file_identifier.startswith(SNIPPET_PREFIX)
+
+
+def collect_snippets(templates):
+    """Get snippets from templates"""
+    snippets = {}
+    for identifier in templates:
+        if is_snippet_file(identifier):
+            snippet_name = identifier[len(SNIPPET_PREFIX):]
+            snippets[snippet_name] = templates[identifier]
+    return snippets

pupak.py

@@ -5,10 +5,10 @@ import sys
 from pathlib import Path
 
 from build import ConfigBuilder
+from config import Config
 from constants import APP_LOG_TAG
 from customlog import setup_logging
 from generator import Generator
-from rawconfig import RawConfig
 
 if __name__ == "__main__":
     # Bootstrapping
@@ -24,7 +24,7 @@ if __name__ == "__main__":
     data_folder = Path(sys.argv[1])
 
     # Read service YAML files and preload templates
-    raw = RawConfig(data_folder)
+    raw = Config(data_folder)
 
     # Fill YAML-jinja2 templates,
     # prepare the final data-model
@@ -35,5 +35,5 @@ if __name__ == "__main__":
     # cfg.validate()
     # print("Data scheme is VALID.")
 
-    generator = Generator(data_folder, builder)
+    generator = Generator(data_folder, builder.config)
     generator.ignite()