From 3964f7d6609527763bcc3d39211f5f08c15c1d7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Robotka?= <robotka.adrian@gmail.com>
Date: Sun, 31 May 2020 18:32:36 +0200
Subject: [PATCH] add snippets functionality
---
build.py | 28 ++++++++++++++++++----------
rawconfig.py => config.py | 24 ++++--------------------
constants.py | 1 +
generator.py | 12 +++++++++---
helper.py | 28 ++++++++++++++++++++++++++++
pupak.py | 6 +++---
6 files changed, 63 insertions(+), 36 deletions(-)
rename rawconfig.py => config.py (70%)
create mode 100644 helper.py
diff --git a/build.py b/build.py
index 165149b..0bc17a4 100644
--- a/build.py
+++ b/build.py
@@ -1,31 +1,39 @@
"""Fill YAML-jinja2 templates, prepare the final data-model"""
import logging
+import jinja2
import yaml
-from jinja2 import Template
-from constants import APP_LOG_TAG
-from rawconfig import RawConfig
+from config import Config
+from constants import APP_LOG_TAG, TEMPLATE_PREFIX
+from helper import collect_snippets
class ConfigBuilder:
"""Fill YAML-jinja2 templates, prepare the final data-model"""
- service_definitions = []
- templates = {}
- def __init__(self, cfg: RawConfig):
+ def __init__(self, cfg: Config):
"""Instantiate ConfigBuilder"""
self.logger = logging.getLogger(APP_LOG_TAG)
- self.service_definitions = cfg.service_definitions
- self.templates = cfg.templates
+ self.config = cfg
def build(self):
"""Fill YAML-jinja2 templates, prepare the final data-model"""
- for service in self.service_definitions:
+ for service in self.config.service_definitions:
for job in service['scraping']:
- template = Template(job['template'])
+ self.logger.debug("Templating: " + service['service_name'] + ' -> ' + job['template'])
+ self.load_template(job)
+
+ env = jinja2.Environment(undefined=jinja2.DebugUndefined)
+ template = env.from_string(job['template'])
data = service.copy()
data['targets'] = job['targets']
+ data['snippet'] = collect_snippets(self.config.templates)
+
output = template.render(data)
# store generated prometheus job config
job['output_yaml'] = yaml.safe_load(output)
+
+ def load_template(self, job):
+ template_identifier = TEMPLATE_PREFIX + job['template']
+ job['template'] = self.config.templates[template_identifier]
diff --git a/rawconfig.py b/config.py
similarity index 70%
rename from rawconfig.py
rename to config.py
index 346f896..b3fab9f 100644
--- a/rawconfig.py
+++ b/config.py
@@ -4,20 +4,11 @@ from pathlib import Path
import yaml
-from constants import APP_LOG_TAG, SERVICE_PREFIX, TEMPLATE_PREFIX
+from constants import APP_LOG_TAG
+from helper import is_service_file, is_template_file
-def is_service_file(file_identifier: str) -> bool:
- """Hmm, is it a service definition file or not"""
- return file_identifier.startswith(SERVICE_PREFIX)
-
-
-def is_template_file(file_identifier: str) -> bool:
- """Hmm, is it a service template file or not"""
- return file_identifier.startswith(TEMPLATE_PREFIX)
-
-
-class RawConfig:
+class Config:
"""Read service YAML files and preload templates"""
service_definitions = []
templates = {}
@@ -29,7 +20,6 @@ class RawConfig:
base_path_len = len(str(path.absolute())) + 1
self.preload_service_files(path.rglob('*.yaml'), base_path_len)
self.preload_template_files(path.rglob('*.yaml.j2'), base_path_len)
- self.fill_templates()
def preload_service_files(self, path_glob, base_path_len: int):
"""Read all (service)YAML files in tha data directory recursively"""
@@ -44,6 +34,7 @@ class RawConfig:
file_identifier = str(yaml_path.absolute())[base_path_len:-len(".yaml.j2")]
if is_template_file(file_identifier):
data = open(str(yaml_path.absolute()), "r")
+ self.logger.debug("Load template file: " + str(yaml_path.absolute()))
self.templates[file_identifier] = data.read()
def read_yaml_file(self, yaml_path: Path):
@@ -55,10 +46,3 @@ class RawConfig:
except yaml.YAMLError as exc:
self.logger.error("Cannot load YAML file.")
self.logger.error(exc)
-
- def fill_templates(self):
- """Fill a service definition template field with its content"""
- for service in self.service_definitions:
- for job in service['scraping']:
- template_identifier = TEMPLATE_PREFIX + job['template']
- job['template'] = self.templates[template_identifier]
diff --git a/constants.py b/constants.py
index 7115878..4b59614 100644
--- a/constants.py
+++ b/constants.py
@@ -4,3 +4,4 @@ GENERATOR_OUTPUT_FOLDER = 'generated/'
OUTPUT_TEMPLATE_FOLDER = 'output-templates/'
SERVICE_PREFIX = 'services/'
TEMPLATE_PREFIX = 'service-templates/'
+SNIPPET_PREFIX = 'service-templates/snippet/'
diff --git a/generator.py b/generator.py
index e8dc223..2cf67bc 100644
--- a/generator.py
+++ b/generator.py
@@ -8,8 +8,9 @@ from pathlib import Path
import jinja2
import yaml
-from build import ConfigBuilder
+from config import Config
from constants import APP_LOG_TAG, OUTPUT_TEMPLATE_FOLDER, GENERATOR_OUTPUT_FOLDER
+from helper import collect_snippets
def autogen_warning():
@@ -23,7 +24,7 @@ def autogen_warning():
class Generator:
"""Generates output from templates and some data"""
- def __init__(self, data_folder: Path, cfg: ConfigBuilder):
+ def __init__(self, data_folder: Path, cfg: Config):
self.logger = logging.getLogger(APP_LOG_TAG)
self.data_folder = data_folder
self.config = cfg
@@ -34,6 +35,7 @@ class Generator:
data = {
'autogen_warning': autogen_warning(),
'generation_info': self.generation_info(),
+ 'snippet': collect_snippets(self.config.templates),
}
self.collect_scrape_configs(data)
self.generate_files(data)
@@ -65,12 +67,16 @@ class Generator:
)
def generate_files(self, data):
+ """Fill jinja2 template files with data"""
base = str(self.data_folder.absolute())
template_folder = base + '/' + OUTPUT_TEMPLATE_FOLDER
out_folder = base + '/' + GENERATOR_OUTPUT_FOLDER
template_loader = jinja2.FileSystemLoader(searchpath=template_folder)
- template_env = jinja2.Environment(loader=template_loader)
+ template_env = jinja2.Environment(
+ loader=template_loader,
+ undefined=jinja2.StrictUndefined
+ )
for filename in os.listdir(template_folder):
output = template_env.get_template(filename).render(data)
diff --git a/helper.py b/helper.py
new file mode 100644
index 0000000..35f2e16
--- /dev/null
+++ b/helper.py
@@ -0,0 +1,28 @@
+"""helper methods"""
+
+from constants import SERVICE_PREFIX, TEMPLATE_PREFIX, SNIPPET_PREFIX
+
+
+def is_service_file(file_identifier: str) -> bool:
+ """Hmm, is it a service definition file or not"""
+ return file_identifier.startswith(SERVICE_PREFIX)
+
+
+def is_template_file(file_identifier: str) -> bool:
+ """Hmm, is it a service template file or not"""
+ return file_identifier.startswith(TEMPLATE_PREFIX)
+
+
+def is_snippet_file(file_identifier: str) -> bool:
+ """Hmm, is it a snippet template file or not"""
+ return file_identifier.startswith(SNIPPET_PREFIX)
+
+
+def collect_snippets(templates):
+ """Get snippets from templates"""
+ snippets = {}
+ for identifier in templates:
+ if is_snippet_file(identifier):
+ snippet_name = identifier[len(SNIPPET_PREFIX):]
+ snippets[snippet_name] = templates[identifier]
+ return snippets
diff --git a/pupak.py b/pupak.py
index 3344373..31cf0f9 100644
--- a/pupak.py
+++ b/pupak.py
@@ -5,10 +5,10 @@ import sys
from pathlib import Path
from build import ConfigBuilder
+from config import Config
from constants import APP_LOG_TAG
from customlog import setup_logging
from generator import Generator
-from rawconfig import RawConfig
if __name__ == "__main__":
# Bootstrapping
@@ -24,7 +24,7 @@ if __name__ == "__main__":
data_folder = Path(sys.argv[1])
# Read service YAML files and preload templates
- raw = RawConfig(data_folder)
+ raw = Config(data_folder)
# Fill YAML-jinja2 templates,
# prepare the final data-model
@@ -35,5 +35,5 @@ if __name__ == "__main__":
# cfg.validate()
# print("Data scheme is VALID.")
- generator = Generator(data_folder, builder)
+ generator = Generator(data_folder, builder.config)
generator.ignite()
--
GitLab