7_runner-interface (#12)

Fix #7

Co-authored-by: gyulaid <gyulaid@gyulai.cloud>
Reviewed-on: https://git.gyulai.cloud/gyulaid/alice/pulls/12


Co-authored-by: Daniel Gyulai <gyulaid@gyulai.cloud>
Co-committed-by: Daniel Gyulai <gyulaid@gyulai.cloud>

.devcontainer/Dockerfile

+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu/.devcontainer/base.Dockerfile
+
+# [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic
+ARG VARIANT="hirsute"
+FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT}
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+

.devcontainer/devcontainer.json

+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu
+{
+	"name": "Ubuntu",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick an Ubuntu version: hirsute, focal, bionic
+		// Use hirsute or bionic on local arm64/Apple Silicon.
+		"args": { "VARIANT": "focal" }
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "uname -a",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"features": {
+		"python": "latest"
+	}
+}

alice-ci/setup.cfg

 [metadata]
 name = alice-ci
-version = 0.0.6
+version = 0.0.7
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md
@@ -16,11 +16,15 @@ classifiers =
 [options]
 package_dir =
     = src
-packages = find:
+packages = alice
 python_requires = >=3.6
 install_requires =
     PyYAML==6.0
     virtualenv==20.14.0
 
+[options.entry_points]
+console_scripts =
+    alice = alice.cli:main
+
 [options.packages.find]
 where = src
\ No newline at end of file

alice-ci/src/alice/cli.py

@@ -6,7 +6,7 @@ from alice.runnerfactory import Factory
 from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
 
 
-def gen_env(self, param_list):
+def gen_env(param_list):
     env_vars = {}
     for item in param_list:
         item_parts = item.split("=")
@@ -24,22 +24,21 @@ def parse_jobs(args):
             envs = gen_env(args.env)
             if args.verbose:
                 print(f"[Alice] Env vars from CLI: {envs}")
-            factory.update_runners({"env": envs})
-        jobParser = ConfigParser(args.input, factory, args.verbose)
+        jobParser = ConfigParser(args.input, factory, gen_env(args.env), args.verbose)
 
-        print("Begin pipeline steps...")
+        print("[Alice] Begin pipeline steps")
         for step in args.steps:
             if step in jobParser.jobs:
                 status = jobParser.execute_job(step)
-                print(f"[Step] {step}: {status}")
+                print(f"[Alice][Step] {step}: {status}")
             else:
-                print(f"Step {step} not found in {args.input}")
+                raise ConfigException(f"Step {step} not found in {args.input}")
                 exit(1)
     except ConfigException as e:
         print(f"Configuration error-> {e}")
         exit(1)
     except NonZeroRetcode:
-        print("FAILED")
+        print("[Alice] FAILED")
         exit(1)
     except RunnerError as e:
         print(f"RunnerError-> {e}")

alice-ci/src/alice/runnerfactory.py

-from os import getcwd
-
 from alice.runners.pythonrunner import PythonRunner
 from alice.exceptions import ConfigException
 
@@ -8,12 +6,13 @@ class Factory():
     def __init__(self, verbose) -> None:
         self.verbose = verbose
         self.runnertypes = self.__load_runners()
+        self.runner_configs = {}
         self.runners = {}
-        self.workdir = getcwd()
         self.globals = {}
 
     def __load_runners(self):
         # TODO: Runners can be imported via cli too
+        # https://git.gyulai.cloud/gyulaid/alice/issues/4
         # module = __import__("module_file")
         # my_class = getattr(module, "class_name")
         runners = {"python": PythonRunner}
@@ -25,15 +24,11 @@ class Factory():
     def set_globals(self, globals):
         self.globals = globals
 
-    def update_globals(self, update):
-        if "env" in update:
-            self.globals["env"].update(update["env"])
-
     def update_runners(self, config):
         for runnertype, runnerconfig in config.items():
             if runnertype != "global":
                 if (self.verbose):
-                    print(f"[Alice] Configuring runner {runnertype}")
+                    print(f"[Alice] Configuring runner: {runnertype}")
                 self.get_runner(runnertype).update_config(runnerconfig)
 
     def get_runner(self, runnertype):
@@ -41,9 +36,10 @@ class Factory():
             if runnertype in self.runnertypes:
                 if (self.verbose):
                     print(f"[Alice] Initializing runner: {runnertype}")
-                self.runners[runnertype] = self.runnertypes[runnertype](self.workdir,
-                                                                        self.globals,
-                                                                        self.verbose)
+                params = {
+                    "verbose": self.verbose
+                }
+                self.runners[runnertype] = self.runnertypes[runnertype](params, self.globals)
             else:
                 raise ConfigException(f"Invalid runner type: {runnertype}")
         return self.runners[runnertype]

alice-ci/src/alice/runners/__init__.py

+# flake8: noqa F401
 from alice.runners.pythonrunner import PythonRunner
-
-__all__ = ["PythonRunner"]

alice-ci/src/alice/runners/pythonrunner.py

@@ -2,21 +2,18 @@ import subprocess
 import os
 import sys
 import shlex
-from tabnanny import verbose
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
 
 
-# same venv across all runs!
 class PythonRunner():
-    def __init__(self, workdir, defaults, verbose) -> None:
-        self.workdir = workdir
-        self.virtual_dir = os.path.abspath(os.path.join(workdir, "venv"))
-        self.config = defaults
-        self.env_vars = os.environ.copy()
-        for env_var in defaults["env"]:
-            self.env_vars[env_var["name"]] = env_var["value"]
-        self.verbose = verbose
+    def __init__(self, params, user_defaults) -> None:
+        self.verbose = params["verbose"]
+        if self.verbose:
+            print("[PythonRunner] Initializing")
+        self.workdir = user_defaults["workdir"]
+        self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv"))
+        self.config = user_defaults
 
         self.__init_venv()
 
@@ -27,6 +24,7 @@ class PythonRunner():
             self.vpython = os.path.join(self.virtual_dir, "bin", "python3")
 
         if not os.path.exists(self.vpython):
+            print("[PythonRunner] Initializing venv")
             with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir],
                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
                 p.wait()
@@ -54,7 +52,7 @@ class PythonRunner():
                         raise(RunnerError(f"[PythonRunner] Could not install dependency: {dependency} ({p.returncode})"))
         if "env" in config:
             for env_var in config["env"]:
-                self.env_vars[env_var["name"]] = env_var["value"]
+                self.config["env"][env_var["name"]] = env_var["value"]
         if "workdir" in config and config["workdir"] is not None:
             self.workdir = os.path.join(self.workdir, config["workdir"])
 
@@ -91,7 +89,7 @@ class PythonRunner():
             pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"]))
         else:
             pwd = self.workdir
-        run_env = self.env_vars.copy()
+        run_env = self.config["env"].copy()
         if "env" in job_spec:
             for env_var in job_spec["env"]:
                 run_env[env_var["name"]] = env_var["value"]
@@ -101,7 +99,10 @@ class PythonRunner():
                 if self.verbose:
                     print(f"[PythonRunner] Raw command: {command}")
                 # TODO: only split if command is not an array
+                if "*" in command:
                     run_command = self.__ghetto_glob(shlex.split(command), pwd)
+                else:
+                    run_command = shlex.split(command)
                 if self.verbose:
                     print(f"[PythonRunner] Command to execute: {run_command}")
                     print(f"[PythonRunner] Workdir: {pwd}")

alice-ci/src/alice/utils.py

-import os
+from os import getcwd, path, environ
 import subprocess
 import yaml
 
@@ -6,26 +6,29 @@ from alice.exceptions import ConfigException
 
 
 class ConfigParser:
-    def __init__(self, file_path, factory, verbose=False) -> None:
+    def __init__(self, file_path, factory, cli_env_vars, verbose=False) -> None:
         self.verbose = verbose
         with open(file_path) as f:
             self.config = yaml.safe_load(f)
         self.factory = factory
-        self.factory.set_globals(self.__gen_globals())
+        self.factory.set_globals(self.__gen_globals(cli_env_vars))
         if "runners" in self.config:
             self.factory.update_runners(self.config["runners"])
         self.jobs = self.__get_jobs()
 
-    # Initialize env, workdir if not present
-    def __gen_globals(self):
+    # Initialize env and workdir if not present in global
+    def __gen_globals(self, cli_vars):
+        env_vars = environ.copy()
+        env_vars.update(cli_vars)
         globals = {
-            "env": [],
-            "workdir": None
+            "env": env_vars,
+            "workdir": getcwd()
         }
         if "runners" in self.config:
             if "global" in self.config["runners"]:
                 if "env" in self.config["runners"]["global"]:
-                    globals["env"] = self.config["runners"]["global"]["env"]
+                    for var in self.config["runners"]["global"]["env"]:
+                        globals["env"][var["name"]] = var["value"]
                 if "workdir" in self.config["runners"]["global"]:
                     globals["workdir"] = self.config["runners"]["global"]["workdir"]
 
@@ -52,20 +55,20 @@ class ConfigParser:
         try:
             target = changes["branch"]
             paths = []
-            for path in changes["paths"]:
-                paths.append(os.path.abspath(path))
+            for _path in changes["paths"]:
+                paths.append(path.abspath(_path))
             # TODO: Error handling
             command = ["git", "diff", "--name-only", target]
             with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
                 p.wait()
                 for line in p.stdout:
-                    change_path = os.path.abspath(line.decode("UTF-8").strip())
-                    for path in paths:
-                        spec_path = os.path.abspath(path)
+                    change_path = path.abspath(line.decode("UTF-8").strip())
+                    for _path in paths:
+                        spec_path = path.abspath(_path)
                         if change_path.startswith(spec_path):
                             if self.verbose:
                                 print(f"[Alice] Modified file: {change_path}")
-                                print(f"[Alice] Path match: {path}")
+                                print(f"[Alice] Path match: {_path}")
                             return True
         except KeyError:
             raise ConfigException(f"Invalid 'changes' config: {changes}")
@@ -83,4 +86,3 @@ class ConfigParser:
                 return "SUCCESS"
             else:
                 return "SKIP, no change detected"
-