diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index 7ab2f9621460a1fc2afb6a8e60a2ba869cabdac2..9e1c79cd7aa1730a8b03b5865c15cfaa10c67ac3 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -1,4 +1,4 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu/.devcontainer/base.Dockerfile
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu/.devcontainer/base.Dockerfile
# [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic
ARG VARIANT="hirsute"
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 6a078e871dde3007872950da06b453d59e7f20af..816edebda4e998a108367b277209f84d444c9de1 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,5 +1,5 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu
{
"name": "Ubuntu",
"build": {
@@ -25,6 +25,8 @@
// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode",
"features": {
- "python": "latest"
+ "docker-from-docker": "20.10",
+ "git": "latest",
+ "python": "3.10"
}
}
diff --git a/alice-ci/src/alice/config.py b/alice-ci/src/alice/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..3598fd392f61a5cd690383c41a4194f816a2d0a2
--- /dev/null
+++ b/alice-ci/src/alice/config.py
@@ -0,0 +1,38 @@
+import logging
+import os
+
+from alice.exceptions import ConfigException
+
+
+class ConfigHolder:
+ __instance = None
+ file_name = ".alice"
+
+ @staticmethod
+ def getInstance():
+ """ Static access method. """
+ if ConfigHolder.__instance is None:
+ ConfigHolder()
+ return ConfigHolder.__instance
+
+ def __init__(self):
+ """ Virtually private constructor. """
+ if ConfigHolder.__instance is not None:
+ raise Exception("This class is a singleton!")
+ else:
+ ConfigHolder.__instance = self
+ config = os.path.abspath(os.path.join(os.getcwd(), self.file_name))
+ self.vars = {}
+ if os.path.isfile(config):
+ with open(config) as f:
+ for line in f:
+ items = line.split("=")
+ if len(items) > 1:
+ self.vars[items[0]] = line.replace(f"{items[0]}=", "")
+ logging.debug(f"Loaded from {self.file_name}: {self.vars}")
+
+ def get(self, key):
+ try:
+ self.vars[key]
+ except KeyError:
+ raise ConfigException(f"{key} not defined in .conf!")
diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py
index e58789115f6817c680b8a271f508800a2932d280..11b7b41fc06f16821f70fbf4bcae3834cc7b15b2 100644
--- a/alice-ci/src/alice/runnerfactory.py
+++ b/alice-ci/src/alice/runnerfactory.py
@@ -3,6 +3,7 @@ from os.path import join, abspath
from alice.runners.pythonrunner import PythonRunner
from alice.runners.pypirunner import PyPiRunner
+from alice.runners.dockerrunner import DockerRunner
from alice.exceptions import ConfigException
@@ -21,7 +22,8 @@ class Factory():
# module = __import__("module_file")
# my_class = getattr(module, "class_name")
self.runnertypes = {"python": PythonRunner,
- "pypi": PyPiRunner}
+ "pypi": PyPiRunner,
+ "docker": DockerRunner}
logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}")
diff --git a/alice-ci/src/alice/runners/dockerrunner.py b/alice-ci/src/alice/runners/dockerrunner.py
index 50a3260f25ae81e207d31a225c29e1a11b30b1a6..3a84fed09d479413d1c6dcca5f601adff508721d 100644
--- a/alice-ci/src/alice/runners/dockerrunner.py
+++ b/alice-ci/src/alice/runners/dockerrunner.py
@@ -1 +1,223 @@
-# TODO Implement
+from enum import Enum
+import json
+import logging
+from os import path, getcwd
+import docker
+
+from alice.runners.pyutils import grab_from, gen_dict
+from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
+
+
+class ImageSource(Enum):
+ NONE = 1
+ BUILD = 2
+ PULL = 3
+
+
+def get_user(config, default):
+ if "credentials" in config:
+ if "username" in config["credentials"]:
+ data = config["credentials"]["username"]
+ if isinstance(data, str):
+ return data
+ else:
+ return grab_from(data)
+ return default
+
+
+def get_pass(config, default):
+ if "credentials" in config:
+ if "password" in config["credentials"]:
+ data = config["credentials"]["password"]
+ if isinstance(data, str):
+ return data
+ else:
+ return grab_from(data)
+ return default
+
+
+def get_provider(config, default, default_type):
+ if "image" in config:
+ build = False
+ pull = False
+ candidate_type = default_type
+ if "build" in config["image"]:
+ build = True
+ if default_type == ImageSource.BUILD:
+ candidate = default.copy(config["image"]["build"])
+ else:
+ candidate = Builder(config["image"]["build"])
+ candidate_type = ImageSource.BUILD
+ elif "pull" in config["image"]:
+ pull = True
+ if default_type == ImageSource.PULL:
+ candidate = default.copy(config["image"]["pull"])
+ else:
+ candidate = Puller(config["image"]["pull"])
+ candidate_type = ImageSource.PULL
+
+ if build and pull:
+ raise ConfigException("[DockerRunner] Can't build and pull the same image!")
+
+ return candidate, candidate_type
+ return default, default_type
+
+
+class Tagger:
+ def __init__(self, config={}) -> None:
+ self.name = config.get("name", None)
+ self.username = get_user(config, None)
+ self.password = get_pass(config, None)
+ self.publish = config.get("publish", False)
+
+ def copy(self, job_config):
+ t = Tagger()
+ t.name = job_config.get("name", self.name)
+ t.username = get_user(job_config, self.username)
+ t.password = get_pass(job_config, self.password)
+ t.publish = job_config.get("publish", self.publish)
+ return t
+
+ def __str__(self) -> str:
+ data = {
+ "name": self.name,
+ "publish": self.publish,
+ "credentials": {
+ "username": self.username,
+ "password": self.password
+ }
+ }
+ return f"{data}"
+
+
+class Builder():
+ def __init__(self, config) -> None:
+ self.dir = path.abspath(config.get("dir", getcwd()))
+ self.dockerfile = config.get("dockerfile", None)
+ self.name = config.get("name", None)
+ self.args = gen_dict(config.get("args", []))
+
+ def copy(self, job_config):
+ b = Builder({})
+ b.dir = path.abspath(path.join(self.dir, job_config.get("dir", ".")))
+ b.dockerfile = job_config.get("dockerfile", self.dockerfile)
+ b.name = job_config.get("name", self.name)
+ b.args = self.args.copy().update(gen_dict(job_config.get("args", [])))
+ return b
+
+ def __str__(self) -> str:
+ data = {
+ "type": "builder",
+ "dir": self.dir,
+ "dockerfile": self.dockerfile,
+ "name": self.name,
+ "args": self.args
+ }
+ return json.dumps(data)
+
+ def prepare(self, client):
+ print(f"[DockerRunner] Building image {self.name}")
+ if self.dockerfile is None:
+ self.dockerfile = "Dockerfile"
+ try:
+ image, log = client.images.build(path=self.dir,
+ dockerfile=self.dockerfile,
+ tag=self.name,
+ buildargs=self.args,
+ labels={"builder": "alice-ci"})
+ for i in log:
+ logging.debug(i)
+ return image
+ except docker.errors.BuildError as e:
+ raise RunnerError(f"[DockerRunner] Build failed: {e}")
+ except docker.errors.APIError as e:
+ raise RunnerError(f"[DockerRunner] Error: {e}")
+
+
+class Puller():
+ def __init__(self, config={}) -> None:
+ self.name = config.get("name", None)
+ self.username = get_user(config, None)
+ self.password = get_pass(config, None)
+
+ def copy(self, job_config={}):
+ p = Puller()
+ p.name = job_config.get("name", self.name)
+ p.username = get_user(job_config, self.username)
+ p.password = get_pass(job_config, self.password)
+
+ def __str__(self) -> str:
+ data = {
+ "name": self.name,
+ "credentials": {
+ "username": self.username,
+ "password": self.password
+ }
+ }
+ return f"{data}"
+
+ def prepare(self, client):
+ print(f"[DockerRunner] Pulling image {self.name}")
+ return client.images.pull(self.name)
+
+
+class DockerConfig:
+ def __init__(self, config={}) -> None:
+ self.username = get_user(config, None)
+ self.password = get_pass(config, None)
+ self.image_provider, self.provider_type = get_provider(config, None, ImageSource.NONE)
+ self.tagger = Tagger(config.get("tag", {}))
+ self.commands = config.get("commands", [])
+
+ def copy(self, job_config={}):
+ d = DockerConfig()
+ d.username = get_user(job_config, self.username)
+ d.password = get_pass(job_config, self.password)
+ d.image_provider, d.provider_type = get_provider(job_config, self.image_provider, self.provider_type)
+ d.tagger = self.tagger.copy(job_config.get("tag", {}))
+ d.commands = self.commands.copy() + job_config.get("commands", [])
+ return d
+
+ def __str__(self) -> str:
+ data = {
+ "credentials": {
+ "username": {self.username},
+ "password": {self.password}
+ },
+ "image": self.image_provider.__str__(),
+ "commands": self.commands,
+ "tag": self.tagger.__str__()
+ }
+ return f"{data}"
+
+
+class DockerRunner():
+ def __init__(self, config) -> None:
+ logging.info("[DockerRunner] Initializing")
+ self.config = DockerConfig(config)
+ self.client = docker.from_env()
+
+ def run(self, job_spec):
+ job_config = self.config.copy(job_spec)
+ if job_config.image_provider is None:
+ raise RunnerError("[DockerRunner] No image provider configured!")
+ image = job_config.image_provider.prepare(self.client)
+ logging.info(f"[DockerRunner] Image: {image.tags} ({image.id})")
+
+ if len(job_config.commands) > 0:
+ container = self.client.containers.run(image=image.id,
+ entrypoint=["sleep", "infinity"],
+ detach=True,
+ auto_remove=True)
+ try:
+ for i in job_config.commands:
+ command = ["/bin/sh", "-c", i]
+ logging.debug(f"[DockerRunner] Command array: {command}")
+ code, output = container.exec_run(cmd=command)
+ for line in output.decode("UTF-8").splitlines():
+ print(f"[{job_spec['name']}] {line}")
+ if code != 0:
+ raise NonZeroRetcode(f"Command {i} returned code {code}")
+ finally:
+ if container is not None:
+ container.stop()
diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py
index 5f0879cdd7e7c6c21f6d1f972b7d051cf439ba78..10918bd06d3b5de8b83e6f9e332555d1267577cb 100644
--- a/alice-ci/src/alice/runners/pypirunner.py
+++ b/alice-ci/src/alice/runners/pypirunner.py
@@ -7,17 +7,10 @@ import sys
from urllib import request, error
from pkg_resources import parse_version
from os import environ, path
-from alice.runners.pyutils import PackageManager, glob
+from alice.runners.pyutils import PackageManager, glob, grab_from
from alice.exceptions import ConfigException, RunnerError
-def grab_from(target):
- if "from_env" in target:
- return environ[target["from_env"]]
- else:
- raise ConfigException(f"Unsupported grabber: {target.keys()}")
-
-
def get_uri(config, default):
url = config.get("repo", {}).get("uri", default)
if url is not None:
diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py
index 6b3f6f1a2f38294ce7f1bb9baa947e9c376396cb..0446cb461d70ce762c21fa5fedbdd457baea6a85 100644
--- a/alice-ci/src/alice/runners/pyutils.py
+++ b/alice-ci/src/alice/runners/pyutils.py
@@ -6,6 +6,7 @@ from pkg_resources import parse_version
import re
from alice.exceptions import RunnerError, ConfigException
+from alice.config import ConfigHolder
class PackageManager:
@@ -24,7 +25,7 @@ class PackageManager:
raise Exception("This class is a singleton!")
else:
PackageManager.__instance = self
- self.package_list = self.__get_packages()
+ self.package_list = self.__get_packages()
def __get_packages(self):
packages = {}
@@ -111,3 +112,33 @@ def glob_command(command, workdir):
for item in command:
new_command += glob(item, workdir)
return new_command
+
+
+def grab_from(target):
+ if "from_env" in target:
+ try:
+ return os.environ[target["from_env"]]
+ except KeyError:
+ raise ConfigException(f"Env var unset: {target['from_env']}")
+ elif "from_cfg" in target:
+ ConfigHolder.getInstance().get(target["from_cfg"])
+ else:
+ raise ConfigException(f"Unsupported grabber: {target.keys()}")
+
+
+def gen_dict(list_of_dicts):
+ """
+ Generates a dictionary from a list of dictionaries composed of
+ 'name' and 'value' keys.
+
+ [{'name': 'a', 'value': 'b'}] => {'a': 'b'}
+ """
+ return_dict = {}
+
+ for _dict in list_of_dicts:
+ try:
+ return_dict[_dict["name"]] = _dict["value"]
+ except KeyError:
+ raise ConfigException(f"Invalid dict item: {_dict}")
+
+ return return_dict
diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml
index 108ba9299642e0ecea4ad0543354ef719ba033d6..b5f086cad0c3f7df8599e1612bb6def7c2467278 100644
--- a/ci-examples/full.yaml
+++ b/ci-examples/full.yaml
@@ -15,6 +15,11 @@ runners:
dependencies:
- flake8
- build
+ docker:
+ credentials:
+ username: D
+ password: D
+
jobs:
- name: env
type: python
@@ -46,6 +51,36 @@ jobs:
from_env: PYPIPASS
packages:
- alice-ci
+ - name: "image"
+ type: docker
+ credentials:
+ username: A
+ #password: B
+ image:
+ build:
+ dir: ci-examples/images/hello
+ #dockerfile: ci-examples/images/hello/Dockerfile
+ dockerfile: Dockerfile
+ name: "sssss"
+ args:
+ - name: CIPASS
+ value: NONE
+ #pull:
+ #name: python:latest
+ #credentials:
+ #username: PASS
+ #password: WORD
+ commands:
+ - which python3
+ - /usr/bin/python3 --version
+ - date
+ tag:
+ publish: true
+ name: published name with repo and everything
+ credentials:
+ username: B
+ password: B
+
pipelines:
default:
- lint
diff --git a/ci-examples/images/hello/Dockerfile b/ci-examples/images/hello/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..17cd4aa2e1b015ea1d9dc0ec1c1b4ecb5497d0aa
--- /dev/null
+++ b/ci-examples/images/hello/Dockerfile
@@ -0,0 +1,9 @@
+FROM ubuntu:latest
+
+RUN apt update && apt install -y python3
+
+ADD hello.py /opt/hello.py
+
+#ENTRYPOINT [ "/bin/sh", "-c" ]
+
+#CMD ["/usr/local/python/bin/python3", "/opt/hello.py"]
\ No newline at end of file
diff --git a/ci-examples/images/hello/hello.py b/ci-examples/images/hello/hello.py
new file mode 100644
index 0000000000000000000000000000000000000000..617e813c91df405b165643c8ed49c3f2749a6296
--- /dev/null
+++ b/ci-examples/images/hello/hello.py
@@ -0,0 +1,2 @@
+if __name__ == "__main__":
+ print("Hi Mom!")
diff --git a/docs/runners/docker.md b/docs/runners/docker.md
new file mode 100644
index 0000000000000000000000000000000000000000..0ae0853d5aca36e45d5d53c16d4d71fb274a1d6b
--- /dev/null
+++ b/docs/runners/docker.md
@@ -0,0 +1,26 @@
+# Schema
+
+```
+name: ""
+type: docker
+credentials: - global ...ish
+ username
+ password
+image: - to use, pull, run
+ build:
+ dir:
+ dockerfile:
+ name: - defaults to step name
+ args:
+ - name:
+ - value:
+ pull: - pulls, current working image - mutually exclusive with build
+ name:
+ credentials: - optional
+command: - overwrite, not append
+ - ...
+tag:
+ publish: true
+ name: - published name with repo and everything
+ credentials:
+```
\ No newline at end of file