diff --git a/.gitignore b/.gitignore
index f4f8132d0eda2ae2ca1e9c58209cf95c0eeba9a4..817529a66d25d8b50ed3e51bd824c85d27a36c1e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -138,3 +138,5 @@ dmypy.json
# Cython debug symbols
cython_debug/
+# persistency dir
+.alice
diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg
index a4c9d71b2dd4408a1bc1197eea02900411772a75..6ae9541073ffc5093fe33f9476874f687e8cbe78 100644
--- a/alice-ci/setup.cfg
+++ b/alice-ci/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = alice-ci
-version = 0.0.11
+version = 0.0.12
author = Daniel Gyulai
description = Alice CI framework
long_description = file: README.md
diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py
index 563005542da8aa7092851b94aacb76cc7225ab2b..8386b15832095a00816e5fd51471bf7a52090588 100644
--- a/alice-ci/src/alice/cli.py
+++ b/alice-ci/src/alice/cli.py
@@ -39,7 +39,7 @@ def parse_jobs(args):
def main():
parser = argparse.ArgumentParser(prog="alice")
parser.add_argument("steps", nargs='*', default=["default"])
- parser.add_argument("-i", "--input", default="alice-ci.yaml")
+ parser.add_argument("-i", "--input", default=".alice-ci.yaml")
parser.add_argument("-e", "--env", nargs='*', default=[])
parser.add_argument("-a", "--addrunner", nargs='*', default=[])
parser.add_argument('--verbose', '-v', action='count', default=0)
@@ -51,6 +51,9 @@ def main():
if not os.path.isfile(args.input):
print(f"No such file: {args.input}")
exit(1)
+ persistency_path = os.path.join(os.getcwd(), ".alice")
+ if not os.path.isdir(persistency_path):
+ os.mkdir(persistency_path)
parse_jobs(args)
diff --git a/alice-ci/src/alice/config.py b/alice-ci/src/alice/config.py
index d0d473b1a666077106f55f042f9921c1278341fd..a904c59b0001079629b16e476445ace01be8f07d 100644
--- a/alice-ci/src/alice/config.py
+++ b/alice-ci/src/alice/config.py
@@ -6,7 +6,7 @@ from .exceptions import ConfigException
class ConfigHolder:
__instance = None
- file_name = ".alice"
+ file_name = os.path.join(os.getcwd(), ".alice", "vars")
@staticmethod
def getInstance():
@@ -25,7 +25,8 @@ class ConfigHolder:
self.vars = {}
if os.path.isfile(config):
with open(config) as f:
- for line in f:
+ for _line in f:
+ line = _line.strip()
items = line.split("=")
if len(items) > 1:
self.vars[items[0]] = line.replace(f"{items[0]}=", "")
@@ -33,6 +34,18 @@ class ConfigHolder:
def get(self, key):
try:
- self.vars[key]
+ return self.vars[key]
except KeyError:
raise ConfigException(f"{key} not defined in .conf!")
+
+ def set(self, key, value):
+ self.vars[key] = value
+ self.commit()
+
+ def soft_set(self, key, value):
+ self.vars[key] = value
+
+ def commit(self):
+ with open(self.file_name, 'w') as f:
+ for k, v in self.vars.items():
+ f.write(f"{k}={v if v is not None else ''}\n")
diff --git a/alice-ci/src/alice/configparser.py b/alice-ci/src/alice/configparser.py
index d65b092619aabfa02d63886b812d66aee903805f..b2ec886c883c05c9925db1e3eb24546f79e0830e 100644
--- a/alice-ci/src/alice/configparser.py
+++ b/alice-ci/src/alice/configparser.py
@@ -81,8 +81,8 @@ class ConfigParser:
def execute_pipeline(self, pipeline_name):
if pipeline_name in self.pipelines:
print(f"[Alice][Pipeline] {pipeline_name}: Start")
- for job in self.pipelines[pipeline_name]:
- self.execute_job(job)
+ for task in self.pipelines[pipeline_name]:
+ self.execute(task)
print(f"[Alice][Pipeline] {pipeline_name}: Success")
def execute_job(self, job_name):
diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py
index 424d213decd945c9fc594daeddfbe5040fdc0508..3661d251e43d1d332fde1472883e5bbb22db0f79 100644
--- a/alice-ci/src/alice/runnerfactory.py
+++ b/alice-ci/src/alice/runnerfactory.py
@@ -4,6 +4,7 @@ from os.path import join, abspath
from .runners.pythonrunner import PythonRunner
from .runners.pypirunner import PyPiRunner
from .runners.dockerrunner import DockerRunner
+from .runners.pypirepo import PypiRepoRunner
from .exceptions import ConfigException
@@ -23,7 +24,8 @@ class Factory():
# my_class = getattr(module, "class_name")
self.runnertypes = {"python": PythonRunner,
"pypi": PyPiRunner,
- "docker": DockerRunner}
+ "docker": DockerRunner,
+ "pypirepo": PypiRepoRunner}
logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}")
diff --git a/alice-ci/src/alice/runners/pypirepo.py b/alice-ci/src/alice/runners/pypirepo.py
new file mode 100644
index 0000000000000000000000000000000000000000..3fd6beebd7a6062d91b4350dceaa7fad43a9506b
--- /dev/null
+++ b/alice-ci/src/alice/runners/pypirepo.py
@@ -0,0 +1,117 @@
+import logging
+import docker
+from os.path import join, isdir
+from os import getcwd, mkdir
+import os
+
+from ..exceptions import RunnerError
+from ..config import ConfigHolder
+
+pipconf = """[global]
+index-url = URL
+trusted-host = BASE
+ pypi.org
+extra-index-url= http://pypi.org/simple"""
+
+
+class RepoConfig:
+ def __init__(self, config={}) -> None:
+ self.port = config.get("port", 8888)
+ self.enabled = config.get("enabled", True)
+ self.container_name = config.get("container_name", "alice-pypiserver")
+
+ def copy(self, job_config):
+ r = RepoConfig()
+ r.container_name = job_config.get("container_name", self.container_name)
+ r.enabled = job_config.get("enabled", self.enabled)
+ r.port = job_config.get("port", self.port)
+ return r
+
+
+class PypiRepoRunner:
+ def __init__(self, config) -> None:
+ logging.info("[PythonRunner] Initializing")
+ self.config = RepoConfig(config)
+ self.client = docker.from_env()
+ self.user = "alice"
+ self.passwd = "alice"
+ self.htpasswd = 'alice:{SHA}UisnajVr3zkBPfq+os1D4UHsyeg='
+
+ def __is_running(self, name):
+ try:
+ self.client.containers.get(name)
+ return True
+ except docker.errors.NotFound:
+ return False
+
+ def run(self, job_spec):
+ job_config = self.config.copy(job_spec)
+ running = self.__is_running(job_config.container_name)
+ print(f"[PyPiRepo] {job_config.container_name} running: {running}")
+
+ persistency_dir = join(getcwd(), ".alice", "pypirepo")
+ if not isdir(persistency_dir):
+ mkdir(persistency_dir)
+
+ package_dir = join(persistency_dir, "packages")
+ if not isdir(package_dir):
+ mkdir(package_dir)
+
+ htpasswd_file = join(persistency_dir, ".htpasswd")
+ with open(htpasswd_file, 'w') as f:
+ f.write(self.htpasswd)
+
+ docker_host_ip = None
+ for network in self.client.networks.list():
+ if network.name == "bridge":
+ docker_host_ip = network.attrs["IPAM"]["Config"][0]["Gateway"]
+ if docker_host_ip is None:
+ raise RunnerError("Unable to determine Docker host IP")
+
+ if job_config.enabled:
+ if not running:
+ c = self.client.containers.run(
+ name=job_config.container_name,
+ image="pypiserver/pypiserver:latest",
+ detach=True,
+ labels={"app": "alice"},
+ command=["--overwrite", "-P", ".htpasswd", "packages"],
+ ports={"8080/tcp": job_config.port},
+ volumes={
+ package_dir: {
+ "bind": "/data/packages",
+ "mode": "rw"
+ },
+ htpasswd_file: {
+ "bind": "/data/.htpasswd",
+ "mode": "ro"
+ }
+ },
+ restart_policy={
+ "Name": "unless-stopped"
+ }
+ )
+ c.reload()
+ print(f"[PyPiRepo] {job_config.container_name} : {c.status}")
+ cfgh = ConfigHolder.getInstance()
+ cfgh.soft_set("PYPI_USER", self.user)
+ cfgh.soft_set("PYPI_PASS", self.passwd)
+ cfgh.soft_set("PYPI_REPO", f"http://localhost:{job_config.port}")
+ cfgh.soft_set("DOCKER_PYPI_USER", self.user)
+ cfgh.soft_set("DOCKER_PYPI_PASS", self.passwd)
+ cfgh.soft_set("DOCKER_PYPI_REPO", f"http://{docker_host_ip}:{job_config.port}")
+ cfgh.commit()
+
+ venv = join(os.getcwd(), "venv")
+ if os.path.isdir(venv):
+ netloc = f"localhost:{job_config.port}"
+ url = f"http://{self.user}:{self.passwd}@{netloc}"
+ conf = pipconf.replace("URL", url).replace("BASE", netloc)
+
+ if os.name == "nt": # Windows
+ filename = join(venv, "pip.ini")
+ else: # Linux & Mac
+ filename = join(venv, "pip.conf")
+ with open(filename, 'w') as f:
+ f.write(conf)
+ print(f"[PyPiRepo] pip conf written to {filename}")
diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py
index 63d0d04e0d79995161520ea24b2b4827ff9f4cef..75b932994d94f03ae534538d6de2fe54de771dae 100644
--- a/alice-ci/src/alice/runners/pypirunner.py
+++ b/alice-ci/src/alice/runners/pypirunner.py
@@ -1,19 +1,62 @@
+from distutils.command.config import config
+from distutils.log import debug
import json
import logging
+from ntpath import join
import os
import re
import subprocess
import sys
-from urllib import request, error
from pkg_resources import parse_version
+from requests import get
+from requests.auth import HTTPBasicAuth
from os import environ, path
+from html.parser import HTMLParser
from alice.runners.pyutils import PackageManager, glob, grab_from
from alice.exceptions import ConfigException, RunnerError
+import hashlib
+from pathlib import Path
+
+
+def md5_update_from_file(filename, hash):
+ assert Path(filename).is_file()
+ with open(str(filename), "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hash.update(chunk)
+ return hash
+
+
+def md5_file(filename):
+ return md5_update_from_file(filename, hashlib.md5()).hexdigest()
+
+
+def md5_update_from_dir(directory, hash, exclude_dirs, exclude_extensions, exclude_dirs_wildcard):
+ assert Path(directory).is_dir()
+ for _path in os.listdir(directory):
+ path = os.path.join(directory, _path)
+ if os.path.isfile(path) :
+ hash.update(_path.encode())
+ logging.debug(f"[PyPiRunner][Hash] File: {path}")
+ hash = md5_update_from_file(path, hash)
+ elif os.path.isdir(path):
+ skip = False
+ for name in exclude_dirs:
+ if name in os.path.basename(_path):
+ skip = True
+ if not skip:
+ hash = md5_update_from_dir(path, hash, exclude_dirs, exclude_extensions, exclude_dirs_wildcard)
+ return hash
+
+
+def md5_dir(directory, exclude_dirs=[], exclude_extensions=[], exclude_dirs_wildcard=[]):
+ return md5_update_from_dir(directory, hashlib.sha1(), exclude_dirs, exclude_extensions, exclude_dirs_wildcard).hexdigest()
def get_uri(config, default):
url = config.get("repo", {}).get("uri", default)
if url is not None:
+ if not isinstance(url, str):
+ url = grab_from(url)
if not re.match('(?:http|ftp|https)://', url):
url = f"https://{url}"
return url
@@ -41,6 +84,19 @@ def get_pass(config, default):
return default
+class SimpleRepoParser(HTMLParser):
+ def __init__(self):
+ HTMLParser.__init__(self)
+ self.packages = []
+
+ def handle_data(self, data):
+ re_groups = re.findall("(\d*\.\d*\.\d*)", data)
+ if len(re_groups) == 1:
+ file_version = re_groups[0]
+ if file_version not in self.packages:
+ self.packages.append(file_version)
+
+
# Parses and stores the config from yaml
class PypiConfig:
def __init__(self, config={}) -> None:
@@ -67,41 +123,106 @@ class PypiConfig:
return p
+# TODO: Refactor to something sensible, more flexible
+class PackageMeta:
+ def __init__(self):
+ self.conf_dir = path.join(os.getcwd(), ".alice", "pypirunner")
+ self.metafile = path.join(self.conf_dir, "packagemeta.json")
+ if not path.isdir(self.conf_dir):
+ os.mkdir(self.conf_dir)
+ if path.isfile(self.metafile):
+ with open(self.metafile) as f:
+ self.metadata = json.load(f)
+ else:
+ self.metadata = {}
+ self.__save()
+
+ def __save(self):
+ with open(self.metafile, 'w') as f:
+ json.dump(self.metadata, f)
+
+ def get(self, package, key):
+ return self.metadata.get(package, {}).get(key, "")
+
+ def set(self, package, key, value):
+ if package not in self.metadata:
+ self.metadata[package] = {}
+ self.metadata[package][key] = value
+ self.__save()
+
+
# TODO: consider "--skip-existing" flag for twine
class PyPiRunner():
def __init__(self, config) -> None:
logging.info("[PyPiRunner] Initializing")
self.workdir = config["workdir"]
self.config = PypiConfig(config)
+ self.metadata = PackageMeta()
- def __versions(self, repo, pkg_name):
- if repo is not None:
- url = f'{repo}/{pkg_name}/json'
+ def __versions(self, config, pkg_name):
+ repo = config.repo_uri
+ if repo is None:
+ repo = "https://pypi.python.org/pypi"
+
+ if config.repo_pass is not None and config.repo_user is not None:
+ logging.info(f"[PyPiRunner][Versions] Set auth headers from config")
+ logging.debug(f"[PyPiRunner][Versions] Auth: {config.repo_user}:{config.repo_pass}")
+ auth = HTTPBasicAuth(config.repo_user, config.repo_pass)
else:
- url = f"https://pypi.python.org/pypi/{pkg_name}/json"
- try:
- releases = json.loads(request.urlopen(url).read())['releases']
- except error.URLError as e:
- raise RunnerError(f"{url}: {e}")
+ logging.info(f"[PyPiRunner][Versions] No auth headers in config, skip")
+ logging.debug(f"[PyPiRunner][Versions] Auth: {config.repo_user}:{config.repo_pass}")
+ auth = None
- return sorted(releases, key=parse_version, reverse=True)
+ try:
+ if repo.endswith("pypi"):
+ url = f'{repo}/{pkg_name}/json'
+ logging.info(f"[PyPiRunner][Versions] Trying JSON API at {url}")
+ response = get(url, auth=auth)
+ if response.status_code == 200:
+ releases = json.loads(response.text)["releases"]
+ return sorted(releases, key=parse_version, reverse=True)
+ else:
+ logging.info(f"[PyPiRunner][Versions] JSON failed: [{response.status_code}]")
+ logging.debug(response.text)
+ repo = f"{repo}/simple"
+ url = f"{repo}/{pkg_name}"
+ logging.info(f"[PyPiRunner][Versions] Trying Simple API at {url}")
+ response = get(url, auth=auth)
+ if response.status_code == 200:
+ parser = SimpleRepoParser()
+ parser.feed(response.text)
+ return sorted(parser.packages, key=parse_version, reverse=True)
+ if response.status_code == 404:
+ return []
+ else:
+ logging.info(f"[PyPiRunner][Versions] Simple failed: [{response.status_code}]")
+ logging.debug(response.text)
+ raise Exception("Failed to fetch available versions")
+
+ except Exception as e:
+ raise RunnerError(f"{url}: {e}")
def build(self, config, package):
+ print(f"[PyPiRunner] Building {package}")
pkg_path = path.join(config.workdir, package)
if not path.isdir(pkg_path):
raise ConfigException(f"Path does not exists: {pkg_path}")
+ PackageManager.getInstance().ensure("build")
command = [sys.executable, "-m", "build", package]
- with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
- p.wait()
- if p.returncode != 0:
- print("STDOUT:")
- sys.stdout.buffer.write(p.stdout.read())
- print("STDERR:")
- sys.stdout.buffer.write(p.stderr.read())
- raise RunnerError(f"[PyPiRunner] Failed to build {package}")
-
- def find_unuploaded(self, repo, file_list, pkg_name):
- versions = self.__versions(repo, pkg_name)
+ if logging.root.isEnabledFor(logging.DEBUG):
+ with subprocess.Popen(command, cwd=config.workdir) as p:
+ p.wait()
+ if p.returncode != 0:
+ raise RunnerError(f"[PyPiRunner] Failed to build {package}")
+ else:
+ with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE) as p:
+ p.wait()
+ if p.returncode != 0:
+ raise RunnerError(f"[PyPiRunner] Failed to build {package}")
+ print(f"[PyPiRunner] Package {package} built")
+
+ def find_unuploaded(self, config, file_list, pkg_name):
+ versions = self.__versions(config, pkg_name)
unuploaded = []
for file in file_list:
# flake8: noqa W605
@@ -113,52 +234,112 @@ class PyPiRunner():
unuploaded.append(file)
else:
print(f"[PyPiRunner] File already uploaded: {os.path.basename(file)}")
+ print(f"[PyPiRunner] Packages to publish: {', '.join(unuploaded) if len(unuploaded) > 1 else 'None'}")
return unuploaded
+ def upload_command(self, config, package, _command, to_upload):
+ unregistered = False
+ command = _command + to_upload
+ with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+ p.wait()
+ if p.returncode != 0:
+ for line in map(lambda x: x.decode('utf8').strip(), p.stderr):
+ if "405 Method Not Allowed" in line:
+ unregistered = True
+ if not unregistered:
+ print("STDOUT:")
+ sys.stdout.buffer.write(p.stdout.read())
+ print("STDERR:")
+ sys.stdout.buffer.write(p.stderr.read())
+ raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})")
+ if unregistered:
+ print("[PyPiRunner] Registering package")
+ register_command = [sys.executable, "-m", "twine", "register", "--verbose", "--non-interactive"]
+ if config.repo_uri is not None:
+ register_command.append("--repository-url")
+ register_command.append(config.repo_uri)
+ if config.repo_user is not None and config.repo_pass is not None:
+ register_command.append("-u")
+ register_command.append(config.repo_user)
+ register_command.append("-p")
+ register_command.append(config.repo_pass)
+ register_command.append(to_upload[0])
+ with subprocess.Popen(register_command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+ p.wait()
+ if p.returncode != 0:
+ print("STDOUT:")
+ sys.stdout.buffer.write(p.stdout.read())
+ print("STDERR:")
+ sys.stdout.buffer.write(p.stderr.read())
+ raise RunnerError(f"[PyPiRunner] Failed to register {package} ({p.returncode})")
+ self.upload_command(config, package, _command, to_upload)
- def upload(self, config, package):
- command = [sys.executable, "-m", "twine", "upload", "--verbose"]
+ def upload(self, config, package, current_version):
+ print(f"[PyPiRunner] Uploading {package}")
+ PackageManager.getInstance().ensure("twine")
+ command = [sys.executable, "-m", "twine", "upload", "--verbose", "--non-interactive"]
if config.repo_uri is not None:
command.append("--repository-url")
command.append(config.repo_uri)
- if config.repo_user is not None:
+ if config.repo_user is not None and config.repo_pass is not None:
command.append("-u")
command.append(config.repo_user)
- if config.repo_pass is not None:
command.append("-p")
command.append(config.repo_pass)
+ else:
+ raise RunnerError("[PyPiRunner] Can't upload without credentials!")
dist_path = os.path.abspath(os.path.join(config.workdir, package, "dist"))
- files = glob(os.path.join(dist_path, "*"), config.workdir)
- for file in files:
- print(f"[PyPiRunner] Found: {file}")
+ _files = glob(os.path.join(dist_path, "*"), config.workdir)
+ files = []
+ for file in _files:
+ if current_version in os.path.basename(file):
+ files.append(file)
+ print(f"[PyPiRunner] Found: {file}")
+ else:
+ logging.info(f"[PyPiRunner] Dropped: {file} doesn't match current version: {current_version}")
- to_upload = self.find_unuploaded(config.repo_uri, files, package)
+ to_upload = self.find_unuploaded(config, files, package)
if len(to_upload) == 0:
return
- command += to_upload
- with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
- p.wait()
- if p.returncode != 0:
- print("STDOUT:")
- sys.stdout.buffer.write(p.stdout.read())
- print("STDERR:")
- sys.stdout.buffer.write(p.stderr.read())
- raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})")
+ #command += to_upload
+ self.upload_command(config, package, command, to_upload)
+ print(f"[PyPiRunner] Uploaded {package}")
+
+ def package_version(self, config, package):
+ cfg_path = path.join(config.workdir, package, "setup.cfg")
+ with open(cfg_path) as f:
+ for line in f:
+ if line.startswith("version"):
+ re_groups = re.findall("(\d*\.\d*\.\d*)", line)
+ if len(re_groups) < 1:
+ raise RunnerError(f"Unable to determine version of package: |{line}|")
+ return re_groups[0]
def run(self, job_spec):
job_config = self.config.copy(job_spec)
-
- PackageManager.getInstance().ensure("build")
+
for package in job_config.packages:
- print(f"[PyPiRunner] Building {package}")
- self.build(job_config, package)
- print(f"[PyPiRunner] Package {package} built")
-
- if job_config.upload:
- PackageManager.getInstance().ensure("twine")
- for package in job_config.packages:
- print(f"[PyPiRunner] Uploading {package}")
- self.upload(job_config, package)
- else:
- print(f"[PyPiRunner] Upload disabled, skiping")
+ pkg_dir = path.join(job_config.workdir, package)
+ pkg_hash = md5_dir(pkg_dir, exclude_dirs=["pycache", "pytest_cache", "build", "dist", "egg-info"])
+ logging.debug(f"[PyPiRunner] {package} hash: {pkg_hash}")
+ pkg_version = self.package_version(job_config, package)
+ logging.debug(f"[PyPiRunner] {package} local version: {pkg_version}")
+ repo_versions = self.__versions(job_config, package)
+ logging.debug(f"[PyPiRunner] {package} remote version: {repo_versions}")
+
+ if pkg_version not in repo_versions:
+ print(f"[PyPiRunner] {package} not found in repo")
+ self.build(job_config, package)
+ self.metadata.set(package, pkg_version, pkg_hash)
+ else:
+ if pkg_hash != self.metadata.get(package, pkg_version):
+ self.build(job_config, package)
+ self.metadata.set(package, pkg_version, pkg_hash)
+ else:
+ print(f"[PyPiRunner] {package} Unchanged since last build")
+
+ if job_config.upload:
+ self.upload(job_config, package, pkg_version)
+ else:
+ print(f"[PyPiRunner] Upload disabled, skipping")
diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py
index c2854bd7694d7fa5cf6287befee580686e378043..93bd4ca9ec569690f5476e755b60debde2bec632 100644
--- a/alice-ci/src/alice/runners/pythonrunner.py
+++ b/alice-ci/src/alice/runners/pythonrunner.py
@@ -5,7 +5,7 @@ import sys
import shlex
from ..exceptions import NonZeroRetcode, RunnerError, ConfigException
-from .pyutils import PackageManager, glob_command
+from .pyutils import PackageManager, glob_command, grab_from
# TODO: Handle config like PyPiConfig
@@ -18,6 +18,7 @@ class PythonRunner:
PackageManager.getInstance().ensure("virtualenv")
self.__init_venv()
+ # TODO: Detect if the prev venv is the same OS type
def __init_venv(self):
if os.name == "nt": # Windows
self.vpython = os.path.join(self.virtual_dir, "Scripts", "python.exe")
@@ -27,12 +28,14 @@ class PythonRunner:
if not os.path.exists(self.vpython):
logging.debug(f"[PythonRunner] Venv not found at {self.vpython}")
logging.info("[PythonRunner] Initializing venv")
+ output = []
with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+ stdout=subprocess.PIPE) as p:
p.wait()
+ for line in p.stdout:
+ output.append(line.decode('utf8').strip())
if p.returncode != 0:
- sys.stdout.buffer.write(p.stderr.read())
- sys.stdout.buffer.write(p.stdout.read())
+ print("\n".join(output))
raise RunnerError("[PythonRunner] Could not create virtualenv")
else:
logging.info(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}")
@@ -42,11 +45,20 @@ class PythonRunner:
if len(dependencies) > 0:
logging.info(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}")
command = [self.vpython, "-m", "pip", "install"] + dependencies
- with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
- p.wait()
- if p.returncode != 0:
- sys.stdout.buffer.write(p.stderr.read())
- raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})"))
+ if logging.root.isEnabledFor(logging.DEBUG):
+ with subprocess.Popen(command) as p:
+ p.wait()
+ if p.returncode != 0:
+ raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})"))
+ else:
+ output = []
+ with subprocess.Popen(command, stdout=subprocess.PIPE) as p:
+ for line in p.stdout:
+ output.append(line.decode('utf8').strip())
+ p.wait()
+ if p.returncode != 0:
+ print("\n".join(output))
+ raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})"))
logging.info("[PythonRunner] Installation done")
# Executes the given job in the one and only venv
@@ -56,10 +68,18 @@ class PythonRunner:
pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"]))
else:
pwd = self.workdir
- run_env = self.config["env"].copy()
+ run_env = {}
+ for k, v in self.config["env"].items():
+ if isinstance(v, str):
+ run_env[k] = v
+ else:
+ run_env[k] = grab_from(v)
if "env" in job_spec:
for env_var in job_spec["env"]:
- run_env[env_var["name"]] = env_var["value"]
+ if isinstance(env_var["value"], str):
+ run_env[env_var["name"]] = env_var["value"]
+ else:
+ run_env[env_var["name"]] = grab_from(env_var["value"])
if "commands" in job_spec:
commands = job_spec["commands"]
for command in commands:
diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py
index d8bb685a1d131326c08f61527a51a4cbfc6f4105..3c8ba6c88574638c1e7812bcc971e0ff897130e9 100644
--- a/alice-ci/src/alice/runners/pyutils.py
+++ b/alice-ci/src/alice/runners/pyutils.py
@@ -125,9 +125,12 @@ def grab_from(target):
except KeyError:
raise ConfigException(f"Env var unset: {target['from_env']}")
elif "from_cfg" in target:
- ConfigHolder.getInstance().get(target["from_cfg"])
+ value = ConfigHolder.getInstance().get(target["from_cfg"])
+ if len(value) == 0:
+ value = None
+ return value
else:
- raise ConfigException(f"Unsupported grabber: {target.keys()}")
+ raise ConfigException(f"Unsupported grabber: {target}")
def gen_dict(list_of_dicts):
diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml
index 0e5714fb79ee59e68cec150695478e1bfe4b12ae..c06827c6424fe5a0a5d457ccc0d0626e24e0b1ff 100644
--- a/ci-examples/full.yaml
+++ b/ci-examples/full.yaml
@@ -84,6 +84,11 @@ jobs:
credentials:
username: B
password: B
+ - name: pypi_init
+ type: pypirepo
+ enabled: true
+ port: 8888
+ container_name: pypiserver
pipelines:
default: