From b0d361f0074897a8096b898e6acd3f063679bc7e Mon Sep 17 00:00:00 2001
From: gyulaid <gyulaid@gyulai.cloud>
Date: Mon, 25 Apr 2022 17:37:30 +0200
Subject: [PATCH] repo image build + bump version
---
alice-ci/setup.cfg | 2 +-
alice-ci/src/alice/cli.py | 1 +
alice-ci/src/alice/runners/pypirepo.py | 102 ++++++++++++++++++-------
3 files changed, 77 insertions(+), 28 deletions(-)
diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg
index 515afc6..ef9edfa 100644
--- a/alice-ci/setup.cfg
+++ b/alice-ci/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = alice-ci
-version = 0.0.13
+version = 0.0.14
author = Daniel Gyulai
description = Alice CI framework
long_description = file: README.md
diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py
index 8386b15..fd587dd 100644
--- a/alice-ci/src/alice/cli.py
+++ b/alice-ci/src/alice/cli.py
@@ -34,6 +34,7 @@ def parse_jobs(args):
exit(1)
except RunnerError as e:
print(f"RunnerError-> {e}")
+ exit(1)
def main():
diff --git a/alice-ci/src/alice/runners/pypirepo.py b/alice-ci/src/alice/runners/pypirepo.py
index 79db938..df047e1 100644
--- a/alice-ci/src/alice/runners/pypirepo.py
+++ b/alice-ci/src/alice/runners/pypirepo.py
@@ -1,12 +1,17 @@
import logging
+import subprocess
import docker
from os.path import join, isdir
from os import getcwd, mkdir
import os
+import requests
+import platform
+import time
from ..exceptions import RunnerError
from ..config import ConfigHolder
+
pipconf = """[global]
index-url = URL
trusted-host = BASE
@@ -30,37 +35,45 @@ class RepoConfig:
class PypiRepoRunner:
def __init__(self, config) -> None:
- logging.info("[PythonRunner] Initializing")
+ logging.info("[PyPiRepo] Initializing")
self.config = RepoConfig(config)
self.client = docker.from_env()
self.user = "alice"
self.passwd = "alice"
self.htpasswd = 'alice:{SHA}UisnajVr3zkBPfq+os1D4UHsyeg='
- def __is_running(self, name):
- try:
- self.client.containers.get(name)
- return True
- except docker.errors.NotFound:
- return False
+ def get_image(self):
+ # TODO: remove when resolved:
+ # Official Docker image support for ARM?
+ # https://github.com/pypiserver/pypiserver/issues/364
+ pypiserver = "https://github.com/pypiserver/pypiserver.git"
+ if platform.machine() == "aarch64":
+ tag = "alice.localhost/pypiserver:arm"
+ try:
+ self.client.images.get(tag)
+ return tag
+ except docker.errors.ImageNotFound:
+ print("[PyPiRepo] Building PyPiServer ARM image, this could take a while")
+ workdir = join(getcwd(), ".alice", "pypirepo", "source")
+ if not os.path.isdir(workdir):
+ os.mkdir(workdir)
+ git_command = ["git", "clone", pypiserver, "--branch=v1.3.2"]
+ output = []
+ with subprocess.Popen(git_command, cwd=workdir, stdout=subprocess.PIPE) as p:
+ for line in p.stdout:
+ output.append(line.decode('utf8').strip())
+ p.wait()
+ if p.returncode != 0:
+ print("\n".join(output))
+ raise(RunnerError("[PyPiRepo] Could not fetch pypiserver source"))
+ source_path = os.path.join(workdir, "pypiserver")
+ self.client.images.build(path=source_path, tag=tag)
+ return tag
+ else:
+ return "pypiserver/pypiserver:latest"
def run(self, job_spec):
job_config = self.config.copy(job_spec)
- running = self.__is_running(job_config.container_name)
- print(f"[PyPiRepo] {job_config.container_name} running: {running}")
-
- persistency_dir = join(getcwd(), ".alice", "pypirepo")
- if not isdir(persistency_dir):
- mkdir(persistency_dir)
-
- package_dir = join(persistency_dir, "packages")
- if not isdir(package_dir):
- mkdir(package_dir)
-
- htpasswd_file = join(persistency_dir, ".htpasswd")
- with open(htpasswd_file, 'w') as f:
- f.write(self.htpasswd)
-
docker_host_ip = None
for network in self.client.networks.list():
if network.name == "bridge":
@@ -72,10 +85,25 @@ class PypiRepoRunner:
raise RunnerError("Unable to determine Docker host IP")
if job_config.enabled:
- if not running:
+ try:
+ c = self.client.containers.get(job_config.container_name)
+ print(f"[PyPiRepo] {job_config.container_name} already running")
+ except docker.errors.NotFound:
+ persistency_dir = join(getcwd(), ".alice", "pypirepo")
+ if not isdir(persistency_dir):
+ mkdir(persistency_dir)
+
+ package_dir = join(persistency_dir, "packages")
+ if not isdir(package_dir):
+ mkdir(package_dir)
+
+ htpasswd_file = join(persistency_dir, ".htpasswd")
+ with open(htpasswd_file, 'w') as f:
+ f.write(self.htpasswd)
+
c = self.client.containers.run(
name=job_config.container_name,
- image="pypiserver/pypiserver:latest",
+ image=self.get_image(),
detach=True,
labels={"app": "alice"},
command=["--overwrite", "-P", ".htpasswd", "packages"],
@@ -94,12 +122,32 @@ class PypiRepoRunner:
"Name": "unless-stopped"
}
)
- c.reload()
- print(f"[PyPiRepo] {job_config.container_name} : {c.status}")
+ print(f"[PyPiRepo] Started {job_config.container_name}")
+
+ c.reload()
+ logging.info(f"[PyPiRepo] {job_config.container_name} : {c.status}")
+ if c.status != "running":
+ raise RunnerError(f"[PyPiRepo] Repo container unstable: {c.status}")
+
+ uri = f"http://localhost:{job_config.port}"
+ unreachable = True
+ attempts = 0
+ while unreachable and attempts < 5:
+ attempts += 1
+ try:
+ requests.get(uri)
+ unreachable = False
+ except Exception as e:
+ logging.info(f"[PyPiRepo] {attempts} - Repo at {uri} is unavailable: {e}")
+ time.sleep(2)
+ if unreachable:
+ raise RunnerError(f"[PyPiRepo] Repo unreachable")
+
+
cfgh = ConfigHolder.getInstance()
cfgh.soft_set("PYPI_USER", self.user)
cfgh.soft_set("PYPI_PASS", self.passwd)
- cfgh.soft_set("PYPI_REPO", f"http://localhost:{job_config.port}")
+ cfgh.soft_set("PYPI_REPO", uri)
cfgh.soft_set("DOCKER_PYPI_USER", self.user)
cfgh.soft_set("DOCKER_PYPI_PASS", self.passwd)
cfgh.soft_set("DOCKER_PYPI_REPO", f"http://{docker_host_ip}:{job_config.port}")
--
GitLab