From 125ebfac956440f3a26e160a02d0c54c402a7bca Mon Sep 17 00:00:00 2001 From: KunoiSayami Date: Tue, 25 Oct 2022 22:12:18 +0800 Subject: feat(script): Add upload rated scripts Signed-off-by: KunoiSayami --- utils/ng/pkgbuild.py | 21 ++-- utils/ng/systemd/update-repo-notify.service | 7 ++ utils/ng/systemd/update-repo-notify.socket | 16 +++ utils/ng/update-db.py | 166 ++++++++++++++++++++++++++++ utils/ng/upload.php | 38 +++++++ 5 files changed, 237 insertions(+), 11 deletions(-) create mode 100644 utils/ng/systemd/update-repo-notify.service create mode 100644 utils/ng/systemd/update-repo-notify.socket create mode 100755 utils/ng/update-db.py create mode 100644 utils/ng/upload.php (limited to 'utils/ng') diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py index 8be755a..0bef847 100755 --- a/utils/ng/pkgbuild.py +++ b/utils/ng/pkgbuild.py @@ -25,7 +25,7 @@ CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "") BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "") TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve() GNUPG_HOME = os.getenv("HOME") -FAIL_REPOS = [] +FAIL_REPOS = {} async def get_arch() -> str: @@ -278,7 +278,7 @@ async def run_build( return p.returncode -async def install_dependency_via_yay(dep: str) -> None: +async def install_dependency_via_yay(dependency: str) -> None: await ( await asyncio.create_subprocess_exec( "yay", @@ -287,7 +287,7 @@ async def install_dependency_via_yay(dep: str) -> None: "--needed", "--noprogressbar", "-S", - dep, + dependency, stdout=None, ) ).wait() @@ -346,7 +346,7 @@ async def do_build(target: PackageVersionWithPath) -> int: if build_ret == 13: logger.warning("Already build %s, skipped.", base_dir_name) return 0 - FAIL_REPOS.append(base_dir_name) + FAIL_REPOS.update({base_dir_name: build_ret}) logger.warning("Build %s fail: %d", base_dir_name, build_ret) return build_ret @@ -354,10 +354,10 @@ async def do_build(target: PackageVersionWithPath) -> int: async def clean_installed_packages() -> int: p = await asyncio.create_subprocess_exec( - "/bin/bash", "-c", - "PKGS=$(pacman -Qdtq | tr '\n' ' '); " - "if [ -n \"$PKGS\" ]; then echo $PKGS | xargs sudo pacman --noconfirm -Rcns ; fi", - stdout=None + "/bin/bash", + "-c", + "pacman -Qdtq | ifne sudo pacman --noconfirm -Rcns", + stdout=None, ) await p.wait() return p.returncode @@ -431,9 +431,8 @@ async def do_work( await fout.write(f"{int(time.time())}") if len(FAIL_REPOS): - logger.error("Build failed repositories:") - for repo in FAIL_REPOS: - logger.error("%s", repo) + logger.error("Build failed repositories (%d):", len(FAIL_REPOS)) + logger.error("%s", ', '.join(f'{repo}({ret_code})' for repo, ret_code in FAIL_REPOS.items())) os.chdir(str(home_directory)) if ret := await upload_packages(): diff --git a/utils/ng/systemd/update-repo-notify.service b/utils/ng/systemd/update-repo-notify.service new file mode 100644 index 0000000..e609a8a --- /dev/null +++ b/utils/ng/systemd/update-repo-notify.service @@ -0,0 +1,7 @@ +[Unit] +Description=Update repo notify service + +[Service] +User=update-database +DynamicUser=yes +ExecStart=/var/lib/update/update-db.py /run/update-repo-notify.sock \ No newline at end of file diff --git a/utils/ng/systemd/update-repo-notify.socket b/utils/ng/systemd/update-repo-notify.socket new file mode 100644 index 0000000..f14743c --- /dev/null +++ b/utils/ng/systemd/update-repo-notify.socket @@ -0,0 +1,16 @@ +[Unit] +Description=Update repo notify socket + +[Socket] +# If /var/run is not implemented as a symlink to /run, you may need to +# specify ListenStream=/var/run/docker.sock instead. +ListenStream=/run/update-repo-notify.sock +SocketMode=0660 +SocketUser=update-database +SocketGroup=http +#Accept=yes +MaxConnections=1 +FlushPending=yes + +[Install] +WantedBy=sockets.target \ No newline at end of file diff --git a/utils/ng/update-db.py b/utils/ng/update-db.py new file mode 100755 index 0000000..02064a4 --- /dev/null +++ b/utils/ng/update-db.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +import argparse +import logging +import os +import pathlib +import signal +import subprocess +import threading +import time +from socketserver import TCPServer, StreamRequestHandler +import socket +from queue import Queue + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +"""class DataHandler(asyncio.Protocol): + def data_received(self, data: bytes) -> None: + logger.debug(data) + + def eof_received(self) -> bool | None: + logger.debug("eof") + + +class Server: + def __init__(self, unix_sockets: str): + self.unix_sockets = unix_sockets + self.server = None + + async def connect(self) -> None: + #self.server = await asyncio.start_unix_server(self.handle_connection, self.unix_sockets, start_serving=False) + self.server = await asyncio.get_running_loop().create_unix_server(DataHandler, None, + sock=socket.fromfd(3, socket.AF_UNIX, socket.SOCK_STREAM)) + + async def serving(self) -> None: + await self.server.serve_forever() + + async def handle_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + logger.debug("Connected") + while True: + task = asyncio.create_task(reader.readline()) + finished, pending = await asyncio.wait([task], timeout=10) + if len(finished): + logger.debug(finished.pop().result()) + break + + writer.close() + await writer.wait_closed() + await self.close() + + async def close(self) -> None: + if self.server: + self.server.close() + await self.server.wait_closed() + + +async def main(unix_sockets: str) -> None: + logger.debug("Create connection") + #os.unlink(unix_sockets) + server = Server(unix_sockets) + await server.connect() + #await server.server()""" + +queue = Queue() + +ARCH = os.getenv("ARCH") + + +def get_arch() -> str: + p = subprocess.Popen(["uname", "-m"], stdout=subprocess.PIPE) + stdout, _ = p.communicate() + return stdout.decode().lower().strip() + + +if ARCH is None: + ARCH = get_arch() + logger.warning("Warning: You should specify arch environment variable, use current platform: %s", ARCH) + + +class Handler(StreamRequestHandler): + def handle(self): + while line := self.rfile.readline(): + if b"EOF" in line: + os.kill(os.getpid(), signal.SIGUSR1) + return + logger.info("Recv: %s", line) + # self.wfile.write(line.upper()) + + +# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4 +class Server(TCPServer): + # The constant would be better initialized by a systemd module + SYSTEMD_FIRST_SOCKET_FD = 3 + + def __init__(self, server_address, handler_cls): + # Invoke base but omit bind/listen steps (performed by systemd activation!) + TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False) + # Override socket + self.socket = socket.fromfd( + self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type + ) + + +class RepoProcessThread(threading.Thread): + def __init__(self, dest: pathlib.Path, key: str, pkg_src: pathlib.Path): + super().__init__() + self.exit_event = threading.Event() + self.dest = dest + self.key = key + self.pkg_src = pkg_src + + def run(self) -> None: + while not self.exit_event.is_set(): + while not queue.empty(): + logger.info("Adding %s", repo := queue.get_nowait()) + self.pkg_src.joinpath(repo) + subprocess.Popen( + ["repo-add", str(self.dest), repo, "-s", "-v", "-R", "-k", self.key], + stdout=subprocess.STDOUT + ).wait() + time.sleep(1) + + def set_exit_event(self) -> None: + self.exit_event.set() + + +def main(cwd: str): + def server_shutdown_wrapper(*_args) -> None: + logger.debug("Shutdown the server") + threading.Thread(target=server.shutdown, daemon=True,).start() + + server = Server(("localhost", 0), Handler) + signal.signal(signal.SIGUSR1, server_shutdown_wrapper) + + remote_path = pathlib.Path(cwd).resolve() + cwd = pathlib.Path(os.getcwd()).resolve() + + RepoProcessThread( + cwd.joinpath("packages", ARCH, "kunoisayami.db.tar.gz").resolve(), + "4A0F0C8BC709ACA4341767FB243975C8DB9656B9", + remote_path + ).start() + server.serve_forever() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("update-db.py") + parser.add_argument("cwd") + + args_ = parser.parse_args() + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + main(args_.cwd) diff --git a/utils/ng/upload.php b/utils/ng/upload.php new file mode 100644 index 0000000..fc60a61 --- /dev/null +++ b/utils/ng/upload.php @@ -0,0 +1,38 @@ + -- cgit v1.3.1