diff options
| -rwxr-xr-x | utils/ng/pkgbuild.py | 5 | ||||
| -rw-r--r-- | utils/ng/systemd/update-repo-notify.service | 4 | ||||
| -rw-r--r-- | utils/ng/systemd/[email protected] | 7 | ||||
| -rwxr-xr-x | utils/ng/trigger.py | 78 | ||||
| -rwxr-xr-x | utils/ng/update-db.py | 157 | ||||
| -rw-r--r-- | utils/ng/upload.php | 5 |
6 files changed, 119 insertions, 137 deletions
diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py index 0bef847..e3f46f1 100755 --- a/utils/ng/pkgbuild.py +++ b/utils/ng/pkgbuild.py @@ -432,7 +432,10 @@ async def do_work( if len(FAIL_REPOS): logger.error("Build failed repositories (%d):", len(FAIL_REPOS)) - logger.error("%s", ', '.join(f'{repo}({ret_code})' for repo, ret_code in FAIL_REPOS.items())) + logger.error( + "%s", + ", ".join(f"{repo}({ret_code})" for repo, ret_code in FAIL_REPOS.items()), + ) os.chdir(str(home_directory)) if ret := await upload_packages(): diff --git a/utils/ng/systemd/update-repo-notify.service b/utils/ng/systemd/update-repo-notify.service index e609a8a..d7e3c9a 100644 --- a/utils/ng/systemd/update-repo-notify.service +++ b/utils/ng/systemd/update-repo-notify.service @@ -2,6 +2,4 @@ Description=Update repo notify service [Service] -User=update-database -DynamicUser=yes -ExecStart=/var/lib/update/update-db.py /run/update-repo-notify.sock
\ No newline at end of file +ExecStart=/var/lib/update/trigger.py
\ No newline at end of file diff --git a/utils/ng/systemd/[email protected] b/utils/ng/systemd/[email protected] new file mode 100644 index 0000000..6fa98da --- /dev/null +++ b/utils/ng/systemd/[email protected] @@ -0,0 +1,7 @@ +[Unit] +Description=Update repo %i service + +[Service] +User=update-database +ExecStart=/var/lib/update/update-db.py /var/lib/update %i AAAABBBBCCCCDDDDDDDDCCCCBBBBAAAA +#ExecStart=rsync
\ No newline at end of file diff --git a/utils/ng/trigger.py b/utils/ng/trigger.py new file mode 100755 index 0000000..ca0cfd1 --- /dev/null +++ b/utils/ng/trigger.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +import argparse +import logging +import os +import signal +import socket +import subprocess +import threading +from socketserver import StreamRequestHandler, TCPServer + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +class Handler(StreamRequestHandler): + def handle(self): + while line := self.rfile.readline(): + if b"ARCH=" in line: + ( + subprocess.Popen( + [ + "systemctl", + "--no-block", + "start", + f"update-repo@{line.split(b'=')[1].decode().strip()}", + ] + ) + ).wait() + os.kill(os.getpid(), signal.SIGUSR1) + return + logger.info("Recv: %s", line) + + +# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4 +class Server(TCPServer): + # The constant would be better initialized by a systemd module + SYSTEMD_FIRST_SOCKET_FD = 3 + + def __init__(self, server_address, handler_cls): + # Invoke base but omit bind/listen steps (performed by systemd activation!) + TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False) + # Override socket + self.socket = socket.fromfd( + self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type + ) + + +def main(): + def server_shutdown_wrapper(*_args) -> None: + logger.debug("Shutdown the server") + threading.Thread( + target=server.shutdown, + daemon=True, + ).start() + + server = Server(("localhost", 0), Handler) + signal.signal(signal.SIGUSR1, server_shutdown_wrapper) + + server.serve_forever() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + + main() diff --git a/utils/ng/update-db.py b/utils/ng/update-db.py index 02064a4..acdc8c8 100755 --- a/utils/ng/update-db.py +++ b/utils/ng/update-db.py @@ -1,153 +1,52 @@ #!/usr/bin/env python import argparse +import asyncio import logging import os import pathlib -import signal +import shutil import subprocess -import threading -import time -from socketserver import TCPServer, StreamRequestHandler -import socket -from queue import Queue logger = logging.getLogger() logger.setLevel(logging.DEBUG) -"""class DataHandler(asyncio.Protocol): - def data_received(self, data: bytes) -> None: - logger.debug(data) - - def eof_received(self) -> bool | None: - logger.debug("eof") - - -class Server: - def __init__(self, unix_sockets: str): - self.unix_sockets = unix_sockets - self.server = None - - async def connect(self) -> None: - #self.server = await asyncio.start_unix_server(self.handle_connection, self.unix_sockets, start_serving=False) - self.server = await asyncio.get_running_loop().create_unix_server(DataHandler, None, - sock=socket.fromfd(3, socket.AF_UNIX, socket.SOCK_STREAM)) - - async def serving(self) -> None: - await self.server.serve_forever() - - async def handle_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - logger.debug("Connected") - while True: - task = asyncio.create_task(reader.readline()) - finished, pending = await asyncio.wait([task], timeout=10) - if len(finished): - logger.debug(finished.pop().result()) - break - - writer.close() - await writer.wait_closed() - await self.close() - - async def close(self) -> None: - if self.server: - self.server.close() - await self.server.wait_closed() - - -async def main(unix_sockets: str) -> None: - logger.debug("Create connection") - #os.unlink(unix_sockets) - server = Server(unix_sockets) - await server.connect() - #await server.server()""" - -queue = Queue() - -ARCH = os.getenv("ARCH") - - -def get_arch() -> str: - p = subprocess.Popen(["uname", "-m"], stdout=subprocess.PIPE) - stdout, _ = p.communicate() - return stdout.decode().lower().strip() - - -if ARCH is None: - ARCH = get_arch() - logger.warning("Warning: You should specify arch environment variable, use current platform: %s", ARCH) - - -class Handler(StreamRequestHandler): - def handle(self): - while line := self.rfile.readline(): - if b"EOF" in line: - os.kill(os.getpid(), signal.SIGUSR1) - return - logger.info("Recv: %s", line) - # self.wfile.write(line.upper()) - - -# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4 -class Server(TCPServer): - # The constant would be better initialized by a systemd module - SYSTEMD_FIRST_SOCKET_FD = 3 - - def __init__(self, server_address, handler_cls): - # Invoke base but omit bind/listen steps (performed by systemd activation!) - TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False) - # Override socket - self.socket = socket.fromfd( - self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type +async def add_package(dest: pathlib.Path, pkg_name: str, key: str) -> None: + await ( + await asyncio.create_subprocess_exec( + "repo-add", + str(dest), + pkg_name, + "-s", + "-v", + "-R", + "-k", + key, + stdout=subprocess.STDOUT, ) + ).wait() -class RepoProcessThread(threading.Thread): - def __init__(self, dest: pathlib.Path, key: str, pkg_src: pathlib.Path): - super().__init__() - self.exit_event = threading.Event() - self.dest = dest - self.key = key - self.pkg_src = pkg_src - - def run(self) -> None: - while not self.exit_event.is_set(): - while not queue.empty(): - logger.info("Adding %s", repo := queue.get_nowait()) - self.pkg_src.joinpath(repo) - subprocess.Popen( - ["repo-add", str(self.dest), repo, "-s", "-v", "-R", "-k", self.key], - stdout=subprocess.STDOUT - ).wait() - time.sleep(1) - - def set_exit_event(self) -> None: - self.exit_event.set() - - -def main(cwd: str): - def server_shutdown_wrapper(*_args) -> None: - logger.debug("Shutdown the server") - threading.Thread(target=server.shutdown, daemon=True,).start() - - server = Server(("localhost", 0), Handler) - signal.signal(signal.SIGUSR1, server_shutdown_wrapper) - - remote_path = pathlib.Path(cwd).resolve() +async def main(remote_dest: str, repo_base_name: str, sign_key: str) -> None: cwd = pathlib.Path(os.getcwd()).resolve() + remote_dest = pathlib.Path(remote_dest).resolve() + database_dest = cwd.joinpath(repo_base_name).resolve() + + for file in os.listdir(str(remote_dest)): + logger.debug("Copy file: %s", file) + shutil.copy(str(remote_dest.joinpath(file).joinpath()), str(cwd)) - RepoProcessThread( - cwd.joinpath("packages", ARCH, "kunoisayami.db.tar.gz").resolve(), - "4A0F0C8BC709ACA4341767FB243975C8DB9656B9", - remote_path - ).start() - server.serve_forever() + for file in os.listdir(str(remote_dest)): + if ".pkg.tar.zst" in file: + await add_package(database_dest, file.split(".pkg.tar.zst")[0], sign_key) if __name__ == "__main__": parser = argparse.ArgumentParser("update-db.py") parser.add_argument("cwd") + parser.add_argument("basename") + parser.add_argument("sign_key") args_ = parser.parse_args() @@ -163,4 +62,4 @@ if __name__ == "__main__": level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", ) - main(args_.cwd) + main(args_.cwd, args_.basename, args_.sign_key) diff --git a/utils/ng/upload.php b/utils/ng/upload.php index fc60a61..da848eb 100644 --- a/utils/ng/upload.php +++ b/utils/ng/upload.php @@ -14,16 +14,13 @@ if (move_uploaded_file($_FILES['file']['tmp_name'], $path)) { echo "The file " . basename( $_FILES['file']['name']) . " has been uploaded"; - $sock = stream_socket_client('unix:///run/update-repo-notify.sock', $errno, $errst); - fwrite($sock, $['file']['name']."\r\n"); - fclose($sock); } else { echo "There was an error uploading the file, please try again!"; } } else if (isset($_POST['action'])) { if ($_POST['action'] === 'UPLOADED') { $sock = stream_socket_client('unix:///run/update-repo-notify.sock', $errno, $errst); - fwrite($sock, "EOF\r\n"); + fwrite($sock, "ARCH=". $_POST['arch'] ."\r\n"); fclose($sock); echo "200 OK STARTED"; } else if ($_POST['action'] === 'REQUIRE_CLEAN') { |