From 685c74c4fa212b132c55be1c91e5b366a3593b83 Mon Sep 17 00:00:00 2001 From: KunoiSayami Date: Wed, 26 Oct 2022 00:05:46 +0800 Subject: refactor: Move script out from ng folder Signed-off-by: KunoiSayami --- utils/ci-upload.sh | 41 --- utils/ng/pkgbuild.py | 485 ---------------------------- utils/ng/pkgbuild_bootstrap | 53 --- utils/ng/systemd/update-repo-notify.service | 5 - utils/ng/systemd/update-repo-notify.socket | 16 - utils/ng/systemd/update-repo@.service | 7 - utils/ng/trigger.py | 78 ----- utils/ng/update-db.py | 65 ---- utils/ng/upload.php | 35 -- utils/ng/upload.py | 85 ----- utils/pkgbuild.py | 485 ++++++++++++++++++++++++++++ utils/pkgbuild.sh | 184 ----------- utils/pkgbuild_bootstrap | 53 +++ utils/repodb.sh | 34 -- utils/systemd/update-repo-notify.service | 5 + utils/systemd/update-repo-notify.socket | 16 + utils/systemd/update-repo@.service | 7 + utils/trigger.py | 78 +++++ utils/update-db.py | 65 ++++ utils/upload.php | 35 ++ utils/upload.py | 85 +++++ 21 files changed, 829 insertions(+), 1088 deletions(-) delete mode 100755 utils/ci-upload.sh delete mode 100755 utils/ng/pkgbuild.py delete mode 100755 utils/ng/pkgbuild_bootstrap delete mode 100644 utils/ng/systemd/update-repo-notify.service delete mode 100644 utils/ng/systemd/update-repo-notify.socket delete mode 100644 utils/ng/systemd/update-repo@.service delete mode 100755 utils/ng/trigger.py delete mode 100755 utils/ng/update-db.py delete mode 100644 utils/ng/upload.php delete mode 100755 utils/ng/upload.py create mode 100755 utils/pkgbuild.py delete mode 100755 utils/pkgbuild.sh create mode 100755 utils/pkgbuild_bootstrap delete mode 100755 utils/repodb.sh create mode 100644 utils/systemd/update-repo-notify.service create mode 100644 utils/systemd/update-repo-notify.socket create mode 100644 utils/systemd/update-repo@.service create mode 100755 utils/trigger.py create mode 100755 utils/update-db.py create mode 100644 utils/upload.php create mode 100755 utils/upload.py (limited to 'utils') diff --git a/utils/ci-upload.sh b/utils/ci-upload.sh deleted file mode 100755 index 9b1b822..0000000 --- a/utils/ci-upload.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeuo pipefail - -# clean up when exit -trap cleanup SIGINT SIGTERM ERR EXIT -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset ARCH - unset PKGDEST -} - -ARCH=$(uname -m) -PKGDEST="${PWD}/packages/$ARCH" - -if [ $(find $PKGDEST -name \*.pkg\* | wc -l) -eq 0 ]; then - echo -e "\033[0;31mCould not find any packaged packages, skip upload\033[0m" - if [ -r .fail ]; then - exit 2; - fi - exit 0 -fi - -curl -d "token=$UPLOAD_TOKEN&action=REQUIRE_CLEAN&arch=$ARCH" -fsSL "$REMOTE_PATH" - -pushd $PKGDEST - -for file in *.pkg*; do - echo "Start upload package $file size: $(ls -lh $file | awk '{print $5}')..." - curl -F "file=@$file" -F "token=$UPLOAD_TOKEN" -F "arch=$ARCH" -fsSL "$REMOTE_PATH" - #echo "Stop upload package $file" -done - -popd - -curl -d "token=$UPLOAD_TOKEN&action=UPLOADED&arch=$ARCH" -fsSL "$REMOTE_PATH" - -if [ -r .fail ]; then - exit 2; -fi diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py deleted file mode 100755 index e3f46f1..0000000 --- a/utils/ng/pkgbuild.py +++ /dev/null @@ -1,485 +0,0 @@ -#!/usr/bin/env python -from __future__ import annotations -import argparse -import asyncio -import dataclasses -import lzma -import logging -import operator -import os -import pathlib -import shutil -import subprocess -import time - -import aiofiles - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -PKGBUILD_DIRECTORY_BASE = "repo" - -CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "") -CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "") -CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "") -BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "") -TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve() -GNUPG_HOME = os.getenv("HOME") -FAIL_REPOS = {} - - -async def get_arch() -> str: - p = await asyncio.create_subprocess_exec("uname", "-m", stdout=subprocess.PIPE) - stdout, _ = await (p.communicate()) - return stdout.decode().lower().strip() - - -ARCH = asyncio.run(get_arch()) - - -@dataclasses.dataclass -class PackageVersionWithPath: - _version: PackageVersion - path: pathlib.Path - - @property - def name(self) -> str: - return self._version.name - - @property - def version(self) -> str: - return self._version.version - - @property - def arch(self) -> bool: - return self._version.arch_match - - -@dataclasses.dataclass -class PackageVersion: - name: str - version: str - arch_match: bool - - def __eq__(self, other: PackageVersion) -> bool: - return other.version == self.version - - def __gt__(self, other: PackageVersion) -> bool: - return self.version > other.version - - def __lt__(self, other: PackageVersion) -> bool: - return self.version < other.version - - def __le__(self, other: PackageVersion) -> bool: - return self < other or self == other - - def __ge__(self, other: PackageVersion) -> bool: - return self > other or self == other - - def __ne__(self, other: PackageVersion) -> bool: - return not self == other - - def __str__(self) -> str: - return f"{self.name} {self.version}" - - -def fetch_database(pkg_db: str) -> dict[str, str]: - name_read = False - version_read = False - pkg_name = "" - package_storage = {} - with lzma.open(pkg_db) as fin: - for line in fin: - line = line.strip() - if name_read: - pkg_name = line.decode() - name_read = False - elif version_read: - package_storage.update({pkg_name: line.decode()}) - # logger.debug("%s %s", pkg_name, line.decode()) - version_read = False - pkg_name = "" - if line == b"%NAME%": - assert not name_read - name_read = True - if line == b"%VERSION%": - assert not version_read - version_read = True - assert (name_read is False and version_read is False) or ( - version_read != name_read - ) - return package_storage - - -@dataclasses.dataclass(init=False) -class BuildEnvs: - src_dest: str - pkg_dest: str - makepkg_conf: str - signing_arg: list[str] - - def __init__(self): - cwd = pathlib.Path(os.getcwd()) - self.pkg_dest = str(cwd.joinpath("packages", ARCH).resolve()) - self.src_dest = str(cwd.joinpath("build").resolve()) - self.makepkg_conf = str(cwd.joinpath("makepkg_current.conf").resolve()) - if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != "": - self.signing_arg = ["--sign"] - else: - logger.info("Skip signing package") - - def get_dict(self) -> dict[str, str | list[str]]: - return { - "src_dest": self.src_dest, - "pkg_dest": self.pkg_dest, - "makepkg_conf": self.makepkg_conf, - "signing_arg": self.signing_arg, - } - - -def get_src_info(s: str) -> PackageVersion: - base = "" - ver = "" - rel = "" - arch = [] - for line in s.splitlines(False): - if (line := line.strip()).startswith("pkgver"): - ver = line.split("=", maxsplit=2)[1] - elif line.startswith("pkgrel"): - rel = line.split("=", maxsplit=2)[1] - elif line.startswith("pkgbase"): - base = line.split("=", maxsplit=2)[1] - elif line.startswith("arch"): - arch.append(line.split("=", maxsplit=2)[1].strip().lower()) - return PackageVersion( - base.strip(), f"{ver.strip()}-{rel.strip()}", ARCH in arch or "any" in arch - ) - - -async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersionWithPath: - if (src_path := pathlib.Path(path.joinpath(".SRCINFO"))).is_file(): - async with aiofiles.open(str(src_path)) as fin: - info = get_src_info(await fin.read()) - else: - p = await asyncio.create_subprocess_exec( - "makepkg", "--printsrcinfo", cwd=src_path.parent, stdout=subprocess.PIPE - ) - (stdout, stderr) = await p.communicate() - info = get_src_info(stdout.decode()) - return PackageVersionWithPath(info, pathlib.Path(path).resolve()) - - -async def get_build_target( - pkg_db: str, - home_directory: pathlib.Path, - build_target: list[str] | None, - build_all: bool, -) -> list[PackageVersionWithPath]: - - repo = fetch_database(pkg_db) - tasks = [] - pending_build = [] - - repo_directory = home_directory.joinpath(PKGBUILD_DIRECTORY_BASE) - - for folder in os.listdir(PKGBUILD_DIRECTORY_BASE): - if folder.startswith("."): - continue - - if (pkg_dir := repo_directory.joinpath(folder).resolve()).is_file(): - continue - - tasks.append(asyncio.create_task(parse_srcinfo(pkg_dir))) - - finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) - - if len(BUILD_OVERRIDE): - logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE) - override = BUILD_OVERRIDE.split(";") - return [ - future.result() for future in finished if future.result().name in override - ] - - if build_all: - return [future.result() for future in finished] - - if len(build_target): - return [ - future.result() - for future in finished - if future.result().name in build_target - ] - - for future in finished: - pkg = future.result() - if pkg.name in repo: - # print(pkg.name, repo[pkg.name], pkg.version) - if repo[pkg.name] < pkg.version: - pending_build.append(pkg) - else: - pending_build.append(pkg) - - # print(pending_build) - return pending_build - - -async def run_hook(origin_path: pathlib.Path, target: str) -> None: - if ( - hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve() - ).is_file(): - logger.info("Running %s hook script", target) - p = await asyncio.create_subprocess_exec( - str(hook_script), stdout=None, stderr=subprocess.STDOUT - ) - await p.wait() - if p.returncode != 0: - logger.error( - "Hook script return code: %d, skip other operation", p.returncode - ) - return - - -async def run_build_with_install( - src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str] -) -> int: - return await run_build(src_dest, pkg_dest, makepkg_conf, signing_arg, "-i") - - -async def run_build( - src_dest: str, - pkg_dest: str, - makepkg_conf: str, - signing_arg: list[str], - *other_args: str, -) -> int: - env = os.environ - env.update( - { - "SRCPKGDEST": src_dest, - "SRCDEST": src_dest, - "PKGDEST": pkg_dest, - "MAKEPKG_CONF": makepkg_conf, - } - ) - p = await asyncio.create_subprocess_exec( - "makepkg", - "--clean", - "-s", - *other_args, - *signing_arg, - "--asdeps", - "--noconfirm", - "--needed", - "--noprogressbar", - env=env, - stdout=None, - ) - await p.wait() - return p.returncode - - -async def install_dependency_via_yay(dependency: str) -> None: - await ( - await asyncio.create_subprocess_exec( - "yay", - "--noconfirm", - "--asdeps", - "--needed", - "--noprogressbar", - "-S", - dependency, - stdout=None, - ) - ).wait() - - -async def do_build(target: PackageVersionWithPath) -> int: - try: - os.chdir(str(target.path)) - logger.info("Building %s (%s)", target.name, target.path.name) - except OSError: - logger.exception("Got exception while chdir to %s", target.path) - - base_dir_name = target.path.name - - await run_hook(target.path, base_dir_name) - - if ( - yay_deps := target.path.parent.joinpath(".yaydeps") - .joinpath(base_dir_name) - .resolve() - ).is_file(): - async with aiofiles.open(str(yay_deps)) as fin: - while dep := (await fin.readline()).strip(): - logger.debug("Find dep %s, build first", dep) - if (dep_dir := target.path.parent.joinpath(dep).resolve()).is_dir(): - os.chdir(str(dep_dir)) - await run_hook(target.path, dep) - if not ( - ret := await run_build_with_install(**BUILD_ENVS.get_dict()) - ): - logger.error( - "Build dependencies package error, skipped next step" - ) - return ret - else: - await install_dependency_via_yay(dep) - - if ( - gpg_keys := target.path.parent.joinpath(".gpg_keys") - .joinpath(base_dir_name) - .resolve() - ).is_file(): - await ( - await asyncio.create_subprocess_exec( - "/bin/bash", str(gpg_keys), stdout=None - ) - ).wait() - await ( - await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None) - ).wait() - - os.chdir(str(target.path)) - build_ret = await run_build(**BUILD_ENVS.get_dict()) - - if build_ret != 0: - if build_ret == 13: - logger.warning("Already build %s, skipped.", base_dir_name) - return 0 - FAIL_REPOS.update({base_dir_name: build_ret}) - logger.warning("Build %s fail: %d", base_dir_name, build_ret) - - return build_ret - - -async def clean_installed_packages() -> int: - p = await asyncio.create_subprocess_exec( - "/bin/bash", - "-c", - "pacman -Qdtq | ifne sudo pacman --noconfirm -Rcns", - stdout=None, - ) - await p.wait() - return p.returncode - - -async def upload_packages() -> None: - remote_path = os.getenv("REMOTE_PATH") - upload_token = os.getenv("UPLOAD_TOKEN") - if remote_path is None or upload_token is None: - logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload") - await ( - await asyncio.create_subprocess_exec( - "./utils/ng/upload.py", - remote_path, - upload_token, - ARCH, - "--directory", - BUILD_ENVS.pkg_dest, - stdout=None, - ) - ).wait() - - -async def do_work( - pkg_db: str, - build_all: bool, - build_target: list[str], - fail_fast: bool = False, - run_auto_remove: bool = True, -) -> None: - - logger.info( - "build_all: %s, fail_fast: %s, auto_remove: %s", - build_all, - fail_fast, - run_auto_remove, - ) - - home_directory = pathlib.Path(os.getcwd()) - - build_target = sorted( - await get_build_target(pkg_db, home_directory, build_target, build_all), - key=operator.attrgetter("name"), - ) - logger.debug( - "Pending build summary (%d): %s", - len(build_target), - ", ".join(map(lambda x: x.name, build_target)), - ) - - for target in build_target: - if not target.arch: - logger.info("Skipped %s (Architecture not match)", target.name) - continue - if (await do_build(target)) != 0: - if fail_fast: - break - else: - await asyncio.sleep(5) - if run_auto_remove: - await clean_installed_packages() - for _ in range(3): - logger.debug("---------------------------------------------------------") - - if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir(): - shutil.rmtree(str(src_dst), ignore_errors=True) - - async with aiofiles.open( - str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve()), "w" - ) as fout: - await fout.write(f"{int(time.time())}") - - if len(FAIL_REPOS): - logger.error("Build failed repositories (%d):", len(FAIL_REPOS)) - logger.error( - "%s", - ", ".join(f"{repo}({ret_code})" for repo, ret_code in FAIL_REPOS.items()), - ) - - os.chdir(str(home_directory)) - if ret := await upload_packages(): - exit(ret) - - -def check_build_target(args: argparse.Namespace) -> bool: - if args.build_all or len(args.build_target): - return True - if "REBUILD ALL" in CI_COMMIT_TITLE or "REBUILD_ALL" in CI_COMMIT_TITLE: - return True - return False - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="pkgbuild.py") - parser.add_argument("db_file") - parser.add_argument("--build_all", action="store_true") - parser.add_argument("--build_target", nargs="*") - parser.add_argument("--sign", action="store_true") - parser.add_argument("--fail_fast", action="store_true") - args_ = parser.parse_args() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - - BUILD_ENVS = BuildEnvs() - if args_.sign: - BUILD_ENVS.signing_arg = ["--sign"] - asyncio.run( - do_work( - args_.db_file, - check_build_target(args_), - args_.build_target, - args_.fail_fast, - ) - ) diff --git a/utils/ng/pkgbuild_bootstrap b/utils/ng/pkgbuild_bootstrap deleted file mode 100755 index 847b8d5..0000000 --- a/utils/ng/pkgbuild_bootstrap +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [ -z "$DOCKER_SETUP_SCRIPT" ]; then - echo $GPG_PRIV_KEY | base64 -d | gpg --import - cd "$(dirname "$0")" - cd ../../ - echo "Current pwd: $(pwd)" -fi - -set -Eeo pipefail -trap cleanup SIGINT SIGTERM ERR EXIT - -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset SRCDEST - unset CONFDEST - unset TMPCONF - unset FLAGS - unset ARCH -} - -ARCH=$(uname -m) -CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf" - -TMPCONF="makepkg_current.conf" -cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF" -cat "$CONFDEST" >> "$TMPCONF" - -FLAGS="" - -if [ "$EUID" -eq 0 ] ; then - echo "\033[0;32mPlease run this script as non-root\033[0m" - exit 3 -fi - -if [ -z "$REMOTE_DATABASE" ]; then - echo "\033[0;32mYou should specify remote address\033[0m" - exit 4 -fi - -if [ -n "$FORCE_BUILD_ALL" ]; then - FLAGS+="--build_all " -fi - -if [ -n "$BUILD_FAIL_FAST" ]; then - FLAGS+="--fail_fast " -fi - -curl -fL -o remote.db "$REMOTE_DATABASE" - -sudo pacman -Sy - -./utils/ng/pkgbuild.py remote.db $FLAGS diff --git a/utils/ng/systemd/update-repo-notify.service b/utils/ng/systemd/update-repo-notify.service deleted file mode 100644 index d7e3c9a..0000000 --- a/utils/ng/systemd/update-repo-notify.service +++ /dev/null @@ -1,5 +0,0 @@ -[Unit] -Description=Update repo notify service - -[Service] -ExecStart=/var/lib/update/trigger.py \ No newline at end of file diff --git a/utils/ng/systemd/update-repo-notify.socket b/utils/ng/systemd/update-repo-notify.socket deleted file mode 100644 index f14743c..0000000 --- a/utils/ng/systemd/update-repo-notify.socket +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Update repo notify socket - -[Socket] -# If /var/run is not implemented as a symlink to /run, you may need to -# specify ListenStream=/var/run/docker.sock instead. -ListenStream=/run/update-repo-notify.sock -SocketMode=0660 -SocketUser=update-database -SocketGroup=http -#Accept=yes -MaxConnections=1 -FlushPending=yes - -[Install] -WantedBy=sockets.target \ No newline at end of file diff --git a/utils/ng/systemd/update-repo@.service b/utils/ng/systemd/update-repo@.service deleted file mode 100644 index 6fa98da..0000000 --- a/utils/ng/systemd/update-repo@.service +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Update repo %i service - -[Service] -User=update-database -ExecStart=/var/lib/update/update-db.py /var/lib/update %i AAAABBBBCCCCDDDDDDDDCCCCBBBBAAAA -#ExecStart=rsync \ No newline at end of file diff --git a/utils/ng/trigger.py b/utils/ng/trigger.py deleted file mode 100755 index ca0cfd1..0000000 --- a/utils/ng/trigger.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -import argparse -import logging -import os -import signal -import socket -import subprocess -import threading -from socketserver import StreamRequestHandler, TCPServer - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -class Handler(StreamRequestHandler): - def handle(self): - while line := self.rfile.readline(): - if b"ARCH=" in line: - ( - subprocess.Popen( - [ - "systemctl", - "--no-block", - "start", - f"update-repo@{line.split(b'=')[1].decode().strip()}", - ] - ) - ).wait() - os.kill(os.getpid(), signal.SIGUSR1) - return - logger.info("Recv: %s", line) - - -# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4 -class Server(TCPServer): - # The constant would be better initialized by a systemd module - SYSTEMD_FIRST_SOCKET_FD = 3 - - def __init__(self, server_address, handler_cls): - # Invoke base but omit bind/listen steps (performed by systemd activation!) - TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False) - # Override socket - self.socket = socket.fromfd( - self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type - ) - - -def main(): - def server_shutdown_wrapper(*_args) -> None: - logger.debug("Shutdown the server") - threading.Thread( - target=server.shutdown, - daemon=True, - ).start() - - server = Server(("localhost", 0), Handler) - signal.signal(signal.SIGUSR1, server_shutdown_wrapper) - - server.serve_forever() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - - main() diff --git a/utils/ng/update-db.py b/utils/ng/update-db.py deleted file mode 100755 index acdc8c8..0000000 --- a/utils/ng/update-db.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -import argparse -import asyncio -import logging -import os -import pathlib -import shutil -import subprocess - - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -async def add_package(dest: pathlib.Path, pkg_name: str, key: str) -> None: - await ( - await asyncio.create_subprocess_exec( - "repo-add", - str(dest), - pkg_name, - "-s", - "-v", - "-R", - "-k", - key, - stdout=subprocess.STDOUT, - ) - ).wait() - - -async def main(remote_dest: str, repo_base_name: str, sign_key: str) -> None: - cwd = pathlib.Path(os.getcwd()).resolve() - remote_dest = pathlib.Path(remote_dest).resolve() - database_dest = cwd.joinpath(repo_base_name).resolve() - - for file in os.listdir(str(remote_dest)): - logger.debug("Copy file: %s", file) - shutil.copy(str(remote_dest.joinpath(file).joinpath()), str(cwd)) - - for file in os.listdir(str(remote_dest)): - if ".pkg.tar.zst" in file: - await add_package(database_dest, file.split(".pkg.tar.zst")[0], sign_key) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("update-db.py") - parser.add_argument("cwd") - parser.add_argument("basename") - parser.add_argument("sign_key") - - args_ = parser.parse_args() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - main(args_.cwd, args_.basename, args_.sign_key) diff --git a/utils/ng/upload.php b/utils/ng/upload.php deleted file mode 100644 index da848eb..0000000 --- a/utils/ng/upload.php +++ /dev/null @@ -1,35 +0,0 @@ - diff --git a/utils/ng/upload.py b/utils/ng/upload.py deleted file mode 100755 index 07f8628..0000000 --- a/utils/ng/upload.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -import argparse -import asyncio -import logging -import os - -import aiohttp - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -def build_data(args: argparse.Namespace, action: str) -> dict[str, str]: - return {"token": args.token, "arch": args.arch, "action": action} - - -# source: https://stackoverflow.com/a/1094933 -def sizeof_fmt(num: int, suffix: str = "B"): - for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: - if abs(num) < 1024.0: - return f"{num:3.1f}{unit}{suffix}" - num /= 1024.0 - return f"{num:.1f}Yi{suffix}" - - -async def send_file( - session: aiohttp.ClientSession, args: argparse.Namespace, file: str -) -> None: - data = aiohttp.FormData() - data.add_field("file", open(file, "rb"), filename=file) - data.add_field("token", args.token) - data.add_field("arch", args.arch) - async with session.post(args.remote_address, data=data): - pass - - -async def main(args: argparse.Namespace) -> int: - pending_upload = [] - for file in os.listdir("."): - if ".pkg" in file: - pending_upload.append(file) - - if not len(pending_upload): - logger.debug("Could not find any packaged packages, skip upload") - return 0 - - async with aiohttp.ClientSession( - raise_for_status=True, timeout=aiohttp.ClientTimeout(60) - ) as session: - async with session.post( - args.remote_address, data=build_data(args, "REQUIRE_CLEAN") - ): - pass - for file in pending_upload: - logger.info("Start upload %s (%s)", sizeof_fmt(os.stat(file).st_size)) - await send_file(session, args, file) - async with session.post(args.remote_address, data=build_data(args, "UPLOADED")): - pass - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("upload.py") - parser.add_argument("remote_address") - parser.add_argument("token") - parser.add_argument("arch") - parser.add_argument("--directory") - args_ = parser.parse_args() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - - if args_.directory is not None: - os.chdir(args_.directory) - - exit(asyncio.run(main(args_))) diff --git a/utils/pkgbuild.py b/utils/pkgbuild.py new file mode 100755 index 0000000..a36f620 --- /dev/null +++ b/utils/pkgbuild.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python +from __future__ import annotations +import argparse +import asyncio +import dataclasses +import lzma +import logging +import operator +import os +import pathlib +import shutil +import subprocess +import time + +import aiofiles + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +PKGBUILD_DIRECTORY_BASE = "repo" + +CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "") +CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "") +CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "") +BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "") +TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve() +GNUPG_HOME = os.getenv("HOME") +FAIL_REPOS = {} + + +async def get_arch() -> str: + p = await asyncio.create_subprocess_exec("uname", "-m", stdout=subprocess.PIPE) + stdout, _ = await (p.communicate()) + return stdout.decode().lower().strip() + + +ARCH = asyncio.run(get_arch()) + + +@dataclasses.dataclass +class PackageVersionWithPath: + _version: PackageVersion + path: pathlib.Path + + @property + def name(self) -> str: + return self._version.name + + @property + def version(self) -> str: + return self._version.version + + @property + def arch(self) -> bool: + return self._version.arch_match + + +@dataclasses.dataclass +class PackageVersion: + name: str + version: str + arch_match: bool + + def __eq__(self, other: PackageVersion) -> bool: + return other.version == self.version + + def __gt__(self, other: PackageVersion) -> bool: + return self.version > other.version + + def __lt__(self, other: PackageVersion) -> bool: + return self.version < other.version + + def __le__(self, other: PackageVersion) -> bool: + return self < other or self == other + + def __ge__(self, other: PackageVersion) -> bool: + return self > other or self == other + + def __ne__(self, other: PackageVersion) -> bool: + return not self == other + + def __str__(self) -> str: + return f"{self.name} {self.version}" + + +def fetch_database(pkg_db: str) -> dict[str, str]: + name_read = False + version_read = False + pkg_name = "" + package_storage = {} + with lzma.open(pkg_db) as fin: + for line in fin: + line = line.strip() + if name_read: + pkg_name = line.decode() + name_read = False + elif version_read: + package_storage.update({pkg_name: line.decode()}) + # logger.debug("%s %s", pkg_name, line.decode()) + version_read = False + pkg_name = "" + if line == b"%NAME%": + assert not name_read + name_read = True + if line == b"%VERSION%": + assert not version_read + version_read = True + assert (name_read is False and version_read is False) or ( + version_read != name_read + ) + return package_storage + + +@dataclasses.dataclass(init=False) +class BuildEnvs: + src_dest: str + pkg_dest: str + makepkg_conf: str + signing_arg: list[str] + + def __init__(self): + cwd = pathlib.Path(os.getcwd()) + self.pkg_dest = str(cwd.joinpath("packages", ARCH).resolve()) + self.src_dest = str(cwd.joinpath("build").resolve()) + self.makepkg_conf = str(cwd.joinpath("makepkg_current.conf").resolve()) + if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != "": + self.signing_arg = ["--sign"] + else: + logger.info("Skip signing package") + + def get_dict(self) -> dict[str, str | list[str]]: + return { + "src_dest": self.src_dest, + "pkg_dest": self.pkg_dest, + "makepkg_conf": self.makepkg_conf, + "signing_arg": self.signing_arg, + } + + +def get_src_info(s: str) -> PackageVersion: + base = "" + ver = "" + rel = "" + arch = [] + for line in s.splitlines(False): + if (line := line.strip()).startswith("pkgver"): + ver = line.split("=", maxsplit=2)[1] + elif line.startswith("pkgrel"): + rel = line.split("=", maxsplit=2)[1] + elif line.startswith("pkgbase"): + base = line.split("=", maxsplit=2)[1] + elif line.startswith("arch"): + arch.append(line.split("=", maxsplit=2)[1].strip().lower()) + return PackageVersion( + base.strip(), f"{ver.strip()}-{rel.strip()}", ARCH in arch or "any" in arch + ) + + +async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersionWithPath: + if (src_path := pathlib.Path(path.joinpath(".SRCINFO"))).is_file(): + async with aiofiles.open(str(src_path)) as fin: + info = get_src_info(await fin.read()) + else: + p = await asyncio.create_subprocess_exec( + "makepkg", "--printsrcinfo", cwd=src_path.parent, stdout=subprocess.PIPE + ) + (stdout, stderr) = await p.communicate() + info = get_src_info(stdout.decode()) + return PackageVersionWithPath(info, pathlib.Path(path).resolve()) + + +async def get_build_target( + pkg_db: str, + home_directory: pathlib.Path, + build_target: list[str] | None, + build_all: bool, +) -> list[PackageVersionWithPath]: + + repo = fetch_database(pkg_db) + tasks = [] + pending_build = [] + + repo_directory = home_directory.joinpath(PKGBUILD_DIRECTORY_BASE) + + for folder in os.listdir(PKGBUILD_DIRECTORY_BASE): + if folder.startswith("."): + continue + + if (pkg_dir := repo_directory.joinpath(folder).resolve()).is_file(): + continue + + tasks.append(asyncio.create_task(parse_srcinfo(pkg_dir))) + + finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + + if len(BUILD_OVERRIDE): + logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE) + override = BUILD_OVERRIDE.split(";") + return [ + future.result() for future in finished if future.result().name in override + ] + + if build_all: + return [future.result() for future in finished] + + if len(build_target): + return [ + future.result() + for future in finished + if future.result().name in build_target + ] + + for future in finished: + pkg = future.result() + if pkg.name in repo: + # print(pkg.name, repo[pkg.name], pkg.version) + if repo[pkg.name] < pkg.version: + pending_build.append(pkg) + else: + pending_build.append(pkg) + + # print(pending_build) + return pending_build + + +async def run_hook(origin_path: pathlib.Path, target: str) -> None: + if ( + hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve() + ).is_file(): + logger.info("Running %s hook script", target) + p = await asyncio.create_subprocess_exec( + str(hook_script), stdout=None, stderr=subprocess.STDOUT + ) + await p.wait() + if p.returncode != 0: + logger.error( + "Hook script return code: %d, skip other operation", p.returncode + ) + return + + +async def run_build_with_install( + src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str] +) -> int: + return await run_build(src_dest, pkg_dest, makepkg_conf, signing_arg, "-i") + + +async def run_build( + src_dest: str, + pkg_dest: str, + makepkg_conf: str, + signing_arg: list[str], + *other_args: str, +) -> int: + env = os.environ + env.update( + { + "SRCPKGDEST": src_dest, + "SRCDEST": src_dest, + "PKGDEST": pkg_dest, + "MAKEPKG_CONF": makepkg_conf, + } + ) + p = await asyncio.create_subprocess_exec( + "makepkg", + "--clean", + "-s", + *other_args, + *signing_arg, + "--asdeps", + "--noconfirm", + "--needed", + "--noprogressbar", + env=env, + stdout=None, + ) + await p.wait() + return p.returncode + + +async def install_dependency_via_yay(dependency: str) -> None: + await ( + await asyncio.create_subprocess_exec( + "yay", + "--noconfirm", + "--asdeps", + "--needed", + "--noprogressbar", + "-S", + dependency, + stdout=None, + ) + ).wait() + + +async def do_build(target: PackageVersionWithPath) -> int: + try: + os.chdir(str(target.path)) + logger.info("Building %s (%s)", target.name, target.path.name) + except OSError: + logger.exception("Got exception while chdir to %s", target.path) + + base_dir_name = target.path.name + + await run_hook(target.path, base_dir_name) + + if ( + yay_deps := target.path.parent.joinpath(".yaydeps") + .joinpath(base_dir_name) + .resolve() + ).is_file(): + async with aiofiles.open(str(yay_deps)) as fin: + while dep := (await fin.readline()).strip(): + logger.debug("Find dep %s, build first", dep) + if (dep_dir := target.path.parent.joinpath(dep).resolve()).is_dir(): + os.chdir(str(dep_dir)) + await run_hook(target.path, dep) + if not ( + ret := await run_build_with_install(**BUILD_ENVS.get_dict()) + ): + logger.error( + "Build dependencies package error, skipped next step" + ) + return ret + else: + await install_dependency_via_yay(dep) + + if ( + gpg_keys := target.path.parent.joinpath(".gpg_keys") + .joinpath(base_dir_name) + .resolve() + ).is_file(): + await ( + await asyncio.create_subprocess_exec( + "/bin/bash", str(gpg_keys), stdout=None + ) + ).wait() + await ( + await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None) + ).wait() + + os.chdir(str(target.path)) + build_ret = await run_build(**BUILD_ENVS.get_dict()) + + if build_ret != 0: + if build_ret == 13: + logger.warning("Already build %s, skipped.", base_dir_name) + return 0 + FAIL_REPOS.update({base_dir_name: build_ret}) + logger.warning("Build %s fail: %d", base_dir_name, build_ret) + + return build_ret + + +async def clean_installed_packages() -> int: + p = await asyncio.create_subprocess_exec( + "/bin/bash", + "-c", + "pacman -Qdtq | ifne sudo pacman --noconfirm -Rcns", + stdout=None, + ) + await p.wait() + return p.returncode + + +async def upload_packages() -> None: + remote_path = os.getenv("REMOTE_PATH") + upload_token = os.getenv("UPLOAD_TOKEN") + if remote_path is None or upload_token is None: + logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload") + await ( + await asyncio.create_subprocess_exec( + "./utils/upload.py", + remote_path, + upload_token, + ARCH, + "--directory", + BUILD_ENVS.pkg_dest, + stdout=None, + ) + ).wait() + + +async def do_work( + pkg_db: str, + build_all: bool, + build_target: list[str], + fail_fast: bool = False, + run_auto_remove: bool = True, +) -> None: + + logger.info( + "build_all: %s, fail_fast: %s, auto_remove: %s", + build_all, + fail_fast, + run_auto_remove, + ) + + home_directory = pathlib.Path(os.getcwd()) + + build_target = sorted( + await get_build_target(pkg_db, home_directory, build_target, build_all), + key=operator.attrgetter("name"), + ) + logger.debug( + "Pending build summary (%d): %s", + len(build_target), + ", ".join(map(lambda x: x.name, build_target)), + ) + + for target in build_target: + if not target.arch: + logger.info("Skipped %s (Architecture not match)", target.name) + continue + if (await do_build(target)) != 0: + if fail_fast: + break + else: + await asyncio.sleep(5) + if run_auto_remove: + await clean_installed_packages() + for _ in range(3): + logger.debug("---------------------------------------------------------") + + if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir(): + shutil.rmtree(str(src_dst), ignore_errors=True) + + async with aiofiles.open( + str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve()), "w" + ) as fout: + await fout.write(f"{int(time.time())}") + + if len(FAIL_REPOS): + logger.error("Build failed repositories (%d):", len(FAIL_REPOS)) + logger.error( + "%s", + ", ".join(f"{repo}({ret_code})" for repo, ret_code in FAIL_REPOS.items()), + ) + + os.chdir(str(home_directory)) + if ret := await upload_packages(): + exit(ret) + + +def check_build_target(args: argparse.Namespace) -> bool: + if args.build_all or len(args.build_target): + return True + if "REBUILD ALL" in CI_COMMIT_TITLE or "REBUILD_ALL" in CI_COMMIT_TITLE: + return True + return False + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="pkgbuild.py") + parser.add_argument("db_file") + parser.add_argument("--build_all", action="store_true") + parser.add_argument("--build_target", nargs="*") + parser.add_argument("--sign", action="store_true") + parser.add_argument("--fail_fast", action="store_true") + args_ = parser.parse_args() + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + + BUILD_ENVS = BuildEnvs() + if args_.sign: + BUILD_ENVS.signing_arg = ["--sign"] + asyncio.run( + do_work( + args_.db_file, + check_build_target(args_), + args_.build_target, + args_.fail_fast, + ) + ) diff --git a/utils/pkgbuild.sh b/utils/pkgbuild.sh deleted file mode 100755 index 0342096..0000000 --- a/utils/pkgbuild.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeo pipefail - -# clean up when exit -trap cleanup SIGINT SIGTERM ERR EXIT -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset SIGNING_ARG - unset UNSUCCESSFUL - unset SKIP_VERIFIED - unset FOLDER_NAME - unset PKGDEST - unset SRCDEST - unset CONFDEST - unset REPO_DEST - rm -f "$TMPCONF" - if ! { [ $# -gt 0 ] && [ "$1" = "--diff" ]; }; then - rm -f "$REPO_DIFF" - fi - [ -r "$FAIL_REPOS" ] && rm -f "$FAIL_REPOS" - unset TMPCONF - unset REPO_PENDING - unset REPO_DIFF - unset PKGBUILD_DIRECTORY_BASE - unset ARCH -} - -function hook { - if [ -r ../.hook/"$1" ]; then - echo "Running $1 Hook script" - ../.hook/"$1" - fi -} - -ARCH=$(uname -m) -PKGBUILD_DIRECTORY_BASE="repo" -FAIL_REPOS="${PWD}/failure_repos" -PKGDEST="${PWD}/packages/$ARCH" -SRCDEST="${PWD}/build" -CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf" -REPO_DIFF="${PWD}/packages/$ARCH/DIFF" -SKIP_VERIFIED=1 -UNSUCCESSFUL=0 -SIGNING_ARG="--nosign" - -TMPCONF=$(mktemp -t makepkg.$ARCH.conf.XXXXXXXXXX) || exit 1 -cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF" -cat "$CONFDEST" >> "$TMPCONF" - -touch "$REPO_DIFF" - -function get_diff_list { - git diff --name-only HEAD^ | while read line; do - if [[ $line =~ $PKGBUILD_DIRECTORY_BASE/\..+ ]] && [ "$line" != "$PKGBUILD_DIRECTORY_BASE/.verified_repos" ]; then - echo "$line" | cut -d'/' -f3 >> "$REPO_DIFF" - elif [[ $line =~ $PKGBUILD_DIRECTORY_BASE ]]; then - echo "$line" | cut -d'/' -f2 >> "$REPO_DIFF" - fi - done - cp "$REPO_DIFF" "${REPO_DIFF}_tmp" - sort -u "${REPO_DIFF}_tmp" > "$REPO_DIFF" - rm -f "${REPO_DIFF}_tmp" -} - -if [ $# -gt 0 ];then - if [ "$1" = "--diff" ]; then - get_diff_list - exit 0 - elif [ "$1" = "--import-gpg" ]; then - if [ -n "$CI_COMMIT_BRANCH" ] && [ -n "$CI_DEFAULT_BRANCH" ] && [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - echo $GPG_PRIV_KEY | base64 -d | gpg --import - else - echo -e "\033[0;32mSkip import gpg key\033[0m" - fi - exit 0 - fi -fi - -if [ "$EUID" -eq 0 ] ; then - echo "\033[0;32mPlease run this script as non-root\033[0m" - exit 3 -fi - -if [ $# -gt 0 ] && [ "$1" = "--all" ] || [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ fix\(repo\)|REBUILD ]]; then - if [ -z "${FORCE_ALL+x}" ] || [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ REBUILD(\ |_)ALL ]]; then - SKIP_VERIFIED=0 - fi - ls $PKGBUILD_DIRECTORY_BASE > "$REPO_DIFF" -else - get_diff_list -fi - -if [ -n "$CI_COMMIT_BRANCH" ] && [ -n "$CI_DEFAULT_BRANCH" ] && [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - SIGNING_ARG="--sign" -else - echo -e "\033[0;32mSkip signing package\033[0m" -fi - -pushd $PKGBUILD_DIRECTORY_BASE || ( echo -e "\033[0;31mError, can't switch to pkgbuild directory\033[0m" && exit 1 ) - -while read -r FOLDER_NAME ; do - if [ ! -d "$FOLDER_NAME" ]; then - echo -e "\033[0;32mSkip folder $FOLDER_NAME\033[0m" - continue - fi - echo -e "\033[0;32mProcessing $FOLDER_NAME\033[0m" - pushd "$FOLDER_NAME" || { echo -e "\033[1;33mWarning, can't chdir to $FOLDER_NAME, skipped\033[0m"; continue; } - - if [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ fix\(repo\)|REBUILD ]] && \ - [ $SKIP_VERIFIED -eq 1 ] && grep -Fxq "$FOLDER_NAME" "../.verified_repos"; then - echo -e "\033[0;32mSkip folder $FOLDER_NAME (verified repository)\033[0m" - popd - continue - fi - - # hook must run before other checker - hook "$FOLDER_NAME" - - # Check PKGBUILD architecture - if ! ( pcregrep -M 'arch=\([^\)]*\)' PKGBUILD | grep -E "$ARCH|any" >/dev/null ); then - echo -e "\033[0;32mSkip folder $FOLDER_NAME (Architecture not match)\033[0m" - popd - continue - fi - - if [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ fix\(repo\)|REBUILD ]] && \ - [ $SKIP_VERIFIED -eq 1 ] && grep -Fxq "$FOLDER_NAME" "../.verified_repos"; then - echo -e "\033[0;32mSkip folder $FOLDER_NAME (verified repository)\033[0m" - popd - continue - fi - - if [ -r ../.yaydeps/"$FOLDER_NAME" ]; then - while read -r YAYDEP ; do - if [ -d "../$YAYDEP" ]; then - pushd "../$YAYDEP" - hook "$YAYDEP" - SRCPKGDEST=$SRCDEST SRCDEST=$SRCDEST PKGDEST=$PKGDEST MAKEPKG_CONF="$TMPCONF" makepkg --clean -s -i $SIGNING_ARG --asdeps --noconfirm --needed --noprogressbar - popd - else - yay -S --noconfirm --needed --asdeps "$YAYDEP" - fi - done < ../.yaydeps/"$FOLDER_NAME" - fi - - if [ -r ../.gpg_keys/"$FOLDER_NAME" ]; then - . ../.gpg_keys/"$FOLDER_NAME" - # source: https://stackoverflow.com/a/53886735 - gpg --list-keys --fingerprint | grep pub -A 1 | grep -Ev "pub|--" | tr -d ' ' \ - | awk 'BEGIN { FS = "\n" } ; { print $1":6:" } ' | gpg --import-ownertrust - fi - - SRCPKGDEST=$SRCDEST SRCDEST=$SRCDEST PKGDEST=$PKGDEST MAKEPKG_CONF="$TMPCONF" makepkg -s --clean $SIGNING_ARG --asdeps --noconfirm --needed --noprogressbar || { - LAST_STATUS=$?; - if [ $LAST_STATUS -eq 13 ]; then - echo -e "\033[0;32mSkip folder that package already build $FOLDER_NAME\033[0m"; - continue - fi - echo "$FOLDER_NAME" >> "$FAIL_REPOS" - echo -e "\033[0;31mSkip folder $FOLDER_NAME. Build fail: $LAST_STATUS\033[0m"; - UNSUCCESSFUL=1; - } - [ -d "$SRCDEST" ] && rm -r "$SRCDEST" - popd -done < "$REPO_DIFF" - -popd - -date +%s > "$PKGDEST/LASTBUILD" - -if [ $UNSUCCESSFUL -eq 1 ]; then - [ -d "$SRCDEST" ] && rm -r "$SRCDEST" - echo -e "\033[0;31mBuild failed repositories:\033[0m" - cat "$FAIL_REPOS" - if [ -n "$CI_DEFAULT_BRANCH" ] && [ -n "$CI_COMMIT_BRANCH" ] && [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - touch .fail - echo -e "\033[0;31mCreate fail flag because \$UNSUCCESSFUL set to 1\033[0m" - else - echo -e "\033[0;31mExit due makepkg failed\033[0m" - exit 2 - fi -fi diff --git a/utils/pkgbuild_bootstrap b/utils/pkgbuild_bootstrap new file mode 100755 index 0000000..7f251f8 --- /dev/null +++ b/utils/pkgbuild_bootstrap @@ -0,0 +1,53 @@ +#!/bin/bash + +if [ -z "$DOCKER_SETUP_SCRIPT" ]; then + echo $GPG_PRIV_KEY | base64 -d | gpg --import + cd "$(dirname "$0")" + cd ../../ + echo "Current pwd: $(pwd)" +fi + +set -Eeo pipefail +trap cleanup SIGINT SIGTERM ERR EXIT + +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + unset SRCDEST + unset CONFDEST + unset TMPCONF + unset FLAGS + unset ARCH +} + +ARCH=$(uname -m) +CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf" + +TMPCONF="makepkg_current.conf" +cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF" +cat "$CONFDEST" >> "$TMPCONF" + +FLAGS="" + +if [ "$EUID" -eq 0 ] ; then + echo "\033[0;32mPlease run this script as non-root\033[0m" + exit 3 +fi + +if [ -z "$REMOTE_DATABASE" ]; then + echo "\033[0;32mYou should specify remote address\033[0m" + exit 4 +fi + +if [ -n "$FORCE_BUILD_ALL" ]; then + FLAGS+="--build_all " +fi + +if [ -n "$BUILD_FAIL_FAST" ]; then + FLAGS+="--fail_fast " +fi + +curl -fL -o remote.db "$REMOTE_DATABASE" + +sudo pacman -Sy + +./utils/pkgbuild.py remote.db $FLAGS diff --git a/utils/repodb.sh b/utils/repodb.sh deleted file mode 100755 index 212b137..0000000 --- a/utils/repodb.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeuo pipefail - -# clean up when exit -trap cleanup SIGINT SIGTERM ERR EXIT -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset ARCH - unset PKGDEST - unset REPO_DEST - unset GPG_KEY - unset PACKAGE_NAME - unset LATEST -} - -if [ -z $ARCH ]; then - ARCH=$(uname -m) - echo "Warning: You should specify arch environment variable, use current platform: $ARCH" -fi - -REPO_BASE_NAME="kunoisayami" -#ARCH=$(uname -m) -REPO_DEST="${PWD}/packages/$ARCH/$REPO_BASE_NAME.db.tar.xz" -GPG_KEY="4A0F0C8BC709ACA4341767FB243975C8DB9656B9" - -while read PACKAGE_NAME; do - #LATEST=$(ls -t "$PKGDEST/$PACKAGE_NAME"* | grep -v ".sig" | head -n 1 | cut -d' ' -f1) - if [ -z $PACKAGE_NAME ]; then - continue - fi - repo-add "$REPO_DEST" "$PACKAGE_NAME" -s -v -R -k $GPG_KEY -done diff --git a/utils/systemd/update-repo-notify.service b/utils/systemd/update-repo-notify.service new file mode 100644 index 0000000..d7e3c9a --- /dev/null +++ b/utils/systemd/update-repo-notify.service @@ -0,0 +1,5 @@ +[Unit] +Description=Update repo notify service + +[Service] +ExecStart=/var/lib/update/trigger.py \ No newline at end of file diff --git a/utils/systemd/update-repo-notify.socket b/utils/systemd/update-repo-notify.socket new file mode 100644 index 0000000..f14743c --- /dev/null +++ b/utils/systemd/update-repo-notify.socket @@ -0,0 +1,16 @@ +[Unit] +Description=Update repo notify socket + +[Socket] +# If /var/run is not implemented as a symlink to /run, you may need to +# specify ListenStream=/var/run/docker.sock instead. +ListenStream=/run/update-repo-notify.sock +SocketMode=0660 +SocketUser=update-database +SocketGroup=http +#Accept=yes +MaxConnections=1 +FlushPending=yes + +[Install] +WantedBy=sockets.target \ No newline at end of file diff --git a/utils/systemd/update-repo@.service b/utils/systemd/update-repo@.service new file mode 100644 index 0000000..6fa98da --- /dev/null +++ b/utils/systemd/update-repo@.service @@ -0,0 +1,7 @@ +[Unit] +Description=Update repo %i service + +[Service] +User=update-database +ExecStart=/var/lib/update/update-db.py /var/lib/update %i AAAABBBBCCCCDDDDDDDDCCCCBBBBAAAA +#ExecStart=rsync \ No newline at end of file diff --git a/utils/trigger.py b/utils/trigger.py new file mode 100755 index 0000000..ca0cfd1 --- /dev/null +++ b/utils/trigger.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +import argparse +import logging +import os +import signal +import socket +import subprocess +import threading +from socketserver import StreamRequestHandler, TCPServer + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +class Handler(StreamRequestHandler): + def handle(self): + while line := self.rfile.readline(): + if b"ARCH=" in line: + ( + subprocess.Popen( + [ + "systemctl", + "--no-block", + "start", + f"update-repo@{line.split(b'=')[1].decode().strip()}", + ] + ) + ).wait() + os.kill(os.getpid(), signal.SIGUSR1) + return + logger.info("Recv: %s", line) + + +# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4 +class Server(TCPServer): + # The constant would be better initialized by a systemd module + SYSTEMD_FIRST_SOCKET_FD = 3 + + def __init__(self, server_address, handler_cls): + # Invoke base but omit bind/listen steps (performed by systemd activation!) + TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False) + # Override socket + self.socket = socket.fromfd( + self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type + ) + + +def main(): + def server_shutdown_wrapper(*_args) -> None: + logger.debug("Shutdown the server") + threading.Thread( + target=server.shutdown, + daemon=True, + ).start() + + server = Server(("localhost", 0), Handler) + signal.signal(signal.SIGUSR1, server_shutdown_wrapper) + + server.serve_forever() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + + main() diff --git a/utils/update-db.py b/utils/update-db.py new file mode 100755 index 0000000..acdc8c8 --- /dev/null +++ b/utils/update-db.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +import argparse +import asyncio +import logging +import os +import pathlib +import shutil +import subprocess + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +async def add_package(dest: pathlib.Path, pkg_name: str, key: str) -> None: + await ( + await asyncio.create_subprocess_exec( + "repo-add", + str(dest), + pkg_name, + "-s", + "-v", + "-R", + "-k", + key, + stdout=subprocess.STDOUT, + ) + ).wait() + + +async def main(remote_dest: str, repo_base_name: str, sign_key: str) -> None: + cwd = pathlib.Path(os.getcwd()).resolve() + remote_dest = pathlib.Path(remote_dest).resolve() + database_dest = cwd.joinpath(repo_base_name).resolve() + + for file in os.listdir(str(remote_dest)): + logger.debug("Copy file: %s", file) + shutil.copy(str(remote_dest.joinpath(file).joinpath()), str(cwd)) + + for file in os.listdir(str(remote_dest)): + if ".pkg.tar.zst" in file: + await add_package(database_dest, file.split(".pkg.tar.zst")[0], sign_key) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("update-db.py") + parser.add_argument("cwd") + parser.add_argument("basename") + parser.add_argument("sign_key") + + args_ = parser.parse_args() + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + main(args_.cwd, args_.basename, args_.sign_key) diff --git a/utils/upload.php b/utils/upload.php new file mode 100644 index 0000000..da848eb --- /dev/null +++ b/utils/upload.php @@ -0,0 +1,35 @@ + diff --git a/utils/upload.py b/utils/upload.py new file mode 100755 index 0000000..07f8628 --- /dev/null +++ b/utils/upload.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +import argparse +import asyncio +import logging +import os + +import aiohttp + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +def build_data(args: argparse.Namespace, action: str) -> dict[str, str]: + return {"token": args.token, "arch": args.arch, "action": action} + + +# source: https://stackoverflow.com/a/1094933 +def sizeof_fmt(num: int, suffix: str = "B"): + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + + +async def send_file( + session: aiohttp.ClientSession, args: argparse.Namespace, file: str +) -> None: + data = aiohttp.FormData() + data.add_field("file", open(file, "rb"), filename=file) + data.add_field("token", args.token) + data.add_field("arch", args.arch) + async with session.post(args.remote_address, data=data): + pass + + +async def main(args: argparse.Namespace) -> int: + pending_upload = [] + for file in os.listdir("."): + if ".pkg" in file: + pending_upload.append(file) + + if not len(pending_upload): + logger.debug("Could not find any packaged packages, skip upload") + return 0 + + async with aiohttp.ClientSession( + raise_for_status=True, timeout=aiohttp.ClientTimeout(60) + ) as session: + async with session.post( + args.remote_address, data=build_data(args, "REQUIRE_CLEAN") + ): + pass + for file in pending_upload: + logger.info("Start upload %s (%s)", sizeof_fmt(os.stat(file).st_size)) + await send_file(session, args, file) + async with session.post(args.remote_address, data=build_data(args, "UPLOADED")): + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("upload.py") + parser.add_argument("remote_address") + parser.add_argument("token") + parser.add_argument("arch") + parser.add_argument("--directory") + args_ = parser.parse_args() + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + + if args_.directory is not None: + os.chdir(args_.directory) + + exit(asyncio.run(main(args_))) -- cgit v1.3.1