diff options
| -rwxr-xr-x | utils/ng/pkgbuild.py | 54 | ||||
| -rwxr-xr-x | utils/ng/pkgbuild_bootstrap | 2 | ||||
| -rwxr-xr-x | utils/ng/upload.py | 79 | ||||
| -rwxr-xr-x | utils/trust_gpg.sh (renamed from utils/gpg_trust) | 0 |
4 files changed, 128 insertions, 7 deletions
diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py index c29ede4..11e79e0 100755 --- a/utils/ng/pkgbuild.py +++ b/utils/ng/pkgbuild.py @@ -5,6 +5,7 @@ import asyncio import dataclasses import lzma import logging +import operator import os import pathlib import shutil @@ -22,7 +23,7 @@ CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "") CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "") CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "") BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "") -TRUST_GPG_SCRIPT = pathlib.Path("./utils/gpg_trust").resolve() +TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve() GNUPG_HOME = os.getenv("HOME") FAIL_REPOS = [] @@ -178,7 +179,8 @@ async def get_build_target(pkg_db: str, home_directory: pathlib.Path, build_targ finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) if len(BUILD_OVERRIDE): - override = BUILD_OVERRIDE.split() + logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE) + override = BUILD_OVERRIDE.split(";") return [future.result() for future in finished if future.result().name in override] if build_all: @@ -226,6 +228,11 @@ async def run_build(src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg return p.returncode +async def install_dep_via_yay(dep: str) -> None: + await (await asyncio.create_subprocess_exec( + "yay", "--noconfirm", "--asdeps", "--needed", "--noprogressbar", "-S", dep, stdout=None)).wait() + + async def do_build(target: PackageVersionWithPath) -> int: try: os.chdir(str(target.path)) @@ -239,32 +246,59 @@ async def do_build(target: PackageVersionWithPath) -> int: if (yay_deps := target.path.parent.joinpath(".yaydeps").joinpath(base_dir_name).resolve()).is_file(): async with aiofiles.open(str(yay_deps)) as fin: - while dep := await fin.readline(): + while dep := (await fin.readline()).strip(): + logger.debug("Find dep %s, build first", dep) if (dep_dir := target.path.parent.joinpath(dep).resolve()).is_dir(): os.chdir(str(dep_dir)) await run_hook(target.path, dep) - await run_build_with_install(**BUILD_ENVS.get_dict()) + if not (ret := await run_build_with_install(**BUILD_ENVS.get_dict())): + logger.error("Build dependencies package error, skipped next step") + return ret + else: + await install_dep_via_yay(dep) if (gpg_keys := target.path.parent.joinpath(".gpg_keys").joinpath(base_dir_name).resolve()).is_file(): await (await asyncio.create_subprocess_exec("/bin/bash", str(gpg_keys), stdout=None)).wait() await (await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None)).wait() + os.chdir(str(target.path)) build_ret = await run_build(**BUILD_ENVS.get_dict()) if build_ret != 0: + if build_ret == 13: + logger.warning("Already build %s, skipped.", base_dir_name) FAIL_REPOS.append(base_dir_name) logger.warning("Build %s fail: %d", base_dir_name, build_ret) return build_ret -async def do_work(pkg_db: str, build_all: bool, build_target: list[str], fail_fast: bool = False) -> None: +async def clean_installed_packages() -> int: + p = await asyncio.create_subprocess_exec("/bin/bash", "-c", "sudo pacman --noconfirm -Rcns $(pacman -Qdtq)", stdout=None) + await p.wait() + return p.returncode + + +async def upload_packages() -> None: + remote_path = os.getenv("REMOTE_PATH") + upload_token = os.getenv("UPLOAD_TOKEN") + if remote_path is None or upload_token is None: + logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload") + await (await asyncio.create_subprocess_exec( + "./utils/ng/upload.py", remote_path, upload_token, ARCH, "--directory", BuildEnvs.pkg_dest, + stdout=None + )).wait() + - logger.info("build_all: %s, fail_fast: %s", build_all, fail_fast) +async def do_work(pkg_db: str, build_all: bool, build_target: list[str], + fail_fast: bool = False, run_auto_remove: bool = True) -> None: + + logger.info("build_all: %s, fail_fast: %s, auto_remove: %s", build_all, fail_fast, run_auto_remove) home_directory = pathlib.Path(os.getcwd()) - build_target = await get_build_target(pkg_db, home_directory, build_target, build_all) + build_target = sorted(await get_build_target(pkg_db, home_directory, build_target, build_all), + key=operator.attrgetter("name")) logger.debug("Pending build summary (%d): %s", len(build_target), ', '.join(map(lambda x: x.name, build_target))) for target in build_target: @@ -273,6 +307,8 @@ async def do_work(pkg_db: str, build_all: bool, build_target: list[str], fail_fa continue if (await do_build(target)) != 0 and fail_fast: break + if run_auto_remove: + await clean_installed_packages() if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir(): shutil.rmtree(str(src_dst), ignore_errors=True) @@ -285,6 +321,10 @@ async def do_work(pkg_db: str, build_all: bool, build_target: list[str], fail_fa for repo in FAIL_REPOS: logger.error("%s", repo) + os.chdir(str(home_directory)) + if ret := await upload_packages(): + exit(ret) + def check_build_target(args: argparse.Namespace) -> bool: if args.build_all or len(args.build_target): diff --git a/utils/ng/pkgbuild_bootstrap b/utils/ng/pkgbuild_bootstrap index 69ff1a6..847b8d5 100755 --- a/utils/ng/pkgbuild_bootstrap +++ b/utils/ng/pkgbuild_bootstrap @@ -48,4 +48,6 @@ fi curl -fL -o remote.db "$REMOTE_DATABASE" +sudo pacman -Sy + ./utils/ng/pkgbuild.py remote.db $FLAGS diff --git a/utils/ng/upload.py b/utils/ng/upload.py new file mode 100755 index 0000000..00b8c02 --- /dev/null +++ b/utils/ng/upload.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +import argparse +import asyncio +import logging +import os + +import aiohttp + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +def build_data(args: argparse.Namespace, action: str) -> dict[str, str]: + return {"token": args.token, "arch": args.arch, "action": action} + + +# source: https://stackoverflow.com/a/1094933 +def sizeof_fmt(num: int, suffix: str = "B"): + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + + +async def send_file(session: aiohttp.ClientSession, args: argparse.Namespace, file: str) -> None: + data = aiohttp.FormData() + data.add_field("file", open(file, "rb"), filename=file) + data.add_field("token", args.token) + data.add_field("arch", args.arch) + async with session.post(args.remote_address, data=data): + pass + + +async def main(args: argparse.Namespace) -> int: + pending_upload = [] + for file in os.listdir("."): + if '.pkg' in file: + pending_upload.append(file) + + if not len(pending_upload): + logger.debug("Could not find any packaged packages, skip upload") + return 0 + + async with aiohttp.ClientSession(raise_for_status=True, timeout=aiohttp.ClientTimeout(60)) as session: + async with session.post(args.remote_address, data=build_data(args, "REQUIRE_CLEAN")): + pass + for file in pending_upload: + logger.info("Start upload %s (%s)", sizeof_fmt(os.stat(file).st_size)) + await send_file(session, args, file) + async with session.post(args.remote_address, data=build_data(args, "UPLOADED")): + pass + + +if __name__ == '__main__': + parser = argparse.ArgumentParser("upload.py") + parser.add_argument("remote_path") + parser.add_argument("token") + parser.add_argument("arch") + parser.add_argument("--directory") + args_ = parser.parse_args() + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + + if args_.directory is not None: + os.chdir(args_.directory) + + exit(asyncio.run(main(args_))) diff --git a/utils/gpg_trust b/utils/trust_gpg.sh index 5d4f8c3..5d4f8c3 100755 --- a/utils/gpg_trust +++ b/utils/trust_gpg.sh |