From 685c74c4fa212b132c55be1c91e5b366a3593b83 Mon Sep 17 00:00:00 2001 From: KunoiSayami Date: Wed, 26 Oct 2022 00:05:46 +0800 Subject: refactor: Move script out from ng folder Signed-off-by: KunoiSayami --- utils/ng/pkgbuild.py | 485 --------------------------------------------------- 1 file changed, 485 deletions(-) delete mode 100755 utils/ng/pkgbuild.py (limited to 'utils/ng/pkgbuild.py') diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py deleted file mode 100755 index e3f46f1..0000000 --- a/utils/ng/pkgbuild.py +++ /dev/null @@ -1,485 +0,0 @@ -#!/usr/bin/env python -from __future__ import annotations -import argparse -import asyncio -import dataclasses -import lzma -import logging -import operator -import os -import pathlib -import shutil -import subprocess -import time - -import aiofiles - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -PKGBUILD_DIRECTORY_BASE = "repo" - -CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "") -CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "") -CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "") -BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "") -TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve() -GNUPG_HOME = os.getenv("HOME") -FAIL_REPOS = {} - - -async def get_arch() -> str: - p = await asyncio.create_subprocess_exec("uname", "-m", stdout=subprocess.PIPE) - stdout, _ = await (p.communicate()) - return stdout.decode().lower().strip() - - -ARCH = asyncio.run(get_arch()) - - -@dataclasses.dataclass -class PackageVersionWithPath: - _version: PackageVersion - path: pathlib.Path - - @property - def name(self) -> str: - return self._version.name - - @property - def version(self) -> str: - return self._version.version - - @property - def arch(self) -> bool: - return self._version.arch_match - - -@dataclasses.dataclass -class PackageVersion: - name: str - version: str - arch_match: bool - - def __eq__(self, other: PackageVersion) -> bool: - return other.version == self.version - - def __gt__(self, other: PackageVersion) -> bool: - return self.version > other.version - - def __lt__(self, other: PackageVersion) -> bool: - return self.version < other.version - - def __le__(self, other: PackageVersion) -> bool: - return self < other or self == other - - def __ge__(self, other: PackageVersion) -> bool: - return self > other or self == other - - def __ne__(self, other: PackageVersion) -> bool: - return not self == other - - def __str__(self) -> str: - return f"{self.name} {self.version}" - - -def fetch_database(pkg_db: str) -> dict[str, str]: - name_read = False - version_read = False - pkg_name = "" - package_storage = {} - with lzma.open(pkg_db) as fin: - for line in fin: - line = line.strip() - if name_read: - pkg_name = line.decode() - name_read = False - elif version_read: - package_storage.update({pkg_name: line.decode()}) - # logger.debug("%s %s", pkg_name, line.decode()) - version_read = False - pkg_name = "" - if line == b"%NAME%": - assert not name_read - name_read = True - if line == b"%VERSION%": - assert not version_read - version_read = True - assert (name_read is False and version_read is False) or ( - version_read != name_read - ) - return package_storage - - -@dataclasses.dataclass(init=False) -class BuildEnvs: - src_dest: str - pkg_dest: str - makepkg_conf: str - signing_arg: list[str] - - def __init__(self): - cwd = pathlib.Path(os.getcwd()) - self.pkg_dest = str(cwd.joinpath("packages", ARCH).resolve()) - self.src_dest = str(cwd.joinpath("build").resolve()) - self.makepkg_conf = str(cwd.joinpath("makepkg_current.conf").resolve()) - if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != "": - self.signing_arg = ["--sign"] - else: - logger.info("Skip signing package") - - def get_dict(self) -> dict[str, str | list[str]]: - return { - "src_dest": self.src_dest, - "pkg_dest": self.pkg_dest, - "makepkg_conf": self.makepkg_conf, - "signing_arg": self.signing_arg, - } - - -def get_src_info(s: str) -> PackageVersion: - base = "" - ver = "" - rel = "" - arch = [] - for line in s.splitlines(False): - if (line := line.strip()).startswith("pkgver"): - ver = line.split("=", maxsplit=2)[1] - elif line.startswith("pkgrel"): - rel = line.split("=", maxsplit=2)[1] - elif line.startswith("pkgbase"): - base = line.split("=", maxsplit=2)[1] - elif line.startswith("arch"): - arch.append(line.split("=", maxsplit=2)[1].strip().lower()) - return PackageVersion( - base.strip(), f"{ver.strip()}-{rel.strip()}", ARCH in arch or "any" in arch - ) - - -async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersionWithPath: - if (src_path := pathlib.Path(path.joinpath(".SRCINFO"))).is_file(): - async with aiofiles.open(str(src_path)) as fin: - info = get_src_info(await fin.read()) - else: - p = await asyncio.create_subprocess_exec( - "makepkg", "--printsrcinfo", cwd=src_path.parent, stdout=subprocess.PIPE - ) - (stdout, stderr) = await p.communicate() - info = get_src_info(stdout.decode()) - return PackageVersionWithPath(info, pathlib.Path(path).resolve()) - - -async def get_build_target( - pkg_db: str, - home_directory: pathlib.Path, - build_target: list[str] | None, - build_all: bool, -) -> list[PackageVersionWithPath]: - - repo = fetch_database(pkg_db) - tasks = [] - pending_build = [] - - repo_directory = home_directory.joinpath(PKGBUILD_DIRECTORY_BASE) - - for folder in os.listdir(PKGBUILD_DIRECTORY_BASE): - if folder.startswith("."): - continue - - if (pkg_dir := repo_directory.joinpath(folder).resolve()).is_file(): - continue - - tasks.append(asyncio.create_task(parse_srcinfo(pkg_dir))) - - finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) - - if len(BUILD_OVERRIDE): - logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE) - override = BUILD_OVERRIDE.split(";") - return [ - future.result() for future in finished if future.result().name in override - ] - - if build_all: - return [future.result() for future in finished] - - if len(build_target): - return [ - future.result() - for future in finished - if future.result().name in build_target - ] - - for future in finished: - pkg = future.result() - if pkg.name in repo: - # print(pkg.name, repo[pkg.name], pkg.version) - if repo[pkg.name] < pkg.version: - pending_build.append(pkg) - else: - pending_build.append(pkg) - - # print(pending_build) - return pending_build - - -async def run_hook(origin_path: pathlib.Path, target: str) -> None: - if ( - hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve() - ).is_file(): - logger.info("Running %s hook script", target) - p = await asyncio.create_subprocess_exec( - str(hook_script), stdout=None, stderr=subprocess.STDOUT - ) - await p.wait() - if p.returncode != 0: - logger.error( - "Hook script return code: %d, skip other operation", p.returncode - ) - return - - -async def run_build_with_install( - src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str] -) -> int: - return await run_build(src_dest, pkg_dest, makepkg_conf, signing_arg, "-i") - - -async def run_build( - src_dest: str, - pkg_dest: str, - makepkg_conf: str, - signing_arg: list[str], - *other_args: str, -) -> int: - env = os.environ - env.update( - { - "SRCPKGDEST": src_dest, - "SRCDEST": src_dest, - "PKGDEST": pkg_dest, - "MAKEPKG_CONF": makepkg_conf, - } - ) - p = await asyncio.create_subprocess_exec( - "makepkg", - "--clean", - "-s", - *other_args, - *signing_arg, - "--asdeps", - "--noconfirm", - "--needed", - "--noprogressbar", - env=env, - stdout=None, - ) - await p.wait() - return p.returncode - - -async def install_dependency_via_yay(dependency: str) -> None: - await ( - await asyncio.create_subprocess_exec( - "yay", - "--noconfirm", - "--asdeps", - "--needed", - "--noprogressbar", - "-S", - dependency, - stdout=None, - ) - ).wait() - - -async def do_build(target: PackageVersionWithPath) -> int: - try: - os.chdir(str(target.path)) - logger.info("Building %s (%s)", target.name, target.path.name) - except OSError: - logger.exception("Got exception while chdir to %s", target.path) - - base_dir_name = target.path.name - - await run_hook(target.path, base_dir_name) - - if ( - yay_deps := target.path.parent.joinpath(".yaydeps") - .joinpath(base_dir_name) - .resolve() - ).is_file(): - async with aiofiles.open(str(yay_deps)) as fin: - while dep := (await fin.readline()).strip(): - logger.debug("Find dep %s, build first", dep) - if (dep_dir := target.path.parent.joinpath(dep).resolve()).is_dir(): - os.chdir(str(dep_dir)) - await run_hook(target.path, dep) - if not ( - ret := await run_build_with_install(**BUILD_ENVS.get_dict()) - ): - logger.error( - "Build dependencies package error, skipped next step" - ) - return ret - else: - await install_dependency_via_yay(dep) - - if ( - gpg_keys := target.path.parent.joinpath(".gpg_keys") - .joinpath(base_dir_name) - .resolve() - ).is_file(): - await ( - await asyncio.create_subprocess_exec( - "/bin/bash", str(gpg_keys), stdout=None - ) - ).wait() - await ( - await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None) - ).wait() - - os.chdir(str(target.path)) - build_ret = await run_build(**BUILD_ENVS.get_dict()) - - if build_ret != 0: - if build_ret == 13: - logger.warning("Already build %s, skipped.", base_dir_name) - return 0 - FAIL_REPOS.update({base_dir_name: build_ret}) - logger.warning("Build %s fail: %d", base_dir_name, build_ret) - - return build_ret - - -async def clean_installed_packages() -> int: - p = await asyncio.create_subprocess_exec( - "/bin/bash", - "-c", - "pacman -Qdtq | ifne sudo pacman --noconfirm -Rcns", - stdout=None, - ) - await p.wait() - return p.returncode - - -async def upload_packages() -> None: - remote_path = os.getenv("REMOTE_PATH") - upload_token = os.getenv("UPLOAD_TOKEN") - if remote_path is None or upload_token is None: - logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload") - await ( - await asyncio.create_subprocess_exec( - "./utils/ng/upload.py", - remote_path, - upload_token, - ARCH, - "--directory", - BUILD_ENVS.pkg_dest, - stdout=None, - ) - ).wait() - - -async def do_work( - pkg_db: str, - build_all: bool, - build_target: list[str], - fail_fast: bool = False, - run_auto_remove: bool = True, -) -> None: - - logger.info( - "build_all: %s, fail_fast: %s, auto_remove: %s", - build_all, - fail_fast, - run_auto_remove, - ) - - home_directory = pathlib.Path(os.getcwd()) - - build_target = sorted( - await get_build_target(pkg_db, home_directory, build_target, build_all), - key=operator.attrgetter("name"), - ) - logger.debug( - "Pending build summary (%d): %s", - len(build_target), - ", ".join(map(lambda x: x.name, build_target)), - ) - - for target in build_target: - if not target.arch: - logger.info("Skipped %s (Architecture not match)", target.name) - continue - if (await do_build(target)) != 0: - if fail_fast: - break - else: - await asyncio.sleep(5) - if run_auto_remove: - await clean_installed_packages() - for _ in range(3): - logger.debug("---------------------------------------------------------") - - if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir(): - shutil.rmtree(str(src_dst), ignore_errors=True) - - async with aiofiles.open( - str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve()), "w" - ) as fout: - await fout.write(f"{int(time.time())}") - - if len(FAIL_REPOS): - logger.error("Build failed repositories (%d):", len(FAIL_REPOS)) - logger.error( - "%s", - ", ".join(f"{repo}({ret_code})" for repo, ret_code in FAIL_REPOS.items()), - ) - - os.chdir(str(home_directory)) - if ret := await upload_packages(): - exit(ret) - - -def check_build_target(args: argparse.Namespace) -> bool: - if args.build_all or len(args.build_target): - return True - if "REBUILD ALL" in CI_COMMIT_TITLE or "REBUILD_ALL" in CI_COMMIT_TITLE: - return True - return False - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="pkgbuild.py") - parser.add_argument("db_file") - parser.add_argument("--build_all", action="store_true") - parser.add_argument("--build_target", nargs="*") - parser.add_argument("--sign", action="store_true") - parser.add_argument("--fail_fast", action="store_true") - args_ = parser.parse_args() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - - BUILD_ENVS = BuildEnvs() - if args_.sign: - BUILD_ENVS.signing_arg = ["--sign"] - asyncio.run( - do_work( - args_.db_file, - check_build_target(args_), - args_.build_target, - args_.fail_fast, - ) - ) -- cgit v1.3.1