diff options
| -rwxr-xr-x | utils/gpg_trust | 3 | ||||
| -rwxr-xr-x[-rw-r--r--] | utils/ng/pkgbuild.py | 222 | ||||
| -rwxr-xr-x | utils/ng/pkgbuild_bootstrap | 43 |
3 files changed, 248 insertions, 20 deletions
diff --git a/utils/gpg_trust b/utils/gpg_trust new file mode 100755 index 0000000..5d4f8c3 --- /dev/null +++ b/utils/gpg_trust @@ -0,0 +1,3 @@ +#!/bin/bash +gpg --list-keys --fingerprint | grep pub -A 1 | grep -Ev "pub|--" | tr -d ' ' \ + | awk 'BEGIN { FS = "\n" } ; { print $1":6:" } ' | gpg --import-ownertrust
\ No newline at end of file diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py index ed1a661..ec7aaad 100644..100755 --- a/utils/ng/pkgbuild.py +++ b/utils/ng/pkgbuild.py @@ -8,17 +8,54 @@ import logging import os import pathlib import subprocess +import time import aiofiles logger = logging.getLogger() logger.setLevel(logging.DEBUG) +PKGBUILD_DIRECTORY_BASE = "repo" + +CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "") +CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "") +CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "") +BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "") +TRUST_GPG_SCRIPT = pathlib.Path("./utils/gpg_trust").resolve() +FAIL_REPOS = [] + + +async def get_arch() -> str: + p = await asyncio.create_subprocess_exec("uname", "-m", stdout=subprocess.PIPE) + stdout, _ = p.communicate() + return stdout.decode().lower() + +ARCH = asyncio.run(get_arch()) + + +class PackageVersionWithPath: + _version: PackageVersion + path: pathlib.Path + + @property + def name(self) -> str: + return self._version.name + + @property + def version(self) -> str: + return self._version.version + + @property + def arch(self) -> bool: + return self._version.arch_match + @dataclasses.dataclass class PackageVersion: name: str version: str + arch_match: bool def __eq__(self, other: PackageVersion) -> bool: return other.version == self.version @@ -55,7 +92,7 @@ def fetch_database(pkg_db: str) -> dict[str, str]: name_read = False elif version_read: package_storage.update({pkg_name: line.decode()}) - #logger.debug(pkg_name, line.decode()) + # logger.debug("%s %s", pkg_name, line.decode()) version_read = False pkg_name = '' if line == b"%NAME%": @@ -68,10 +105,37 @@ def fetch_database(pkg_db: str) -> dict[str, str]: return package_storage [email protected](init=False) +class BuildEnvs: + src_dest: str + pkg_dest: str + makepkg_conf: str + signing_arg: list[str] + + def __init__(self): + cwd = pathlib.Path(os.getcwd()) + self.pkg_dest = str(cwd.joinpath("packages", ARCH).resolve()) + self.src_dest = str(cwd.joinpath("build").resolve()) + self.makepkg_conf = str(cwd.joinpath("makepkg_current.conf").resolve()) + if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != '': + self.signing_arg = ["--sign"] + else: + logging.info("Skip signing package") + + def get_dict(self) -> dict[str, str | list[str]]: + return { + "src_dest": self.src_dest, + "pkg_dest": self.src_dest, "makepkg_conf": self.makepkg_conf, "signing_arg": list[str]} + + +BUILD_ENVS = BuildEnvs() + + def get_src_info(s: str) -> PackageVersion: base = '' ver = '' rel = '' + arch = [] for line in s.splitlines(False): if (line := line.strip()).startswith('pkgver'): ver = line.split('=', maxsplit=2)[1] @@ -79,50 +143,168 @@ def get_src_info(s: str) -> PackageVersion: rel = line.split('=', maxsplit=2)[1] elif line.startswith('pkgbase'): base = line.split('=', maxsplit=2)[1] - return PackageVersion(base.strip(), f'{ver.strip()}-{rel.strip()}') + elif line.startswith('arch'): + arch.append(line.split('=', maxsplit=2)[1].strip().lower()) + return PackageVersion(base.strip(), f'{ver.strip()}-{rel.strip()}', ARCH in arch or 'any' in arch) -async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersion: - if (path := pathlib.Path(path.joinpath('.SRCINFO'))).is_file(): - async with aiofiles.open(str(path)) as fin: - return get_src_info(await fin.read()) +async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersionWithPath: + if (src_path := pathlib.Path(path.joinpath('.SRCINFO'))).is_file(): + async with aiofiles.open(str(src_path)) as fin: + info = get_src_info(await fin.read()) else: - p = await asyncio.create_subprocess_exec('makepkg', '--printsrcinfo', stdout=subprocess.PIPE, cwd=path.parent) + p = await asyncio.create_subprocess_exec('makepkg', '--printsrcinfo', cwd=src_path.parent) (stdout, stderr) = (await p.communicate()) - return get_src_info(stdout.decode()) + info = get_src_info(stdout.decode()) + return PackageVersionWithPath(info, pathlib.Path(path).resolve()) -async def do_work(pkg_db: str): +async def get_build_target(pkg_db: str, home_directory: pathlib.Path, build_target: list[str] | None, build_all: bool) -> list[PackageVersionWithPath]: repo = fetch_database(pkg_db) - home_directory = pathlib.Path(os.getcwd()) - tasks = [] pending_build = [] - for folder in os.listdir("."): + repo_directory = home_directory.joinpath(PKGBUILD_DIRECTORY_BASE) + + for folder in os.listdir(PKGBUILD_DIRECTORY_BASE): if folder.startswith("."): continue if os.path.isfile(folder): continue - tasks.append(asyncio.create_task(parse_srcinfo(home_directory.joinpath(folder)))) + tasks.append(asyncio.create_task(parse_srcinfo(repo_directory.joinpath(folder)))) finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + if len(BUILD_OVERRIDE): + override = BUILD_OVERRIDE.split() + return [future.result() for future in finished if future.result().name in override] + + if build_all: + return [future.result() for future in finished] + + if len(build_target): + return [future.result() for future in finished if future.result().name in build_target] + for future in finished: pkg = future.result() if pkg.name in repo: - #print(pkg.name, repo[pkg.name], pkg.version) + # print(pkg.name, repo[pkg.name], pkg.version) if repo[pkg.name] < pkg.version: - pending_build.append(pkg.name) + pending_build.append(pkg) else: - pending_build.append(pkg.name) + pending_build.append(pkg) + + # print(pending_build) + logger.debug("Pending build summary: %s", pending_build) + return pending_build + + +async def run_hook(origin_path: pathlib.Path, target: str) -> None: + if (hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve()).is_file(): + logging.info("Running %s hook script") + p = await asyncio.create_subprocess_exec(str(hook_script), stdout=None, stderr=subprocess.STDOUT) + await p.wait() + if p.returncode != 0: + logging.error("Hook script return code: %d, skip other operation", p.returncode) + return + + +async def run_build(src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str]) -> int: + p = await asyncio.create_subprocess_exec( + "makepkg", "--clean", "-s", "-i", *signing_arg, "--asdeps", "--noconfirm", "--needed", "--noprogressbar", + env={"SRCPKGDEST": src_dest, "SRCDEST": src_dest, "PKGDEST": pkg_dest, "MAKEPKG_CONF": makepkg_conf}, + stdout=None + ) + await p.wait() + return p.returncode + + +async def do_build(target: PackageVersionWithPath) -> None: + try: + os.chdir(str(target.path)) + logging.info("Building %s (%s)", target.name, target.path.stem) + except OSError: + logging.exception("Got exception while chdir to %s", target.path) + + base_dir_name = target.path.stem + + await run_hook(target.path, base_dir_name) + + if (yay_deps := target.path.parent.joinpath(".yaydeps").joinpath(base_dir_name).resolve()).is_file(): + async with aiofiles.open(str(yay_deps)) as fin: + while dep := await fin.readline(): + if (dep_dir := target.path.parent.joinpath(dep).resolve()).is_dir(): + os.chdir(str(dep_dir)) + await run_hook(target.path, dep) + await run_build(**BUILD_ENVS.get_dict()) + + if (gpg_keys := target.path.parent.joinpath(".gpg_keys").joinpath(base_dir_name).resolve()).is_file(): + await (await asyncio.create_subprocess_exec("/bin/bash", str(gpg_keys), stdout=None)).wait() + await (await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None)).wait() + + build_ret = await run_build(**BUILD_ENVS.get_dict()) + + if build_ret != 0: + FAIL_REPOS.append(base_dir_name) + logger.warning("Build %s fail: %d", base_dir_name, build_ret) + + +async def do_work(pkg_db: str, build_all: bool, build_target: list[str]): + + home_directory = pathlib.Path(os.getcwd()) + + build_target = await get_build_target(pkg_db, home_directory, build_target, build_all) + + for target in build_target: + if not target.arch: + logging.info("Skipped %s (Architecture not match)", target.name) + continue + await do_build(target) + + if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir(): + src_dst.rmdir() + + async with aiofiles.open(str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve())) as fout: + await fout.write(f"{int(time.time())}") + + if len(FAIL_REPOS): + logging.error("Build failed repositories:") + for repo in FAIL_REPOS: + logging.error("%s", repo) + + +def check_build_target(args: argparse.Namespace) -> bool: + if args.build_all or len(args.build_target): + return True + if 'REBUILD ALL' in CI_COMMIT_TITLE or 'REBUILD_ALL' in CI_COMMIT_TITLE: + return True + return False - print(pending_build) if __name__ == '__main__': parser = argparse.ArgumentParser(description='pkgbuild.py') - parser.add_argument('db_file') - args = parser.parse_args() - asyncio.run(do_work(args.db_file))
\ No newline at end of file + parser.add_argument('db_file', required=True) + parser.add_argument('build_all', action='store_true') + parser.add_argument('build_target', nargs='*') + parser.add_argument('sign', action='store_true') + args_ = parser.parse_args() + if args_.sign: + BUILD_ENVS.pkg_dest = ['--sign'] + + + try: + import coloredlogs + + coloredlogs.install( + logging.DEBUG, + fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + except ModuleNotFoundError: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", + ) + + asyncio.run(do_work(args_.db_file, check_build_target(args_), args_.build_target)) diff --git a/utils/ng/pkgbuild_bootstrap b/utils/ng/pkgbuild_bootstrap new file mode 100755 index 0000000..a510c53 --- /dev/null +++ b/utils/ng/pkgbuild_bootstrap @@ -0,0 +1,43 @@ +#!/bin/bash + +set -Eeo pipefail +trap cleanup SIGINT SIGTERM ERR EXIT + +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + unset SIGNING_ARG + unset UNSUCCESSFUL + unset SKIP_VERIFIED + unset FOLDER_NAME + unset PKGDEST + unset SRCDEST + unset CONFDEST + unset REPO_DEST + unset TMPCONF + unset REPO_PENDING + unset REPO_DIFF + unset PKGBUILD_DIRECTORY_BASE + unset ARCH +} + +ARCH=$(uname -m) +CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf" + +TMPCONF="makepkg_current.conf" +cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF" +cat "$CONFDEST" >> "$TMPCONF" + + +if [ "$EUID" -eq 0 ] ; then + echo "\033[0;32mPlease run this script as non-root\033[0m" + exit 3 +fi + +if [ -z "$1" ]; then + echo "\033[0;32mYou should specify remote address\033[0m" + exit 4 +fi + +curl -fL -o remote.db "$1" + +./uitls/ng/pkgbuild.py remote.db |