diff options
Diffstat (limited to 'utils/ng/pkgbuild.py')
| -rwxr-xr-x | utils/ng/pkgbuild.py | 236 |
1 files changed, 174 insertions, 62 deletions
diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py index 11e79e0..ff52dc9 100755 --- a/utils/ng/pkgbuild.py +++ b/utils/ng/pkgbuild.py @@ -33,6 +33,7 @@ async def get_arch() -> str: stdout, _ = await (p.communicate()) return stdout.decode().lower().strip() + ARCH = asyncio.run(get_arch()) @@ -79,13 +80,13 @@ class PackageVersion: return not self == other def __str__(self) -> str: - return f'{self.name} {self.version}' + return f"{self.name} {self.version}" def fetch_database(pkg_db: str) -> dict[str, str]: name_read = False version_read = False - pkg_name = '' + pkg_name = "" package_storage = {} with lzma.open(pkg_db) as fin: for line in fin: @@ -97,14 +98,16 @@ def fetch_database(pkg_db: str) -> dict[str, str]: package_storage.update({pkg_name: line.decode()}) # logger.debug("%s %s", pkg_name, line.decode()) version_read = False - pkg_name = '' + pkg_name = "" if line == b"%NAME%": assert not name_read name_read = True if line == b"%VERSION%": assert not version_read version_read = True - assert (name_read is False and version_read is False) or (version_read != name_read) + assert (name_read is False and version_read is False) or ( + version_read != name_read + ) return package_storage @@ -120,7 +123,7 @@ class BuildEnvs: self.pkg_dest = str(cwd.joinpath("packages", ARCH).resolve()) self.src_dest = str(cwd.joinpath("build").resolve()) self.makepkg_conf = str(cwd.joinpath("makepkg_current.conf").resolve()) - if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != '': + if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != "": self.signing_arg = ["--sign"] else: logger.info("Skip signing package") @@ -128,38 +131,50 @@ class BuildEnvs: def get_dict(self) -> dict[str, str | list[str]]: return { "src_dest": self.src_dest, - "pkg_dest": self.pkg_dest, "makepkg_conf": self.makepkg_conf, "signing_arg": self.signing_arg} + "pkg_dest": self.pkg_dest, + "makepkg_conf": self.makepkg_conf, + "signing_arg": self.signing_arg, + } def get_src_info(s: str) -> PackageVersion: - base = '' - ver = '' - rel = '' + base = "" + ver = "" + rel = "" arch = [] for line in s.splitlines(False): - if (line := line.strip()).startswith('pkgver'): - ver = line.split('=', maxsplit=2)[1] - elif line.startswith('pkgrel'): - rel = line.split('=', maxsplit=2)[1] - elif line.startswith('pkgbase'): - base = line.split('=', maxsplit=2)[1] - elif line.startswith('arch'): - arch.append(line.split('=', maxsplit=2)[1].strip().lower()) - return PackageVersion(base.strip(), f'{ver.strip()}-{rel.strip()}', ARCH in arch or 'any' in arch) + if (line := line.strip()).startswith("pkgver"): + ver = line.split("=", maxsplit=2)[1] + elif line.startswith("pkgrel"): + rel = line.split("=", maxsplit=2)[1] + elif line.startswith("pkgbase"): + base = line.split("=", maxsplit=2)[1] + elif line.startswith("arch"): + arch.append(line.split("=", maxsplit=2)[1].strip().lower()) + return PackageVersion( + base.strip(), f"{ver.strip()}-{rel.strip()}", ARCH in arch or "any" in arch + ) async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersionWithPath: - if (src_path := pathlib.Path(path.joinpath('.SRCINFO'))).is_file(): + if (src_path := pathlib.Path(path.joinpath(".SRCINFO"))).is_file(): async with aiofiles.open(str(src_path)) as fin: info = get_src_info(await fin.read()) else: - p = await asyncio.create_subprocess_exec('makepkg', '--printsrcinfo', cwd=src_path.parent, stdout=subprocess.PIPE) - (stdout, stderr) = (await p.communicate()) + p = await asyncio.create_subprocess_exec( + "makepkg", "--printsrcinfo", cwd=src_path.parent, stdout=subprocess.PIPE + ) + (stdout, stderr) = await p.communicate() info = get_src_info(stdout.decode()) return PackageVersionWithPath(info, pathlib.Path(path).resolve()) -async def get_build_target(pkg_db: str, home_directory: pathlib.Path, build_target: list[str] | None, build_all: bool) -> list[PackageVersionWithPath]: +async def get_build_target( + pkg_db: str, + home_directory: pathlib.Path, + build_target: list[str] | None, + build_all: bool, +) -> list[PackageVersionWithPath]: repo = fetch_database(pkg_db) tasks = [] @@ -181,13 +196,19 @@ async def get_build_target(pkg_db: str, home_directory: pathlib.Path, build_targ if len(BUILD_OVERRIDE): logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE) override = BUILD_OVERRIDE.split(";") - return [future.result() for future in finished if future.result().name in override] + return [ + future.result() for future in finished if future.result().name in override + ] if build_all: return [future.result() for future in finished] if len(build_target): - return [future.result() for future in finished if future.result().name in build_target] + return [ + future.result() + for future in finished + if future.result().name in build_target + ] for future in finished: pkg = future.result() @@ -203,34 +224,73 @@ async def get_build_target(pkg_db: str, home_directory: pathlib.Path, build_targ async def run_hook(origin_path: pathlib.Path, target: str) -> None: - if (hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve()).is_file(): + if ( + hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve() + ).is_file(): logger.info("Running %s hook script", target) - p = await asyncio.create_subprocess_exec(str(hook_script), stdout=None, stderr=subprocess.STDOUT) + p = await asyncio.create_subprocess_exec( + str(hook_script), stdout=None, stderr=subprocess.STDOUT + ) await p.wait() if p.returncode != 0: - logger.error("Hook script return code: %d, skip other operation", p.returncode) + logger.error( + "Hook script return code: %d, skip other operation", p.returncode + ) return -async def run_build_with_install(src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str]) -> int: +async def run_build_with_install( + src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str] +) -> int: return await run_build(src_dest, pkg_dest, makepkg_conf, signing_arg, "-i") -async def run_build(src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str], *other_args: str) -> int: +async def run_build( + src_dest: str, + pkg_dest: str, + makepkg_conf: str, + signing_arg: list[str], + *other_args: str, +) -> int: env = os.environ - env.update({"SRCPKGDEST": src_dest, "SRCDEST": src_dest, "PKGDEST": pkg_dest, "MAKEPKG_CONF": makepkg_conf}) + env.update( + { + "SRCPKGDEST": src_dest, + "SRCDEST": src_dest, + "PKGDEST": pkg_dest, + "MAKEPKG_CONF": makepkg_conf, + } + ) p = await asyncio.create_subprocess_exec( - "makepkg", "--clean", "-s", *other_args, *signing_arg, "--asdeps", "--noconfirm", "--needed", "--noprogressbar", + "makepkg", + "--clean", + "-s", + *other_args, + *signing_arg, + "--asdeps", + "--noconfirm", + "--needed", + "--noprogressbar", env=env, - stdout=None + stdout=None, ) await p.wait() return p.returncode async def install_dep_via_yay(dep: str) -> None: - await (await asyncio.create_subprocess_exec( - "yay", "--noconfirm", "--asdeps", "--needed", "--noprogressbar", "-S", dep, stdout=None)).wait() + await ( + await asyncio.create_subprocess_exec( + "yay", + "--noconfirm", + "--asdeps", + "--needed", + "--noprogressbar", + "-S", + dep, + stdout=None, + ) + ).wait() async def do_build(target: PackageVersionWithPath) -> int: @@ -244,22 +304,40 @@ async def do_build(target: PackageVersionWithPath) -> int: await run_hook(target.path, base_dir_name) - if (yay_deps := target.path.parent.joinpath(".yaydeps").joinpath(base_dir_name).resolve()).is_file(): + if ( + yay_deps := target.path.parent.joinpath(".yaydeps") + .joinpath(base_dir_name) + .resolve() + ).is_file(): async with aiofiles.open(str(yay_deps)) as fin: while dep := (await fin.readline()).strip(): logger.debug("Find dep %s, build first", dep) if (dep_dir := target.path.parent.joinpath(dep).resolve()).is_dir(): os.chdir(str(dep_dir)) await run_hook(target.path, dep) - if not (ret := await run_build_with_install(**BUILD_ENVS.get_dict())): - logger.error("Build dependencies package error, skipped next step") + if not ( + ret := await run_build_with_install(**BUILD_ENVS.get_dict()) + ): + logger.error( + "Build dependencies package error, skipped next step" + ) return ret else: await install_dep_via_yay(dep) - if (gpg_keys := target.path.parent.joinpath(".gpg_keys").joinpath(base_dir_name).resolve()).is_file(): - await (await asyncio.create_subprocess_exec("/bin/bash", str(gpg_keys), stdout=None)).wait() - await (await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None)).wait() + if ( + gpg_keys := target.path.parent.joinpath(".gpg_keys") + .joinpath(base_dir_name) + .resolve() + ).is_file(): + await ( + await asyncio.create_subprocess_exec( + "/bin/bash", str(gpg_keys), stdout=None + ) + ).wait() + await ( + await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None) + ).wait() os.chdir(str(target.path)) build_ret = await run_build(**BUILD_ENVS.get_dict()) @@ -274,7 +352,9 @@ async def do_build(target: PackageVersionWithPath) -> int: async def clean_installed_packages() -> int: - p = await asyncio.create_subprocess_exec("/bin/bash", "-c", "sudo pacman --noconfirm -Rcns $(pacman -Qdtq)", stdout=None) + p = await asyncio.create_subprocess_exec( + "/bin/bash", "-c", "sudo pacman --noconfirm -Rcns $(pacman -Qdtq)", stdout=None + ) await p.wait() return p.returncode @@ -284,22 +364,45 @@ async def upload_packages() -> None: upload_token = os.getenv("UPLOAD_TOKEN") if remote_path is None or upload_token is None: logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload") - await (await asyncio.create_subprocess_exec( - "./utils/ng/upload.py", remote_path, upload_token, ARCH, "--directory", BuildEnvs.pkg_dest, - stdout=None - )).wait() + await ( + await asyncio.create_subprocess_exec( + "./utils/ng/upload.py", + remote_path, + upload_token, + ARCH, + "--directory", + BuildEnvs.pkg_dest, + stdout=None, + ) + ).wait() -async def do_work(pkg_db: str, build_all: bool, build_target: list[str], - fail_fast: bool = False, run_auto_remove: bool = True) -> None: +async def do_work( + pkg_db: str, + build_all: bool, + build_target: list[str], + fail_fast: bool = False, + run_auto_remove: bool = True, +) -> None: - logger.info("build_all: %s, fail_fast: %s, auto_remove: %s", build_all, fail_fast, run_auto_remove) + logger.info( + "build_all: %s, fail_fast: %s, auto_remove: %s", + build_all, + fail_fast, + run_auto_remove, + ) home_directory = pathlib.Path(os.getcwd()) - build_target = sorted(await get_build_target(pkg_db, home_directory, build_target, build_all), - key=operator.attrgetter("name")) - logger.debug("Pending build summary (%d): %s", len(build_target), ', '.join(map(lambda x: x.name, build_target))) + build_target = sorted( + await get_build_target(pkg_db, home_directory, build_target, build_all), + key=operator.attrgetter("name"), + ) + logger.debug( + "Pending build summary (%d): %s", + len(build_target), + ", ".join(map(lambda x: x.name, build_target)), + ) for target in build_target: if not target.arch: @@ -313,7 +416,9 @@ async def do_work(pkg_db: str, build_all: bool, build_target: list[str], if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir(): shutil.rmtree(str(src_dst), ignore_errors=True) - async with aiofiles.open(str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve()), "w") as fout: + async with aiofiles.open( + str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve()), "w" + ) as fout: await fout.write(f"{int(time.time())}") if len(FAIL_REPOS): @@ -329,18 +434,18 @@ async def do_work(pkg_db: str, build_all: bool, build_target: list[str], def check_build_target(args: argparse.Namespace) -> bool: if args.build_all or len(args.build_target): return True - if 'REBUILD ALL' in CI_COMMIT_TITLE or 'REBUILD_ALL' in CI_COMMIT_TITLE: + if "REBUILD ALL" in CI_COMMIT_TITLE or "REBUILD_ALL" in CI_COMMIT_TITLE: return True return False -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='pkgbuild.py') - parser.add_argument('db_file') - parser.add_argument('--build_all', action='store_true') - parser.add_argument('--build_target', nargs='*') - parser.add_argument('--sign', action='store_true') - parser.add_argument('--fail_fast', action='store_true') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="pkgbuild.py") + parser.add_argument("db_file") + parser.add_argument("--build_all", action="store_true") + parser.add_argument("--build_target", nargs="*") + parser.add_argument("--sign", action="store_true") + parser.add_argument("--fail_fast", action="store_true") args_ = parser.parse_args() try: @@ -358,5 +463,12 @@ if __name__ == '__main__': BUILD_ENVS = BuildEnvs() if args_.sign: - BUILD_ENVS.signing_arg = ['--sign'] - asyncio.run(do_work(args_.db_file, check_build_target(args_), args_.build_target, args_.fail_fast)) + BUILD_ENVS.signing_arg = ["--sign"] + asyncio.run( + do_work( + args_.db_file, + check_build_target(args_), + args_.build_target, + args_.fail_fast, + ) + ) |