feat(script): Implement all function

Signed-off-by: KunoiSayami <[email protected]>

4 files changed, 128 insertions, 7 deletions

diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py
index c29ede4..11e79e0 100755
--- a/utils/ng/pkgbuild.py
+++ b/utils/ng/pkgbuild.py
@@ -5,6 +5,7 @@ import asyncio
 import dataclasses
 import lzma
 import logging
+import operator
 import os
 import pathlib
 import shutil
@@ -22,7 +23,7 @@ CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "")
 CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "")
 CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "")
 BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "")
-TRUST_GPG_SCRIPT = pathlib.Path("./utils/gpg_trust").resolve()
+TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve()
 GNUPG_HOME = os.getenv("HOME")
 FAIL_REPOS = []
 
@@ -178,7 +179,8 @@ async def get_build_target(pkg_db: str, home_directory: pathlib.Path, build_targ
     finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
 
     if len(BUILD_OVERRIDE):
-        override = BUILD_OVERRIDE.split()
+        logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE)
+        override = BUILD_OVERRIDE.split(";")
         return [future.result() for future in finished if future.result().name in override]
 
     if build_all:
@@ -226,6 +228,11 @@ async def run_build(src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg
     return p.returncode
 
 
+async def install_dep_via_yay(dep: str) -> None:
+    await (await asyncio.create_subprocess_exec(
+        "yay", "--noconfirm", "--asdeps", "--needed", "--noprogressbar", "-S", dep, stdout=None)).wait()
+
+
 async def do_build(target: PackageVersionWithPath) -> int:
     try:
         os.chdir(str(target.path))
@@ -239,32 +246,59 @@ async def do_build(target: PackageVersionWithPath) -> int:
 
     if (yay_deps := target.path.parent.joinpath(".yaydeps").joinpath(base_dir_name).resolve()).is_file():
         async with aiofiles.open(str(yay_deps)) as fin:
-            while dep := await fin.readline():
+            while dep := (await fin.readline()).strip():
+                logger.debug("Find dep %s, build first", dep)
                 if (dep_dir := target.path.parent.joinpath(dep).resolve()).is_dir():
                     os.chdir(str(dep_dir))
                     await run_hook(target.path, dep)
-                    await run_build_with_install(**BUILD_ENVS.get_dict())
+                    if not (ret := await run_build_with_install(**BUILD_ENVS.get_dict())):
+                        logger.error("Build dependencies package error, skipped next step")
+                        return ret
+                else:
+                    await install_dep_via_yay(dep)
 
     if (gpg_keys := target.path.parent.joinpath(".gpg_keys").joinpath(base_dir_name).resolve()).is_file():
         await (await asyncio.create_subprocess_exec("/bin/bash", str(gpg_keys), stdout=None)).wait()
         await (await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None)).wait()
 
+    os.chdir(str(target.path))
     build_ret = await run_build(**BUILD_ENVS.get_dict())
 
     if build_ret != 0:
+        if build_ret == 13:
+            logger.warning("Already build %s, skipped.", base_dir_name)
         FAIL_REPOS.append(base_dir_name)
         logger.warning("Build %s fail: %d", base_dir_name, build_ret)
 
     return build_ret
 
 
-async def do_work(pkg_db: str, build_all: bool, build_target: list[str], fail_fast: bool = False) -> None:
+async def clean_installed_packages() -> int:
+    p = await asyncio.create_subprocess_exec("/bin/bash", "-c", "sudo pacman --noconfirm -Rcns $(pacman -Qdtq)", stdout=None)
+    await p.wait()
+    return p.returncode
+
+
+async def upload_packages() -> None:
+    remote_path = os.getenv("REMOTE_PATH")
+    upload_token = os.getenv("UPLOAD_TOKEN")
+    if remote_path is None or upload_token is None:
+        logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload")
+    await (await asyncio.create_subprocess_exec(
+        "./utils/ng/upload.py", remote_path, upload_token, ARCH, "--directory", BuildEnvs.pkg_dest,
+        stdout=None
+    )).wait()
+
 
-    logger.info("build_all: %s, fail_fast: %s", build_all, fail_fast)
+async def do_work(pkg_db: str, build_all: bool, build_target: list[str],
+                  fail_fast: bool = False, run_auto_remove: bool = True) -> None:
+
+    logger.info("build_all: %s, fail_fast: %s, auto_remove: %s", build_all, fail_fast, run_auto_remove)
 
     home_directory = pathlib.Path(os.getcwd())
 
-    build_target = await get_build_target(pkg_db, home_directory, build_target, build_all)
+    build_target = sorted(await get_build_target(pkg_db, home_directory, build_target, build_all),
+                          key=operator.attrgetter("name"))
     logger.debug("Pending build summary (%d): %s", len(build_target), ', '.join(map(lambda x: x.name, build_target)))
 
     for target in build_target:
@@ -273,6 +307,8 @@ async def do_work(pkg_db: str, build_all: bool, build_target: list[str], fail_fa
             continue
         if (await do_build(target)) != 0 and fail_fast:
             break
+        if run_auto_remove:
+            await clean_installed_packages()
 
     if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir():
         shutil.rmtree(str(src_dst), ignore_errors=True)
@@ -285,6 +321,10 @@ async def do_work(pkg_db: str, build_all: bool, build_target: list[str], fail_fa
         for repo in FAIL_REPOS:
             logger.error("%s", repo)
 
+    os.chdir(str(home_directory))
+    if ret := await upload_packages():
+        exit(ret)
+
 
 def check_build_target(args: argparse.Namespace) -> bool:
     if args.build_all or len(args.build_target):
diff --git a/utils/ng/pkgbuild_bootstrap b/utils/ng/pkgbuild_bootstrap
index 69ff1a6..847b8d5 100755
--- a/utils/ng/pkgbuild_bootstrap
+++ b/utils/ng/pkgbuild_bootstrap
@@ -48,4 +48,6 @@ fi
 
 curl -fL -o remote.db "$REMOTE_DATABASE"
 
+sudo pacman -Sy
+
 ./utils/ng/pkgbuild.py remote.db $FLAGS
diff --git a/utils/ng/upload.py b/utils/ng/upload.py
new file mode 100755
index 0000000..00b8c02
--- /dev/null
+++ b/utils/ng/upload.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+import argparse
+import asyncio
+import logging
+import os
+
+import aiohttp
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+
+def build_data(args: argparse.Namespace, action: str) -> dict[str, str]:
+    return {"token": args.token, "arch": args.arch, "action": action}
+
+
+# source: https://stackoverflow.com/a/1094933
+def sizeof_fmt(num: int, suffix: str = "B"):
+    for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
+        if abs(num) < 1024.0:
+            return f"{num:3.1f}{unit}{suffix}"
+        num /= 1024.0
+    return f"{num:.1f}Yi{suffix}"
+
+
+async def send_file(session: aiohttp.ClientSession, args: argparse.Namespace, file: str) -> None:
+    data = aiohttp.FormData()
+    data.add_field("file", open(file, "rb"), filename=file)
+    data.add_field("token", args.token)
+    data.add_field("arch", args.arch)
+    async with session.post(args.remote_address, data=data):
+        pass
+
+
+async def main(args: argparse.Namespace) -> int:
+    pending_upload = []
+    for file in os.listdir("."):
+        if '.pkg' in file:
+            pending_upload.append(file)
+
+    if not len(pending_upload):
+        logger.debug("Could not find any packaged packages, skip upload")
+        return 0
+
+    async with aiohttp.ClientSession(raise_for_status=True, timeout=aiohttp.ClientTimeout(60)) as session:
+        async with session.post(args.remote_address, data=build_data(args, "REQUIRE_CLEAN")):
+            pass
+        for file in pending_upload:
+            logger.info("Start upload %s (%s)", sizeof_fmt(os.stat(file).st_size))
+            await send_file(session, args, file)
+        async with session.post(args.remote_address, data=build_data(args, "UPLOADED")):
+            pass
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser("upload.py")
+    parser.add_argument("remote_path")
+    parser.add_argument("token")
+    parser.add_argument("arch")
+    parser.add_argument("--directory")
+    args_ = parser.parse_args()
+
+    try:
+        import coloredlogs
+
+        coloredlogs.install(
+            logging.DEBUG,
+            fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
+        )
+    except ModuleNotFoundError:
+        logging.basicConfig(
+            level=logging.DEBUG,
+            format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
+        )
+
+    if args_.directory is not None:
+        os.chdir(args_.directory)
+
+    exit(asyncio.run(main(args_)))
diff --git a/utils/gpg_trust b/utils/trust_gpg.sh
index 5d4f8c3..5d4f8c3 100755
--- a/utils/gpg_trust
+++ b/utils/trust_gpg.sh