feat(script): Add upload rated scripts

Signed-off-by: KunoiSayami <[email protected]>

5 files changed, 237 insertions, 11 deletions

diff --git a/utils/ng/pkgbuild.py b/utils/ng/pkgbuild.py
index 8be755a..0bef847 100755
--- a/utils/ng/pkgbuild.py
+++ b/utils/ng/pkgbuild.py
@@ -25,7 +25,7 @@ CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "")
 BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "")
 TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve()
 GNUPG_HOME = os.getenv("HOME")
-FAIL_REPOS = []
+FAIL_REPOS = {}
 
 
 async def get_arch() -> str:
@@ -278,7 +278,7 @@ async def run_build(
     return p.returncode
 
 
-async def install_dependency_via_yay(dep: str) -> None:
+async def install_dependency_via_yay(dependency: str) -> None:
     await (
         await asyncio.create_subprocess_exec(
             "yay",
@@ -287,7 +287,7 @@ async def install_dependency_via_yay(dep: str) -> None:
             "--needed",
             "--noprogressbar",
             "-S",
-            dep,
+            dependency,
             stdout=None,
         )
     ).wait()
@@ -346,7 +346,7 @@ async def do_build(target: PackageVersionWithPath) -> int:
         if build_ret == 13:
             logger.warning("Already build %s, skipped.", base_dir_name)
             return 0
-        FAIL_REPOS.append(base_dir_name)
+        FAIL_REPOS.update({base_dir_name: build_ret})
         logger.warning("Build %s fail: %d", base_dir_name, build_ret)
 
     return build_ret
@@ -354,10 +354,10 @@ async def do_build(target: PackageVersionWithPath) -> int:
 
 async def clean_installed_packages() -> int:
     p = await asyncio.create_subprocess_exec(
-        "/bin/bash", "-c",
-        "PKGS=$(pacman -Qdtq | tr '\n' ' '); "
-        "if [ -n \"$PKGS\" ]; then echo $PKGS | xargs sudo pacman --noconfirm -Rcns ; fi",
-        stdout=None
+        "/bin/bash",
+        "-c",
+        "pacman -Qdtq | ifne sudo pacman --noconfirm -Rcns",
+        stdout=None,
     )
     await p.wait()
     return p.returncode
@@ -431,9 +431,8 @@ async def do_work(
         await fout.write(f"{int(time.time())}")
 
     if len(FAIL_REPOS):
-        logger.error("Build failed repositories:")
-        for repo in FAIL_REPOS:
-            logger.error("%s", repo)
+        logger.error("Build failed repositories (%d):", len(FAIL_REPOS))
+        logger.error("%s", ', '.join(f'{repo}({ret_code})' for repo, ret_code in FAIL_REPOS.items()))
 
     os.chdir(str(home_directory))
     if ret := await upload_packages():
diff --git a/utils/ng/systemd/update-repo-notify.service b/utils/ng/systemd/update-repo-notify.service
new file mode 100644
index 0000000..e609a8a
--- /dev/null
+++ b/utils/ng/systemd/update-repo-notify.service
@@ -0,0 +1,7 @@
+[Unit]
+Description=Update repo notify service
+
+[Service]
+User=update-database
+DynamicUser=yes
+ExecStart=/var/lib/update/update-db.py /run/update-repo-notify.sock
\ No newline at end of file
diff --git a/utils/ng/systemd/update-repo-notify.socket b/utils/ng/systemd/update-repo-notify.socket
new file mode 100644
index 0000000..f14743c
--- /dev/null
+++ b/utils/ng/systemd/update-repo-notify.socket
@@ -0,0 +1,16 @@
+[Unit]
+Description=Update repo notify socket
+
+[Socket]
+# If /var/run is not implemented as a symlink to /run, you may need to
+# specify ListenStream=/var/run/docker.sock instead.
+ListenStream=/run/update-repo-notify.sock
+SocketMode=0660
+SocketUser=update-database
+SocketGroup=http
+#Accept=yes
+MaxConnections=1
+FlushPending=yes
+
+[Install]
+WantedBy=sockets.target
\ No newline at end of file
diff --git a/utils/ng/update-db.py b/utils/ng/update-db.py
new file mode 100755
index 0000000..02064a4
--- /dev/null
+++ b/utils/ng/update-db.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+import argparse
+import logging
+import os
+import pathlib
+import signal
+import subprocess
+import threading
+import time
+from socketserver import TCPServer, StreamRequestHandler
+import socket
+from queue import Queue
+
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+
+"""class DataHandler(asyncio.Protocol):
+    def data_received(self, data: bytes) -> None:
+        logger.debug(data)
+
+    def eof_received(self) -> bool | None:
+        logger.debug("eof")
+
+
+class Server:
+    def __init__(self, unix_sockets: str):
+        self.unix_sockets = unix_sockets
+        self.server = None
+
+    async def connect(self) -> None:
+        #self.server = await asyncio.start_unix_server(self.handle_connection, self.unix_sockets, start_serving=False)
+        self.server = await asyncio.get_running_loop().create_unix_server(DataHandler, None,
+                                                            sock=socket.fromfd(3, socket.AF_UNIX, socket.SOCK_STREAM))
+
+    async def serving(self) -> None:
+        await self.server.serve_forever()
+
+    async def handle_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+        logger.debug("Connected")
+        while True:
+            task = asyncio.create_task(reader.readline())
+            finished, pending = await asyncio.wait([task], timeout=10)
+            if len(finished):
+                logger.debug(finished.pop().result())
+            break
+
+        writer.close()
+        await writer.wait_closed()
+        await self.close()
+
+    async def close(self) -> None:
+        if self.server:
+            self.server.close()
+            await self.server.wait_closed()
+
+
+async def main(unix_sockets: str) -> None:
+    logger.debug("Create connection")
+    #os.unlink(unix_sockets)
+    server = Server(unix_sockets)
+    await server.connect()
+    #await server.server()"""
+
+queue = Queue()
+
+ARCH = os.getenv("ARCH")
+
+
+def get_arch() -> str:
+    p = subprocess.Popen(["uname", "-m"], stdout=subprocess.PIPE)
+    stdout, _ = p.communicate()
+    return stdout.decode().lower().strip()
+
+
+if ARCH is None:
+    ARCH = get_arch()
+    logger.warning("Warning: You should specify arch environment variable, use current platform: %s", ARCH)
+
+
+class Handler(StreamRequestHandler):
+    def handle(self):
+        while line := self.rfile.readline():
+            if b"EOF" in line:
+                os.kill(os.getpid(), signal.SIGUSR1)
+                return
+            logger.info("Recv: %s", line)
+            # self.wfile.write(line.upper())
+
+
+# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4
+class Server(TCPServer):
+    # The constant would be better initialized by a systemd module
+    SYSTEMD_FIRST_SOCKET_FD = 3
+
+    def __init__(self, server_address, handler_cls):
+        # Invoke base but omit bind/listen steps (performed by systemd activation!)
+        TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False)
+        # Override socket
+        self.socket = socket.fromfd(
+            self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type
+        )
+
+
+class RepoProcessThread(threading.Thread):
+    def __init__(self, dest: pathlib.Path, key: str, pkg_src: pathlib.Path):
+        super().__init__()
+        self.exit_event = threading.Event()
+        self.dest = dest
+        self.key = key
+        self.pkg_src = pkg_src
+
+    def run(self) -> None:
+        while not self.exit_event.is_set():
+            while not queue.empty():
+                logger.info("Adding %s", repo := queue.get_nowait())
+                self.pkg_src.joinpath(repo)
+                subprocess.Popen(
+                    ["repo-add", str(self.dest), repo, "-s", "-v", "-R", "-k", self.key],
+                    stdout=subprocess.STDOUT
+                ).wait()
+            time.sleep(1)
+
+    def set_exit_event(self) -> None:
+        self.exit_event.set()
+
+
+def main(cwd: str):
+    def server_shutdown_wrapper(*_args) -> None:
+        logger.debug("Shutdown the server")
+        threading.Thread(target=server.shutdown, daemon=True,).start()
+
+    server = Server(("localhost", 0), Handler)
+    signal.signal(signal.SIGUSR1, server_shutdown_wrapper)
+
+    remote_path = pathlib.Path(cwd).resolve()
+    cwd = pathlib.Path(os.getcwd()).resolve()
+
+    RepoProcessThread(
+        cwd.joinpath("packages", ARCH, "kunoisayami.db.tar.gz").resolve(),
+        "4A0F0C8BC709ACA4341767FB243975C8DB9656B9",
+        remote_path
+    ).start()
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("update-db.py")
+    parser.add_argument("cwd")
+
+    args_ = parser.parse_args()
+
+    try:
+        import coloredlogs
+
+        coloredlogs.install(
+            logging.DEBUG,
+            fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
+        )
+    except ModuleNotFoundError:
+        logging.basicConfig(
+            level=logging.DEBUG,
+            format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
+        )
+    main(args_.cwd)
diff --git a/utils/ng/upload.php b/utils/ng/upload.php
new file mode 100644
index 0000000..fc60a61
--- /dev/null
+++ b/utils/ng/upload.php
@@ -0,0 +1,38 @@
+<?PHP
+    if (isset($_POST) && isset($_POST['token']) &&
+        $_POST['token']  === 'be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09';
+    ) {
+            if (!isset($_POST['arch'])) {
+                    http_response_code(400);
+                    echo "400 Bad Request: Should specify arch field while post files";
+                    die();
+            }
+            if (!empty($_FILES['file'])) {
+
+            $path = "/var/www/repo/". $_POST['arch'] . "/";
+            $path = $path . basename($_FILES['file']['name']);
+
+            if (move_uploaded_file($_FILES['file']['tmp_name'], $path)) {
+                echo "The file " . basename( $_FILES['file']['name']) . " has been uploaded";
+                $sock = stream_socket_client('unix:///run/update-repo-notify.sock', $errno, $errst);
+                fwrite($sock, $['file']['name']."\r\n");
+                fclose($sock);
+            } else {
+                echo "There was an error uploading the file, please try again!";
+            }
+        } else if (isset($_POST['action'])) {
+            if ($_POST['action'] === 'UPLOADED') {
+                $sock = stream_socket_client('unix:///run/update-repo-notify.sock', $errno, $errst);
+                fwrite($sock, "EOF\r\n");
+                fclose($sock);
+                echo "200 OK STARTED";
+            } else if ($_POST['action'] === 'REQUIRE_CLEAN') {
+                shell_exec("/usr/bin/rm -rf /var/www/repo/". $_POST['arch'] . "*");
+                shell_exec("/usr/bin/mkdir /var/www/repo/". $_POST['arch']);
+                echo "200 OK CLEANED";
+            }
+        }
+    } else {
+        http_response_code(403);
+    }
+?>