diff options
| author | KunoiSayami <[email protected]> | 2023-11-17 13:55:49 +0800 |
|---|---|---|
| committer | KunoiSayami <[email protected]> | 2023-11-17 13:55:49 +0800 |
| commit | 3eb6ed90365270c18e2a746ae0e849421c1805d1 (patch) | |
| tree | c9f3947af0849fea8c653d9d8c0eacfd13ff2775 | |
| parent | b070b9d828933eeb6466b7e575bd882b303fda5d (diff) | |
refactor: Remove utils from repository
Signed-off-by: KunoiSayami <[email protected]>
| -rw-r--r-- | utils/aur/AURUtils/__init__.py | 0 | ||||
| -rw-r--r-- | utils/aur/AURUtils/aur_deps.py | 9 | ||||
| -rw-r--r-- | utils/aur/AURUtils/errors.py | 16 | ||||
| -rw-r--r-- | utils/aur/AURUtils/rpc.py | 88 | ||||
| -rw-r--r-- | utils/aur/AURUtils/srcinfo.py | 97 | ||||
| -rw-r--r-- | utils/aur/AURUtils/types.py | 78 | ||||
| -rw-r--r-- | utils/aur/AURUtils/version.py | 5 | ||||
| -rwxr-xr-x | utils/aur/check-aur-dependencies.py | 80 | ||||
| -rwxr-xr-x | utils/aur/check-aur-update.py | 145 | ||||
| -rw-r--r-- | utils/aur/requirements.txt | 2 | ||||
| -rwxr-xr-x | utils/deploy.sh | 18 | ||||
| -rwxr-xr-x | utils/parse-pkgbuild.sh | 8 | ||||
| -rwxr-xr-x | utils/pkgbuild.py | 566 | ||||
| -rwxr-xr-x | utils/pkgbuild_bootstrap | 53 | ||||
| -rwxr-xr-x | utils/pkgman.py | 61 | ||||
| -rw-r--r-- | utils/systemd/update-repo-copy.service | 8 | ||||
| -rw-r--r-- | utils/systemd/update-repo-notify.service | 5 | ||||
| -rw-r--r-- | utils/systemd/update-repo-notify.socket | 16 | ||||
| -rw-r--r-- | utils/systemd/[email protected] | 7 | ||||
| -rwxr-xr-x | utils/systemd/wait-services.sh | 6 | ||||
| -rwxr-xr-x | utils/trigger.py | 78 | ||||
| -rwxr-xr-x | utils/trust_gpg.sh | 3 | ||||
| -rwxr-xr-x | utils/update-db.py | 71 | ||||
| -rw-r--r-- | utils/upload.php | 36 | ||||
| -rwxr-xr-x | utils/upload.py | 101 |
25 files changed, 0 insertions, 1557 deletions
diff --git a/utils/aur/AURUtils/__init__.py b/utils/aur/AURUtils/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/utils/aur/AURUtils/__init__.py +++ /dev/null diff --git a/utils/aur/AURUtils/aur_deps.py b/utils/aur/AURUtils/aur_deps.py deleted file mode 100644 index 3a4bc44..0000000 --- a/utils/aur/AURUtils/aur_deps.py +++ /dev/null @@ -1,9 +0,0 @@ -import re - -signs = [r'\>\=', r'\<\=', r'\=\=', r'\>', r'\<'] -signs_or = r'|'.join(signs) -deps_pattern = re.compile(rf'({signs_or}).*') - - -def depends_strip(deps: str) -> str: - return deps_pattern.sub('', deps)
\ No newline at end of file diff --git a/utils/aur/AURUtils/errors.py b/utils/aur/AURUtils/errors.py deleted file mode 100644 index 2e1fd87..0000000 --- a/utils/aur/AURUtils/errors.py +++ /dev/null @@ -1,16 +0,0 @@ -class RPCError(Exception): - - def __init__(self, method: str, message: str): - self.method = method - self.message = message - - def __str__(self): - return f'Method({self.method}): {self.message}' - - -class PackageNotFound(Exception): - pass - - -class MaxRetries(Exception): - pass diff --git a/utils/aur/AURUtils/rpc.py b/utils/aur/AURUtils/rpc.py deleted file mode 100644 index 6120d41..0000000 --- a/utils/aur/AURUtils/rpc.py +++ /dev/null @@ -1,88 +0,0 @@ -import asyncio -import random - -import aiohttp - -from .errors import RPCError, MaxRetries -from .types import Results, UrlParameterType - -DEFAULT_AURWEB_VERSION = 5 -MAX_WORKERS = 10 - -METHODS = {'search', 'info', 'multiinfo', 'msearch', 'suggest', - 'suggest-pkgbase', 'get-comment-form'} -FIELDS = {'name', 'name-desc', 'maintainer', 'depends', - 'makedepends', 'optdepends', 'checkdepends'} - - -class AsyncRPC: - """ - https://wiki.archlinux.org/title/Aurweb_RPC_interface - """ - - RPC_URL = 'https://aur.archlinux.org/rpc/' - - def __init__(self, max_workers: int = MAX_WORKERS): - self._client = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) - self.sem = asyncio.Semaphore(max_workers) - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close() - - async def close(self): - await self._client.close() - - async def _post(self, params: dict) -> dict: - async with self.sem: - for i in range(5): - try: - async with self._client.get(self.RPC_URL, params=params) as resp: - resp.raise_for_status() - return await resp.json() - except aiohttp.ServerTimeoutError: - await asyncio.sleep(3 * random.random()) - continue - raise MaxRetries - - async def request(self, - method: str, - method_params: dict[str, UrlParameterType], - ): - params = { - 'v': DEFAULT_AURWEB_VERSION, - 'type': method, - **method_params - } - json_response = await self._post(params) - if 'error' in json_response: - raise RPCError(method, json_response['error']) - return Results.parse(json_response) - - async def search(self, - arg: UrlParameterType, - search_by: str = None, - ): - method_params = { - 'by': search_by or [], - 'arg': arg, - } - return await self.request('search', method_params) - - async def info(self, - arg: UrlParameterType, - ): - return await self.multiinfo(arg) - - async def multiinfo(self, - arg: UrlParameterType, - ): - method_params = { - 'arg[]': arg, - } - return await self.request('multiinfo', method_params) - - async def msearch(self, arg: UrlParameterType): - return await self.search(arg, search_by='maintainer') diff --git a/utils/aur/AURUtils/srcinfo.py b/utils/aur/AURUtils/srcinfo.py deleted file mode 100644 index 34d2240..0000000 --- a/utils/aur/AURUtils/srcinfo.py +++ /dev/null @@ -1,97 +0,0 @@ -import re -from collections import defaultdict -from itertools import chain -from typing import Optional, Union - -from .errors import PackageNotFound -from .aur_deps import depends_strip - -key_value_pattern = re.compile(r'(?P<KEY>.+?)=(?P<VALUE>.+)') -multivalued_attrs = { - 'source', 'provides', 'conflicts', 'depends', 'replaces', 'optdepends', 'makedepends', 'checkdepends', - 'md5sums', 'sha1sums', 'sha224sums', 'sha256sums', 'sha384sums', 'sha512sums', 'arch' -} - - -class SRCINFO: - - def __init__(self, pkgbase: str, pkgbase_opt: dict, pkgs_opt: dict): - self.pkgbase = pkgbase - self.pkgbase_opt = pkgbase_opt - self.pkgs_opt = pkgs_opt - - @classmethod - def parse(cls, source: str): - pkgbase = None - pkgbase_opt = defaultdict(list) - pkgs_opt = {} - handler = None - for l in source.splitlines(): - m = key_value_pattern.search(l.strip()) - if m is None: - continue - key, value = m.group('KEY').strip(), m.group('VALUE').strip() - if key == 'pkgbase': - pkgbase = value - handler = pkgbase_opt - elif key == 'pkgname': - pkgs_opt[value] = defaultdict(list) - handler = pkgs_opt[value] - else: - if key in multivalued_attrs: - handler[key].append(value) - else: - handler[key] = value - return SRCINFO(pkgbase, pkgbase_opt, pkgs_opt) - - @property - def pkgs(self) -> dict: - return { - k: {i: self.get_pkg_opt(k, i) for i in chain(self.pkgbase_opt.keys(), v.keys())} - for k, v in self.pkgs_opt.items() - } - - @property - def pkgs_name(self) -> list: - return list(self.pkgs_opt.keys()) - - def _chain_pkg_opt(self, pkgname: str, opt: str) -> chain: - if pkgname not in self.pkgs_name: - raise PackageNotFound(f'{pkgname} not found in {self.pkgbase}') - return chain(self.pkgbase_opt.get(opt, []), self.pkgs_opt[pkgname].get(opt, [])) - - def _chain_opt(self, opt: str) -> chain: - return chain(self.pkgbase_opt.get(opt, []), *(i.get(opt, []) for i in self.pkgs_opt.values())) - - def get_pkg_opt(self, pkgname: str, opt: str) -> Union[list, str]: - if pkgname not in self.pkgs_name: - raise PackageNotFound(f'{pkgname} not found in {self.pkgbase}') - if opt in multivalued_attrs: - return list(self._chain_pkg_opt(pkgname, opt)) - return self.pkgbase_opt.get(opt) - - def get_opt(self, opt: str) -> Union[list, str]: - if opt in multivalued_attrs: - return list(self._chain_opt(opt)) - return self.pkgbase_opt.get(opt) - - @property - def depends(self) -> list[str]: - return list(map(depends_strip, self._chain_opt('depends'))) - - @property - def makedepends(self) -> list[str]: - return list(map(depends_strip, self._chain_opt('makedepends'))) - - @property - def optdepends(self) -> list[str]: - return list(self._chain_opt('optdepends')) - - @property - def version(self) -> tuple[Optional[str], str, str]: - return self.pkgbase_opt.get('epoch', None), \ - self.pkgbase_opt['pkgver'], self.pkgbase_opt['pkgrel'] - - def get_version(self) -> str: - epoch = f"{self.pkgbase_opt['epoch']}:" if self.pkgbase_opt.get('epoch') else '' - return f"{epoch}{self.pkgbase_opt['pkgver']}-{self.pkgbase_opt['pkgrel']}" diff --git a/utils/aur/AURUtils/types.py b/utils/aur/AURUtils/types.py deleted file mode 100644 index 9830b08..0000000 --- a/utils/aur/AURUtils/types.py +++ /dev/null @@ -1,78 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime -from collections.abc import Iterator -from typing import Union - -from .aur_deps import depends_strip - -UrlParameterType = Union[str, list[str]] - - -@dataclass() -class Package: - - ID: int - Name: str - PackageBaseID: int - PackageBase: str - Version: str - Description: str - URL: str - NumVotes: int - Popularity: float - OutOfDate: int - Maintainer: str - FirstSubmitted: int - LastModified: int - URLPath: str - Conflicts: list[str] = None - Provides: list[str] = None - Depends: list[str] = None - MakeDepends: list[str] = None - OptDepends: list[str] = None - License: list[str] = None - Keywords: list[str] = None - - @property - def depends(self) -> list[str]: - return list(map(depends_strip, self.Depends)) - - @property - def makedepends(self) -> list[str]: - return list(map(depends_strip, self.MakeDepends)) - - @property - def optdepends(self) -> list[str]: - return list(self.OptDepends) - - @property - def first_submitted(self) -> datetime: - return datetime.fromtimestamp(self.FirstSubmitted) - - @property - def last_modified(self) -> datetime: - return datetime.fromtimestamp(self.LastModified) - - def get_version(self) -> str: - return self.Version - - -@dataclass -class Results: - - resultcount: int - results: list - type: str - version: int - - @classmethod - def parse(cls, json_response: dict): - return Results(**json_response) - - def empty(self) -> bool: - return self.resultcount == 0 - - def packages(self) -> Iterator[Package]: - if self.type in {'multiinfo', 'search'}: - return [Package(**i) for i in self.results] - return [] diff --git a/utils/aur/AURUtils/version.py b/utils/aur/AURUtils/version.py deleted file mode 100644 index b3d5329..0000000 --- a/utils/aur/AURUtils/version.py +++ /dev/null @@ -1,5 +0,0 @@ -import pyalpm # type: ignore - - -def vercmp(ver1: str, ver2: str) -> int: - return pyalpm.vercmp(ver1, ver2)
\ No newline at end of file diff --git a/utils/aur/check-aur-dependencies.py b/utils/aur/check-aur-dependencies.py deleted file mode 100755 index 38b6436..0000000 --- a/utils/aur/check-aur-dependencies.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2021 KunoiSayami and contributors -# -# This module is part of KunoiSayami/repos and is released under -# the AGPL v3 License: https://www.gnu.org/licenses/agpl-3.0.txt -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. -import argparse -import asyncio -import heapq -import logging -import re -from pathlib import Path - -import aiofiles -import aiohttp - - -async def check_pkg(session: aiohttp.ClientSession, num: int, pkg: str) -> tuple[int, bool]: - logging.info(f'Checking {pkg}') - ret = await session.head(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={pkg}') - return num, ret.status == 200 - - -async def main() -> int: - parser = argparse.ArgumentParser(description='check-aur-dependencies.py') - parser.add_argument('paths', metavar='sub_path', type=str, nargs='*', help='sub-path for repo root') - - args = parser.parse_args() - repo_dir = Path(*args.paths) - pkgbuild_file = repo_dir.joinpath('PKGBUILD') - if not pkgbuild_file.exists(): - logging.error("Can't locate PKGBUILD file, please specify directory in arguments or chdir to folder") - return 1 - - async with aiofiles.open(pkgbuild_file, 'r', encoding='utf-8') as f: - pkgbuild_text = await f.read() - - runtime_deps_match = re.search(r"depends=\((?P<DEPS>([^)]|\n)*)\)", pkgbuild_text, re.M) - if not runtime_deps_match: - logging.warning("Can't found 'depends' keyword, if you think this is mistake, please report issue") - - make_deps_match = re.search(r"makedepends=\((?P<DEPS>([^)]|\n)*)\)", pkgbuild_text, re.M) - if not make_deps_match: - logging.warning("Can't found `makedepends' keyword, if you think this is mistake, please report issue") - - deps = [dep.strip('\'') for m in {runtime_deps_match, make_deps_match} if m for dep in m.group('DEPS').split()] - - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: - tasks = [asyncio.create_task(check_pkg(session, n, pkg)) for n, pkg in enumerate(deps)] - yay_dep_nums = [] - for task in asyncio.as_completed(tasks): - n, r = await task - if r: - heapq.heappush(yay_dep_nums, n) - - yaydeps_dir = repo_dir.parent.joinpath('.yaydeps') - - if len(yay_dep_nums) > 0: - async with aiofiles.open(yaydeps_dir.joinpath(repo_dir.stem), 'w', encoding='utf-8') as f: - await f.write('\n'.join([*(deps[heapq.heappop(yay_dep_nums)] for _ in range(len(yay_dep_nums))), ''])) - - return 0 - - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s') - exit(asyncio.run(main())) diff --git a/utils/aur/check-aur-update.py b/utils/aur/check-aur-update.py deleted file mode 100755 index 4be4e09..0000000 --- a/utils/aur/check-aur-update.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import argparse -import heapq -import logging -from collections.abc import Coroutine -from pathlib import Path -from typing import Optional - -import aiohttp -import aiofiles - -from AURUtils.srcinfo import SRCINFO -from AURUtils.version import vercmp - - -MAX_WORKERS = 16 - - -async def check_pkg(session: aiohttp.ClientSession, num: int, pkg: str) -> tuple[int, bool]: - logging.info(f'Checking {pkg}') - ret = await session.head(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={pkg}') - return num, ret.status == 200 - - -async def check_aur_update( - session: aiohttp.ClientSession, - item: Path, dry_run: bool, - parse_pkgbuild: Optional[Path] = None, - check_deps: bool = True -) -> None: - if item.name.startswith('.') or not item.is_dir(): - return - pkgbuild_file = item.joinpath('PKGBUILD') - if not pkgbuild_file.exists(): - logging.warning(f'PKGBUILD not in {str(item)}, SKIP!') - return - srcinfo_file = item.joinpath('.SRCINFO') - if not srcinfo_file.exists(): - if parse_pkgbuild.exists(): - logging.warning(f'.SRCINFO not in {str(item)}, try to parse') - proc = await asyncio.create_subprocess_exec( - f'{str(parse_pkgbuild.resolve())}', - f'{str(pkgbuild_file)}', - stdout=asyncio.subprocess.PIPE, - ) - srcinfo = SRCINFO.parse((await proc.stdout.read()).decode()) - logging.info(f'extract .SRCINFO from {srcinfo.pkgbase}') - else: - logging.warning(f'.SRCINFO not in {str(item)}, no tool to parse') - return - else: - async with aiofiles.open(srcinfo_file, 'r') as f: - srcinfo = SRCINFO.parse(await f.read()) - version = srcinfo.get_version() - async with session.get(f'https://aur.archlinux.org/cgit/aur.git/plain/.SRCINFO?h={srcinfo.pkgbase}') as resp: - if resp.status != 200: - logging.info(f'{item.name} not register in AUR') - return - new_srcinfo = SRCINFO.parse(await resp.text()) - new_version = new_srcinfo.get_version() - result = vercmp(new_version, version) - if result == 0: - return - elif result < 0: - logging.warning(f'Warning: {item.name}({version}) is newer than AUR') - else: - if item.joinpath('.git').exists(): - if not dry_run: - await asyncio.create_subprocess_exec('git', '-C', f'{str(item)}', 'pull', 'origin', 'master') - logging.info(f'Upgrade {item.name} from {version} to {new_version}') - else: - logging.info(f'Found update {item.name}({new_version}) (local: {version})') - if check_deps: - deps = [*new_srcinfo.makedepends, *new_srcinfo.depends] - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: - tasks = [asyncio.create_task(check_pkg(session, n, pkg)) for n, pkg in enumerate(deps)] - yay_dep_nums = [] - for task in asyncio.as_completed(tasks): - n, r = await task - if r: - heapq.heappush(yay_dep_nums, n) - - yaydeps_dir = item.parent.joinpath('.yaydeps') - - if len(yay_dep_nums) > 0: - async with aiofiles.open(yaydeps_dir.joinpath(item.stem), 'w', encoding='utf-8') as f: - await f.write( - '\n'.join([*(deps[heapq.heappop(yay_dep_nums)] for _ in range(len(yay_dep_nums))), ''])) - - -async def with_sem(sem: asyncio.Semaphore, coro: Coroutine): - async with sem: - return await coro - - -async def main() -> int: - parser = argparse.ArgumentParser(description='aur-check-update.py') - parser.add_argument('--dry-run', dest='dry', help='dry run', action='store_true') - parser.add_argument('--check-deps', dest='deps', help='check deps', action='store_true') - parser.add_argument('--parse-pkgbuild', dest='script', metavar='SCRIPT', - help='a script to parse pkgbuild', - action='store') - parser.add_argument('--miax-workers', dest='workers', metavar='MAX_WORKERS', - help=f'aur check max workers, default: {MAX_WORKERS}', - default=MAX_WORKERS, action='store', type=int) - parser.add_argument(dest='repo_dir', metavar='REPO_DIR', help='repository directory', nargs='?') - - args = parser.parse_args() - - dry_run = args.dry - check_deps = args.deps - sem = asyncio.Semaphore(args.workers) - - if args.repo_dir is None: - logging.error('repository directory not found') - return 1 - - script_file = Path(args.script) - if args.script is None or not (script_file.is_file() and script_file.exists()): - logging.info(f'Parse script {str(script_file)+" " if args.script is None else ""} not found') - script_file = None - - repo_dir = Path(args.repo_dir) - - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: - tasks = [asyncio.create_task( - with_sem(sem, check_aur_update( - session=session, - item=item, - dry_run=dry_run, - parse_pkgbuild=script_file, - check_deps=check_deps - )) - ) for item in repo_dir.iterdir()] - await asyncio.gather(*tasks) - - return 0 - - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s') - asyncio.run(main()) diff --git a/utils/aur/requirements.txt b/utils/aur/requirements.txt deleted file mode 100644 index 785d447..0000000 --- a/utils/aur/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -aiofiles~=0.7.0 -aiohttp~=3.8.0 diff --git a/utils/deploy.sh b/utils/deploy.sh deleted file mode 100755 index bf2c695..0000000 --- a/utils/deploy.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeuo pipefail - -if [ $# -eq 0 ]; then - echo 'Should provide least one argument' - exit 1 -fi - -if [ -z ${SSH_CLIENT+x} ]; then - ADDITIONAL_PARAM="--partial" -else - ADDITIONAL_PARAM="--partial -v" -fi - -/usr/bin/date +%s > "packages/LASTSYNC" -/usr/bin/rsync $ADDITIONAL_PARAM -r -c --update --links --delete --exclude=.gitignore "packages/" $1:/var/www/repo/ diff --git a/utils/parse-pkgbuild.sh b/utils/parse-pkgbuild.sh deleted file mode 100755 index c0da48a..0000000 --- a/utils/parse-pkgbuild.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# no makepkg version -# https://github.com/archlinuxcn/repo/blob/master/parse-pkgbuild - -pushd $(dirname $@) > /dev/null -makepkg --printsrcinfo -popd > /dev/null diff --git a/utils/pkgbuild.py b/utils/pkgbuild.py deleted file mode 100755 index 5dfbd6a..0000000 --- a/utils/pkgbuild.py +++ /dev/null @@ -1,566 +0,0 @@ -#!/usr/bin/env python -from __future__ import annotations -import argparse -import asyncio -import dataclasses -import lzma -import logging -import operator -import os -import pathlib -import shutil -import subprocess -import time - -import aiofiles - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -PKGBUILD_DIRECTORY_BASE = "repo" - -CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "") -CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "") -CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "") -BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "") -TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve() -NO_UPLOAD = os.getenv("BUILD_NO_UPLOAD") is None -FAIL_REPOS = [] - - -async def get_arch() -> str: - p = await asyncio.create_subprocess_exec("uname", "-m", stdout=subprocess.PIPE) - stdout, _ = await p.communicate() - return stdout.decode().lower().strip() - - -ARCH = asyncio.run(get_arch()) - - -class PackageVersionWithPath: - pkg_version: PackageVersion - path: pathlib.Path - # For debug use - spend_time: float - - @property - def name(self) -> str: - return self.pkg_version.name - - @property - def version(self) -> str: - return self.pkg_version.version - - @property - def arch(self) -> bool: - return self.pkg_version.arch_match - - def __str__(self) -> str: - if self.spend_time < 0: - return self.pkg_version.__str__() - return f"{self.pkg_version.__str__()} (spend: {self.spend_time:.2f}s)" - - -class PackageVersion: - name: str - version: str - arch_match: bool - - def __eq__(self, other: PackageVersion) -> bool: - return other.version == self.version - - def __gt__(self, other: PackageVersion) -> bool: - return self.version > other.version - - def __lt__(self, other: PackageVersion) -> bool: - return self.version < other.version - - def __le__(self, other: PackageVersion) -> bool: - return self < other or self == other - - def __ge__(self, other: PackageVersion) -> bool: - return self > other or self == other - - def __ne__(self, other: PackageVersion) -> bool: - return not self == other - - def __str__(self) -> str: - return f"{self.name} {self.version}" - - -class MakepkgErrorReason: - package: str - code: int - post_install_dependency: str | None - - def to_str(self) -> str: - if self.post_install_dependency is not None: - return f"{self.package}(dependency)[{self.post_install_dependency}]({self.code})" - else: - return f"{self.package}({self.code})" - - -def fetch_database(pkg_db: str) -> dict[str, str]: - name_read = False - version_read = False - pkg_name = "" - package_storage = {} - with lzma.open(pkg_db) as fin: - for line in fin: - line = line.strip() - if name_read: - pkg_name = line.decode() - name_read = False - elif version_read: - package_storage.update({pkg_name: line.decode()}) - # logger.debug("%s %s", pkg_name, line.decode()) - version_read = False - pkg_name = "" - if line == b"%NAME%": - assert not name_read - name_read = True - if line == b"%VERSION%": - assert not version_read - version_read = True - assert (name_read is False and version_read is False) or ( - version_read != name_read - ) - return package_storage - - [email protected](init=False) -class BuildEnvs: - src_dest: str - pkg_dest: str - makepkg_conf: str - signing_arg: list[str] - - def __init__(self): - cwd = pathlib.Path(os.getcwd()) - self.pkg_dest = str(cwd.joinpath("packages", ARCH).resolve()) - self.src_dest = str(cwd.joinpath("build").resolve()) - self.makepkg_conf = str(cwd.joinpath("makepkg_current.conf").resolve()) - if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != "": - self.signing_arg = ["--sign"] - else: - logger.info("Skip signing package") - - def get_dict(self) -> dict[str, str | list[str]]: - return { - "src_dest": self.src_dest, - "pkg_dest": self.pkg_dest, - "makepkg_conf": self.makepkg_conf, - "signing_arg": self.signing_arg, - } - - -def get_src_info(s: str) -> PackageVersion: - base = "" - ver = "" - rel = "" - arch = [] - for line in s.splitlines(False): - if (line := line.strip()).startswith("pkgver"): - ver = line.split("=", maxsplit=2)[1] - elif line.startswith("pkgrel"): - rel = line.split("=", maxsplit=2)[1] - elif line.startswith("pkgbase"): - base = line.split("=", maxsplit=2)[1] - elif line.startswith("arch"): - arch.append(line.split("=", maxsplit=2)[1].strip().lower()) - return PackageVersion( - base.strip(), f"{ver.strip()}-{rel.strip()}", ARCH in arch or "any" in arch - ) - - -async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersionWithPath: - if (src_path := pathlib.Path(path.joinpath(".SRCINFO"))).is_file(): - async with aiofiles.open(str(src_path)) as fin: - info = get_src_info(await fin.read()) - spend_time = -1.0 - else: - start_time = time.time() - p = await asyncio.create_subprocess_exec( - "makepkg", "--printsrcinfo", cwd=src_path.parent, stdout=subprocess.PIPE - ) - (stdout, stderr) = await p.communicate() - info = get_src_info(stdout.decode()) - spend_time = time.time() - start_time - return PackageVersionWithPath(info, pathlib.Path(path).resolve(), spend_time) - - -async def fetch_packages_from_directory( - repo_directory: pathlib.Path, -) -> set[asyncio.Task[PackageVersionWithPath]]: - tasks = [] - - for folder in os.listdir(str(repo_directory)): - if folder.startswith("."): - continue - - if (pkg_dir := repo_directory.joinpath(folder).resolve()).is_file(): - continue - - tasks.append(asyncio.create_task(parse_srcinfo(pkg_dir))) - - finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) - return finished - - -async def get_build_target( - pkg_db: str, - home_directory: pathlib.Path, - build_target: list[str] | None, - build_all: bool, -) -> list[PackageVersionWithPath]: - repo = fetch_database(pkg_db) - pending_build = [] - - finished = await fetch_packages_from_directory( - home_directory.joinpath(PKGBUILD_DIRECTORY_BASE) - ) - - if len(BUILD_OVERRIDE): - logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE) - override = BUILD_OVERRIDE.split(";") - return [ - future.result() for future in finished if future.result().name in override - ] - - if build_all: - return [future.result() for future in finished] - - if len(build_target): - return [ - future.result() - for future in finished - if future.result().name in build_target - ] - - for future in finished: - pkg = future.result() - if pkg.name in repo: - # print(pkg.name, repo[pkg.name], pkg.version) - if repo[pkg.name] < pkg.version: - pending_build.append(pkg) - else: - pending_build.append(pkg) - - # print(pending_build) - return pending_build - - -async def run_hook(origin_path: pathlib.Path, target: str) -> None: - if ( - hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve() - ).is_file(): - logger.info("Running %s hook script", target) - p = await asyncio.create_subprocess_exec( - str(hook_script), stdout=None, stderr=subprocess.STDOUT - ) - await p.wait() - if p.returncode != 0: - logger.error( - "Hook script return code: %d, skip other operation", p.returncode - ) - return - - -async def run_build_with_install( - src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str] -) -> int: - return await run_build(src_dest, pkg_dest, makepkg_conf, signing_arg, "-i") - - -async def run_build( - src_dest: str, - pkg_dest: str, - makepkg_conf: str, - signing_arg: list[str], - *other_args: str, -) -> int: - env = os.environ - env.update( - { - "SRCPKGDEST": src_dest, - "SRCDEST": src_dest, - "PKGDEST": pkg_dest, - "MAKEPKG_CONF": makepkg_conf, - } - ) - p = await asyncio.create_subprocess_exec( - "makepkg", - "--clean", - "-s", - *other_args, - *signing_arg, - "--asdeps", - "--noconfirm", - "--needed", - "--noprogressbar", - env=env, - stdout=None, - ) - await p.wait() - return p.returncode - - -async def install_dependency_via_yay(dependency: str) -> int: - await ( - p := await asyncio.create_subprocess_exec( - "yay", - "--noconfirm", - "--asdeps", - "--needed", - "--noprogressbar", - "-S", - dependency, - stdout=None, - ) - ).wait() - return p.returncode - - -async def build_yay_dependency(target: PackageVersionWithPath, dependency: str) -> int: - if (dep_dir := target.path.parent.joinpath(dependency).resolve()).is_dir(): - os.chdir(str(dep_dir)) - await run_hook(target.path, dependency) - return await run_build_with_install(**BUILD_ENVS.get_dict()) - else: - return await install_dependency_via_yay(dependency) - - -async def do_build(target: PackageVersionWithPath) -> int: - try: - os.chdir(str(target.path)) - logger.info("Building %s (%s)", target.name, target.path.name) - except OSError: - logger.exception("Got exception while chdir to %s", target.path) - raise - - base_dir_name = target.path.name - - await run_hook(target.path, base_dir_name) - - if ( - yay_deps := target.path.parent.joinpath(".yaydeps") - .joinpath(base_dir_name) - .resolve() - ).is_file(): - async with aiofiles.open(str(yay_deps)) as fin: - while dep := (await fin.readline()).strip(): - logger.debug("Find dependency %s, build first", dep) - if ret := await build_yay_dependency(target, dep): - logger.error( - "Build dependencies package error, skipped next step (%d)", - ret, - ) - FAIL_REPOS.append(MakepkgErrorReason(target.name, ret, dep)) - return ret - - if ( - gpg_keys := target.path.parent.joinpath(".gpg_keys") - .joinpath(base_dir_name) - .resolve() - ).is_file(): - await ( - await asyncio.create_subprocess_exec( - "/bin/bash", str(gpg_keys), stdout=None - ) - ).wait() - await ( - await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None) - ).wait() - - os.chdir(str(target.path)) - build_ret = await run_build(**BUILD_ENVS.get_dict()) - - if build_ret != 0: - if build_ret == 13: - logger.warning("Already build %s, skipped.", base_dir_name) - return 0 - FAIL_REPOS.append(MakepkgErrorReason(base_dir_name, build_ret, None)) - logger.warning("Build %s fail: %d", base_dir_name, build_ret) - - return build_ret - - -async def clean_installed_packages() -> int: - p = await asyncio.create_subprocess_exec( - "/bin/bash", - "-c", - "pacman -Qdtq | ifne sudo pacman --noconfirm -Rcns -", - stdout=None, - ) - await p.wait() - return p.returncode - - -async def upload_packages() -> int: - remote_path = os.getenv("REMOTE_PATH") - upload_token = os.getenv("UPLOAD_TOKEN") - if remote_path is None or upload_token is None: - logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload") - await ( - p := await asyncio.create_subprocess_exec( - "./utils/upload.py", - remote_path, - upload_token, - ARCH, - "--directory", - BUILD_ENVS.pkg_dest, - stdout=None, - ) - ).wait() - return p.returncode - - -async def do_work( - pkg_db: str, - build_all: bool, - build_target: list[str], - fail_fast: bool = False, - run_auto_remove: bool = True, - dry_run: bool = False, -) -> int: - logger.info( - "build_all: %s, fail_fast: %s, auto_remove: %s, dry_run: %s", - build_all, - fail_fast, - run_auto_remove, - dry_run, - ) - - home_directory = pathlib.Path(os.getcwd()) - - build_target = sorted( - await get_build_target(pkg_db, home_directory, build_target, build_all), - key=operator.attrgetter("name"), - ) - logger.debug( - "Pending build summary (%d): %s", - len(build_target), - ", ".join(map(lambda x: x.name, build_target)), - ) - - for target in build_target: - if not target.arch: - logger.info("Skipped %s (Architecture not match)", target.name) - continue - if dry_run: - logger.info("Skipped %s (--dry-run specified)", target.name) - continue - - if (await do_build(target)) != 0: - if fail_fast: - break - else: - await asyncio.sleep(3) - - if run_auto_remove: - await clean_installed_packages() - for _ in range(3): - logger.debug("---------------------------------------------------------") - - if dry_run: - return 0 - - if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir(): - shutil.rmtree(str(src_dst), ignore_errors=True) - - # Override last build timestamp - async with aiofiles.open( - str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve()), "w" - ) as fout: - await fout.write(f"{int(time.time())}") - - # Print all failed repos - if len(FAIL_REPOS): - logger.error("Build failed repositories (%d):", len(FAIL_REPOS)) - logger.error( - "%s", - ", ".join(repo.to_str() for repo in FAIL_REPOS), - ) - - os.chdir(str(home_directory)) - if ret := await upload_packages(): - logger.error("Got error in upload packages (%d)", ret) - return ret - - if len(FAIL_REPOS): - return 1 - - -def check_build_target(args: argparse.Namespace) -> bool: - if args.build_all or len(args.build_target): - return True - if "REBUILD ALL" in CI_COMMIT_TITLE or "REBUILD_ALL" in CI_COMMIT_TITLE: - return True - return False - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="pkgbuild.py", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("db_file", help="Remote database file") - parser.add_argument( - "--build-all", help="Force build all package", action="store_true" - ) - parser.add_argument( - "--build-target", nargs="*", default=list(), help="Override build targets" - ) - parser.add_argument( - "--sign", default=False, action="store_true", help="Sign package after build" - ) - parser.add_argument( - "--fail-fast", - default=False, - action="store_true", - help="If build failed, exit script", - ) - parser.add_argument( - "--disable-auto-remove", - default=False, - action="store_true", - help="Disable auto remove after build package", - ) - parser.add_argument( - "--dry-run", - default=False, - action="store_true", - help="Run but not actually build package", - ) - args_ = parser.parse_args() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - - BUILD_ENVS = BuildEnvs() - if args_.sign: - BUILD_ENVS.signing_arg = ["--sign"] - ret_ = asyncio.run( - do_work( - args_.db_file, - check_build_target(args_), - args_.build_target, - args_.fail_fast, - args_.disable_auto_remove, - args_.dry_run, - ) - ) - exit(ret_) diff --git a/utils/pkgbuild_bootstrap b/utils/pkgbuild_bootstrap deleted file mode 100755 index e0d23c6..0000000 --- a/utils/pkgbuild_bootstrap +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [ -z "$DOCKER_SETUP_SCRIPT" ]; then - echo $GPG_PRIV_KEY | base64 -d | gpg --import - cd "$(dirname "$0")" - cd ../../ - echo "Current pwd: $(pwd)" - sudo pacman -Sy -fi - -set -Eeo pipefail -trap cleanup SIGINT SIGTERM ERR EXIT - -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset SRCDEST - unset CONFDEST - unset TMPCONF - unset FLAGS - unset ARCH -} - -ARCH=$(uname -m) -CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf" - -TMPCONF="makepkg_current.conf" -cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF" -cat "$CONFDEST" >> "$TMPCONF" - -FLAGS="" - -if [ "$EUID" -eq 0 ] ; then - echo "\033[0;32mPlease run this script as non-root\033[0m" - exit 3 -fi - -if [ -z "$REMOTE_DATABASE" ]; then - echo "\033[0;32mYou should specify remote address\033[0m" - exit 4 -fi - -if [ -n "$FORCE_BUILD_ALL" ]; then - FLAGS+="--build-all " -fi - -if [ -n "$BUILD_FAIL_FAST" ]; then - FLAGS+="--fail-fast " -fi - -curl -fL -o remote.db "$REMOTE_DATABASE" - - -./utils/pkgbuild.py remote.db $FLAGS diff --git a/utils/pkgman.py b/utils/pkgman.py deleted file mode 100755 index 0e7388f..0000000 --- a/utils/pkgman.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -import argparse -import asyncio -import os -import pathlib - -try: - import pkgbuild -except ModuleNotFoundError: - import utils.pkgbuild as pkgbuild - - -async def fetch_and_get(_remote_addr: str): - raise NotImplementedError("TODO!") - - -async def show_diff(database: str, path: str) -> None: - pkg_dir = pathlib.Path(os.getcwd()).joinpath(path).resolve() - database_result = pkgbuild.fetch_database(database) - pending = [] - for package in await pkgbuild.fetch_packages_from_directory(pkg_dir): - if (package := package.result()).name in database_result: - if database_result[package.name] < package.version: - pending.append(package) - else: - pending.append(package) - for package in pending: - print(package.name, package.version) - - -async def main(args: argparse.Namespace): - if args.sub == "show": - print(pkgbuild.fetch_database(args.database)) - elif args.sub == "show_local": - pkg_dir = pathlib.Path(os.getcwd()).joinpath(args.path).resolve() - for package in await pkgbuild.fetch_packages_from_directory(pkg_dir): - print(package.result()) - elif args.sub == "show_internet": - await fetch_and_get(args.database) - elif args.sub == 'diff': - await show_diff(args.database, args.path) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser('pkgman.py') - sub = parser.add_subparsers(title="subcommand", dest="sub") - decmp_parser = sub.add_parser("show", description="Show databases") - decmp_parser.add_argument("database") - - local_show_parser = sub.add_parser("show_local") - local_show_parser.add_argument("path", default="./repo") - - decmp_online_parser = sub.add_parser("show_internet", description="Show remote databases") - decmp_online_parser.add_argument("database") - - show_diff_parser = sub.add_parser("diff") - show_diff_parser.add_argument("database") - show_diff_parser.add_argument("path") - - args_ = parser.parse_args() - asyncio.run(main(args_)) diff --git a/utils/systemd/update-repo-copy.service b/utils/systemd/update-repo-copy.service deleted file mode 100644 index af79ee5..0000000 --- a/utils/systemd/update-repo-copy.service +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Update repo copy service - -[Service] -WorkingDirectory=/var/lib/update/packages/ -ExecStartPre=/var/lib/update/wait-services.sh -ExecStart=/usr/bin/rsync -r --links -vct --update --delete . /var/www/packages
\ No newline at end of file diff --git a/utils/systemd/update-repo-notify.service b/utils/systemd/update-repo-notify.service deleted file mode 100644 index d7e3c9a..0000000 --- a/utils/systemd/update-repo-notify.service +++ /dev/null @@ -1,5 +0,0 @@ -[Unit] -Description=Update repo notify service - -[Service] -ExecStart=/var/lib/update/trigger.py
\ No newline at end of file diff --git a/utils/systemd/update-repo-notify.socket b/utils/systemd/update-repo-notify.socket deleted file mode 100644 index f14743c..0000000 --- a/utils/systemd/update-repo-notify.socket +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Update repo notify socket - -[Socket] -# If /var/run is not implemented as a symlink to /run, you may need to -# specify ListenStream=/var/run/docker.sock instead. -ListenStream=/run/update-repo-notify.sock -SocketMode=0660 -SocketUser=update-database -SocketGroup=http -#Accept=yes -MaxConnections=1 -FlushPending=yes - -[Install] -WantedBy=sockets.target
\ No newline at end of file diff --git a/utils/systemd/[email protected] b/utils/systemd/[email protected] deleted file mode 100644 index 6fa98da..0000000 --- a/utils/systemd/[email protected] +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Update repo %i service - -[Service] -User=update-database -ExecStart=/var/lib/update/update-db.py /var/lib/update %i AAAABBBBCCCCDDDDDDDDCCCCBBBBAAAA -#ExecStart=rsync
\ No newline at end of file diff --git a/utils/systemd/wait-services.sh b/utils/systemd/wait-services.sh deleted file mode 100755 index be20239..0000000 --- a/utils/systemd/wait-services.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -echo "Checking is service running" -while /usr/bin/systemctl status --no-page | grep update-repo@ | grep -v grep; -do - sleep 1 -done
\ No newline at end of file diff --git a/utils/trigger.py b/utils/trigger.py deleted file mode 100755 index ca0cfd1..0000000 --- a/utils/trigger.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -import argparse -import logging -import os -import signal -import socket -import subprocess -import threading -from socketserver import StreamRequestHandler, TCPServer - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -class Handler(StreamRequestHandler): - def handle(self): - while line := self.rfile.readline(): - if b"ARCH=" in line: - ( - subprocess.Popen( - [ - "systemctl", - "--no-block", - "start", - f"update-repo@{line.split(b'=')[1].decode().strip()}", - ] - ) - ).wait() - os.kill(os.getpid(), signal.SIGUSR1) - return - logger.info("Recv: %s", line) - - -# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4 -class Server(TCPServer): - # The constant would be better initialized by a systemd module - SYSTEMD_FIRST_SOCKET_FD = 3 - - def __init__(self, server_address, handler_cls): - # Invoke base but omit bind/listen steps (performed by systemd activation!) - TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False) - # Override socket - self.socket = socket.fromfd( - self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type - ) - - -def main(): - def server_shutdown_wrapper(*_args) -> None: - logger.debug("Shutdown the server") - threading.Thread( - target=server.shutdown, - daemon=True, - ).start() - - server = Server(("localhost", 0), Handler) - signal.signal(signal.SIGUSR1, server_shutdown_wrapper) - - server.serve_forever() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - - main() diff --git a/utils/trust_gpg.sh b/utils/trust_gpg.sh deleted file mode 100755 index 5d4f8c3..0000000 --- a/utils/trust_gpg.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -gpg --list-keys --fingerprint | grep pub -A 1 | grep -Ev "pub|--" | tr -d ' ' \ - | awk 'BEGIN { FS = "\n" } ; { print $1":6:" } ' | gpg --import-ownertrust
\ No newline at end of file diff --git a/utils/update-db.py b/utils/update-db.py deleted file mode 100755 index 3e318f5..0000000 --- a/utils/update-db.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -import argparse -import asyncio -import logging -import os -import pathlib -import shutil - - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -async def add_package(dest: pathlib.Path, pkg_name: str, key: str) -> None: - # logger.debug("%s %s %s", dest, pkg_name, key) - await ( - await asyncio.create_subprocess_exec( - "/usr/bin/repo-add", - str(dest), - pkg_name, - "-s", - "-v", - "-R", - "-k", - key, - stdout=None, - ) - ).wait() - - -async def kill_gpg_agnet() -> None: - await (await asyncio.create_subprocess_exec("/usr/bin/gpg-connect-agent", 'killagent', '/bye')).wait() - - -async def main(remote_dest: str, repo_base_name: str, sign_key: str) -> None: - cwd = pathlib.Path(os.getcwd()).resolve() - remote_dest = pathlib.Path(remote_dest).resolve() - database_dest = cwd.joinpath(repo_base_name).resolve() - - for file in os.listdir(str(remote_dest)): - logger.debug("Copy file: %s", file) - shutil.copy(str(remote_dest.joinpath(file).joinpath()), str(cwd)) - - for file in os.listdir(str(remote_dest)): - if ".pkg.tar.zst" in file and not file.endswith(".sig"): - await add_package(database_dest, file, sign_key) - - await kill_gpg_agnet() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("update-db.py") - parser.add_argument("cwd") - parser.add_argument("basename") - parser.add_argument("sign_key") - - args_ = parser.parse_args() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - asyncio.run(main(args_.cwd, args_.basename, args_.sign_key)) diff --git a/utils/upload.php b/utils/upload.php deleted file mode 100644 index 602ed69..0000000 --- a/utils/upload.php +++ /dev/null @@ -1,36 +0,0 @@ -<?PHP - if (isset($_POST) && isset($_POST['token']) && - $_POST['token'] === 'be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09'; - ) { - if (!isset($_POST['arch'])) { - http_response_code(400); - echo "400 Bad Request: Should specify arch field while post files"; - die(); - } - if (!empty($_FILES['file'])) { - - $path = "/var/www/repo/". $_POST['arch'] . "/"; - $file_name = urldecode($_FILES['file']['name']); - $path = $path . basename($file_name); - - if (move_uploaded_file($_FILES['file']['tmp_name'], $path)) { - echo "The file " . basename($file_name) . " has been uploaded"; - } else { - echo "There was an error uploading the file, please try again!"; - } - } else if (isset($_POST['action'])) { - if ($_POST['action'] === 'UPLOADED') { - $sock = stream_socket_client('unix:///run/update-repo-notify.sock', $errno, $errst); - fwrite($sock, "ARCH=". $_POST['arch'] ."\r\n"); - fclose($sock); - echo "200 OK STARTED"; - } else if ($_POST['action'] === 'REQUIRE_CLEAN') { - shell_exec("/usr/bin/rm -rf /var/www/repo/". $_POST['arch'] . "*"); - shell_exec("/usr/bin/mkdir /var/www/repo/". $_POST['arch']); - echo "200 OK CLEANED"; - } - } - } else { - http_response_code(403); - } -?> diff --git a/utils/upload.py b/utils/upload.py deleted file mode 100755 index 2fabd7c..0000000 --- a/utils/upload.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -import argparse -import asyncio -import logging -import os - -import aiohttp -import aiohttp_socks - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -def build_data(args: argparse.Namespace, action: str) -> dict[str, str]: - return {"token": args.token, "arch": args.arch, "action": action} - - -# source: https://stackoverflow.com/a/1094933 -def sizeof_fmt(num: int, suffix: str = "B"): - for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: - if abs(num) < 1024.0: - return f"{num:3.1f}{unit}{suffix}" - num /= 1024.0 - return f"{num:.1f}Yi{suffix}" - - -# TODO: use upload in chunks to big file -async def send_file( - session: aiohttp.ClientSession, args: argparse.Namespace, file: str -) -> None: - data = aiohttp.FormData() - data.add_field("file", open(file, "rb"), filename=file) - data.add_field("token", args.token) - data.add_field("arch", args.arch) - async with session.post(args.remote_address, data=data) as req: - logger.debug("%s", (await req.text()).strip()) - - -async def main(args: argparse.Namespace) -> int: - pending_upload = [] - for file in os.listdir("."): - if ".pkg" in file: - pending_upload.append(file) - - logger.debug("Pending upload (%d): %s", len(pending_upload), pending_upload) - - if not len(pending_upload): - logger.info("Could not find any packaged packages, skip upload") - return 0 - - if proxy := os.getenv("http_proxy"): - logger.debug("Using proxy %s", proxy) - proxies = aiohttp_socks.ProxyConnector.from_url(proxy, rdns=False) - else: - proxies = None - - async with aiohttp.ClientSession( - raise_for_status=True, timeout=aiohttp.ClientTimeout(30), connector=proxies - ) as session: - async with session.post( - args.remote_address, data=build_data(args, "REQUIRE_CLEAN") - ): - pass - for file in pending_upload: - logger.info("Start upload %s (%s)", file, sizeof_fmt(os.stat(file).st_size)) - for retries in range(3): - try: - await send_file(session, args, file) - break - except asyncio.TimeoutError: - if retries == 2: - raise - async with session.post(args.remote_address, data=build_data(args, "UPLOADED")): - pass - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("upload.py") - parser.add_argument("remote_address") - parser.add_argument("token") - parser.add_argument("arch") - parser.add_argument("--directory") - args_ = parser.parse_args() - - try: - import coloredlogs - - coloredlogs.install( - logging.DEBUG, - fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - except ModuleNotFoundError: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s", - ) - - if args_.directory is not None: - os.chdir(args_.directory) - - exit(asyncio.run(main(args_))) |
