aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--utils/aur/AURUtils/__init__.py0
-rw-r--r--utils/aur/AURUtils/aur_deps.py9
-rw-r--r--utils/aur/AURUtils/errors.py16
-rw-r--r--utils/aur/AURUtils/rpc.py88
-rw-r--r--utils/aur/AURUtils/srcinfo.py97
-rw-r--r--utils/aur/AURUtils/types.py78
-rw-r--r--utils/aur/AURUtils/version.py5
-rwxr-xr-xutils/aur/check-aur-dependencies.py80
-rwxr-xr-xutils/aur/check-aur-update.py145
-rw-r--r--utils/aur/requirements.txt2
-rwxr-xr-xutils/deploy.sh18
-rwxr-xr-xutils/parse-pkgbuild.sh8
-rwxr-xr-xutils/pkgbuild.py566
-rwxr-xr-xutils/pkgbuild_bootstrap53
-rwxr-xr-xutils/pkgman.py61
-rw-r--r--utils/systemd/update-repo-copy.service8
-rw-r--r--utils/systemd/update-repo-notify.service5
-rw-r--r--utils/systemd/update-repo-notify.socket16
-rw-r--r--utils/systemd/[email protected]7
-rwxr-xr-xutils/systemd/wait-services.sh6
-rwxr-xr-xutils/trigger.py78
-rwxr-xr-xutils/trust_gpg.sh3
-rwxr-xr-xutils/update-db.py71
-rw-r--r--utils/upload.php36
-rwxr-xr-xutils/upload.py101
25 files changed, 0 insertions, 1557 deletions
diff --git a/utils/aur/AURUtils/__init__.py b/utils/aur/AURUtils/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/utils/aur/AURUtils/__init__.py
+++ /dev/null
diff --git a/utils/aur/AURUtils/aur_deps.py b/utils/aur/AURUtils/aur_deps.py
deleted file mode 100644
index 3a4bc44..0000000
--- a/utils/aur/AURUtils/aur_deps.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import re
-
-signs = [r'\>\=', r'\<\=', r'\=\=', r'\>', r'\<']
-signs_or = r'|'.join(signs)
-deps_pattern = re.compile(rf'({signs_or}).*')
-
-
-def depends_strip(deps: str) -> str:
- return deps_pattern.sub('', deps) \ No newline at end of file
diff --git a/utils/aur/AURUtils/errors.py b/utils/aur/AURUtils/errors.py
deleted file mode 100644
index 2e1fd87..0000000
--- a/utils/aur/AURUtils/errors.py
+++ /dev/null
@@ -1,16 +0,0 @@
-class RPCError(Exception):
-
- def __init__(self, method: str, message: str):
- self.method = method
- self.message = message
-
- def __str__(self):
- return f'Method({self.method}): {self.message}'
-
-
-class PackageNotFound(Exception):
- pass
-
-
-class MaxRetries(Exception):
- pass
diff --git a/utils/aur/AURUtils/rpc.py b/utils/aur/AURUtils/rpc.py
deleted file mode 100644
index 6120d41..0000000
--- a/utils/aur/AURUtils/rpc.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import asyncio
-import random
-
-import aiohttp
-
-from .errors import RPCError, MaxRetries
-from .types import Results, UrlParameterType
-
-DEFAULT_AURWEB_VERSION = 5
-MAX_WORKERS = 10
-
-METHODS = {'search', 'info', 'multiinfo', 'msearch', 'suggest',
- 'suggest-pkgbase', 'get-comment-form'}
-FIELDS = {'name', 'name-desc', 'maintainer', 'depends',
- 'makedepends', 'optdepends', 'checkdepends'}
-
-
-class AsyncRPC:
- """
- https://wiki.archlinux.org/title/Aurweb_RPC_interface
- """
-
- RPC_URL = 'https://aur.archlinux.org/rpc/'
-
- def __init__(self, max_workers: int = MAX_WORKERS):
- self._client = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10))
- self.sem = asyncio.Semaphore(max_workers)
-
- async def __aenter__(self):
- return self
-
- async def __aexit__(self, exc_type, exc_val, exc_tb):
- await self.close()
-
- async def close(self):
- await self._client.close()
-
- async def _post(self, params: dict) -> dict:
- async with self.sem:
- for i in range(5):
- try:
- async with self._client.get(self.RPC_URL, params=params) as resp:
- resp.raise_for_status()
- return await resp.json()
- except aiohttp.ServerTimeoutError:
- await asyncio.sleep(3 * random.random())
- continue
- raise MaxRetries
-
- async def request(self,
- method: str,
- method_params: dict[str, UrlParameterType],
- ):
- params = {
- 'v': DEFAULT_AURWEB_VERSION,
- 'type': method,
- **method_params
- }
- json_response = await self._post(params)
- if 'error' in json_response:
- raise RPCError(method, json_response['error'])
- return Results.parse(json_response)
-
- async def search(self,
- arg: UrlParameterType,
- search_by: str = None,
- ):
- method_params = {
- 'by': search_by or [],
- 'arg': arg,
- }
- return await self.request('search', method_params)
-
- async def info(self,
- arg: UrlParameterType,
- ):
- return await self.multiinfo(arg)
-
- async def multiinfo(self,
- arg: UrlParameterType,
- ):
- method_params = {
- 'arg[]': arg,
- }
- return await self.request('multiinfo', method_params)
-
- async def msearch(self, arg: UrlParameterType):
- return await self.search(arg, search_by='maintainer')
diff --git a/utils/aur/AURUtils/srcinfo.py b/utils/aur/AURUtils/srcinfo.py
deleted file mode 100644
index 34d2240..0000000
--- a/utils/aur/AURUtils/srcinfo.py
+++ /dev/null
@@ -1,97 +0,0 @@
-import re
-from collections import defaultdict
-from itertools import chain
-from typing import Optional, Union
-
-from .errors import PackageNotFound
-from .aur_deps import depends_strip
-
-key_value_pattern = re.compile(r'(?P<KEY>.+?)=(?P<VALUE>.+)')
-multivalued_attrs = {
- 'source', 'provides', 'conflicts', 'depends', 'replaces', 'optdepends', 'makedepends', 'checkdepends',
- 'md5sums', 'sha1sums', 'sha224sums', 'sha256sums', 'sha384sums', 'sha512sums', 'arch'
-}
-
-
-class SRCINFO:
-
- def __init__(self, pkgbase: str, pkgbase_opt: dict, pkgs_opt: dict):
- self.pkgbase = pkgbase
- self.pkgbase_opt = pkgbase_opt
- self.pkgs_opt = pkgs_opt
-
- @classmethod
- def parse(cls, source: str):
- pkgbase = None
- pkgbase_opt = defaultdict(list)
- pkgs_opt = {}
- handler = None
- for l in source.splitlines():
- m = key_value_pattern.search(l.strip())
- if m is None:
- continue
- key, value = m.group('KEY').strip(), m.group('VALUE').strip()
- if key == 'pkgbase':
- pkgbase = value
- handler = pkgbase_opt
- elif key == 'pkgname':
- pkgs_opt[value] = defaultdict(list)
- handler = pkgs_opt[value]
- else:
- if key in multivalued_attrs:
- handler[key].append(value)
- else:
- handler[key] = value
- return SRCINFO(pkgbase, pkgbase_opt, pkgs_opt)
-
- @property
- def pkgs(self) -> dict:
- return {
- k: {i: self.get_pkg_opt(k, i) for i in chain(self.pkgbase_opt.keys(), v.keys())}
- for k, v in self.pkgs_opt.items()
- }
-
- @property
- def pkgs_name(self) -> list:
- return list(self.pkgs_opt.keys())
-
- def _chain_pkg_opt(self, pkgname: str, opt: str) -> chain:
- if pkgname not in self.pkgs_name:
- raise PackageNotFound(f'{pkgname} not found in {self.pkgbase}')
- return chain(self.pkgbase_opt.get(opt, []), self.pkgs_opt[pkgname].get(opt, []))
-
- def _chain_opt(self, opt: str) -> chain:
- return chain(self.pkgbase_opt.get(opt, []), *(i.get(opt, []) for i in self.pkgs_opt.values()))
-
- def get_pkg_opt(self, pkgname: str, opt: str) -> Union[list, str]:
- if pkgname not in self.pkgs_name:
- raise PackageNotFound(f'{pkgname} not found in {self.pkgbase}')
- if opt in multivalued_attrs:
- return list(self._chain_pkg_opt(pkgname, opt))
- return self.pkgbase_opt.get(opt)
-
- def get_opt(self, opt: str) -> Union[list, str]:
- if opt in multivalued_attrs:
- return list(self._chain_opt(opt))
- return self.pkgbase_opt.get(opt)
-
- @property
- def depends(self) -> list[str]:
- return list(map(depends_strip, self._chain_opt('depends')))
-
- @property
- def makedepends(self) -> list[str]:
- return list(map(depends_strip, self._chain_opt('makedepends')))
-
- @property
- def optdepends(self) -> list[str]:
- return list(self._chain_opt('optdepends'))
-
- @property
- def version(self) -> tuple[Optional[str], str, str]:
- return self.pkgbase_opt.get('epoch', None), \
- self.pkgbase_opt['pkgver'], self.pkgbase_opt['pkgrel']
-
- def get_version(self) -> str:
- epoch = f"{self.pkgbase_opt['epoch']}:" if self.pkgbase_opt.get('epoch') else ''
- return f"{epoch}{self.pkgbase_opt['pkgver']}-{self.pkgbase_opt['pkgrel']}"
diff --git a/utils/aur/AURUtils/types.py b/utils/aur/AURUtils/types.py
deleted file mode 100644
index 9830b08..0000000
--- a/utils/aur/AURUtils/types.py
+++ /dev/null
@@ -1,78 +0,0 @@
-from dataclasses import dataclass
-from datetime import datetime
-from collections.abc import Iterator
-from typing import Union
-
-from .aur_deps import depends_strip
-
-UrlParameterType = Union[str, list[str]]
-
-
-@dataclass()
-class Package:
-
- ID: int
- Name: str
- PackageBaseID: int
- PackageBase: str
- Version: str
- Description: str
- URL: str
- NumVotes: int
- Popularity: float
- OutOfDate: int
- Maintainer: str
- FirstSubmitted: int
- LastModified: int
- URLPath: str
- Conflicts: list[str] = None
- Provides: list[str] = None
- Depends: list[str] = None
- MakeDepends: list[str] = None
- OptDepends: list[str] = None
- License: list[str] = None
- Keywords: list[str] = None
-
- @property
- def depends(self) -> list[str]:
- return list(map(depends_strip, self.Depends))
-
- @property
- def makedepends(self) -> list[str]:
- return list(map(depends_strip, self.MakeDepends))
-
- @property
- def optdepends(self) -> list[str]:
- return list(self.OptDepends)
-
- @property
- def first_submitted(self) -> datetime:
- return datetime.fromtimestamp(self.FirstSubmitted)
-
- @property
- def last_modified(self) -> datetime:
- return datetime.fromtimestamp(self.LastModified)
-
- def get_version(self) -> str:
- return self.Version
-
-
-@dataclass
-class Results:
-
- resultcount: int
- results: list
- type: str
- version: int
-
- @classmethod
- def parse(cls, json_response: dict):
- return Results(**json_response)
-
- def empty(self) -> bool:
- return self.resultcount == 0
-
- def packages(self) -> Iterator[Package]:
- if self.type in {'multiinfo', 'search'}:
- return [Package(**i) for i in self.results]
- return []
diff --git a/utils/aur/AURUtils/version.py b/utils/aur/AURUtils/version.py
deleted file mode 100644
index b3d5329..0000000
--- a/utils/aur/AURUtils/version.py
+++ /dev/null
@@ -1,5 +0,0 @@
-import pyalpm # type: ignore
-
-
-def vercmp(ver1: str, ver2: str) -> int:
- return pyalpm.vercmp(ver1, ver2) \ No newline at end of file
diff --git a/utils/aur/check-aur-dependencies.py b/utils/aur/check-aur-dependencies.py
deleted file mode 100755
index 38b6436..0000000
--- a/utils/aur/check-aur-dependencies.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (C) 2021 KunoiSayami and contributors
-#
-# This module is part of KunoiSayami/repos and is released under
-# the AGPL v3 License: https://www.gnu.org/licenses/agpl-3.0.txt
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import argparse
-import asyncio
-import heapq
-import logging
-import re
-from pathlib import Path
-
-import aiofiles
-import aiohttp
-
-
-async def check_pkg(session: aiohttp.ClientSession, num: int, pkg: str) -> tuple[int, bool]:
- logging.info(f'Checking {pkg}')
- ret = await session.head(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={pkg}')
- return num, ret.status == 200
-
-
-async def main() -> int:
- parser = argparse.ArgumentParser(description='check-aur-dependencies.py')
- parser.add_argument('paths', metavar='sub_path', type=str, nargs='*', help='sub-path for repo root')
-
- args = parser.parse_args()
- repo_dir = Path(*args.paths)
- pkgbuild_file = repo_dir.joinpath('PKGBUILD')
- if not pkgbuild_file.exists():
- logging.error("Can't locate PKGBUILD file, please specify directory in arguments or chdir to folder")
- return 1
-
- async with aiofiles.open(pkgbuild_file, 'r', encoding='utf-8') as f:
- pkgbuild_text = await f.read()
-
- runtime_deps_match = re.search(r"depends=\((?P<DEPS>([^)]|\n)*)\)", pkgbuild_text, re.M)
- if not runtime_deps_match:
- logging.warning("Can't found 'depends' keyword, if you think this is mistake, please report issue")
-
- make_deps_match = re.search(r"makedepends=\((?P<DEPS>([^)]|\n)*)\)", pkgbuild_text, re.M)
- if not make_deps_match:
- logging.warning("Can't found `makedepends' keyword, if you think this is mistake, please report issue")
-
- deps = [dep.strip('\'') for m in {runtime_deps_match, make_deps_match} if m for dep in m.group('DEPS').split()]
-
- async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session:
- tasks = [asyncio.create_task(check_pkg(session, n, pkg)) for n, pkg in enumerate(deps)]
- yay_dep_nums = []
- for task in asyncio.as_completed(tasks):
- n, r = await task
- if r:
- heapq.heappush(yay_dep_nums, n)
-
- yaydeps_dir = repo_dir.parent.joinpath('.yaydeps')
-
- if len(yay_dep_nums) > 0:
- async with aiofiles.open(yaydeps_dir.joinpath(repo_dir.stem), 'w', encoding='utf-8') as f:
- await f.write('\n'.join([*(deps[heapq.heappop(yay_dep_nums)] for _ in range(len(yay_dep_nums))), '']))
-
- return 0
-
-
-if __name__ == '__main__':
- logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s')
- exit(asyncio.run(main()))
diff --git a/utils/aur/check-aur-update.py b/utils/aur/check-aur-update.py
deleted file mode 100755
index 4be4e09..0000000
--- a/utils/aur/check-aur-update.py
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-
-import asyncio
-import argparse
-import heapq
-import logging
-from collections.abc import Coroutine
-from pathlib import Path
-from typing import Optional
-
-import aiohttp
-import aiofiles
-
-from AURUtils.srcinfo import SRCINFO
-from AURUtils.version import vercmp
-
-
-MAX_WORKERS = 16
-
-
-async def check_pkg(session: aiohttp.ClientSession, num: int, pkg: str) -> tuple[int, bool]:
- logging.info(f'Checking {pkg}')
- ret = await session.head(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={pkg}')
- return num, ret.status == 200
-
-
-async def check_aur_update(
- session: aiohttp.ClientSession,
- item: Path, dry_run: bool,
- parse_pkgbuild: Optional[Path] = None,
- check_deps: bool = True
-) -> None:
- if item.name.startswith('.') or not item.is_dir():
- return
- pkgbuild_file = item.joinpath('PKGBUILD')
- if not pkgbuild_file.exists():
- logging.warning(f'PKGBUILD not in {str(item)}, SKIP!')
- return
- srcinfo_file = item.joinpath('.SRCINFO')
- if not srcinfo_file.exists():
- if parse_pkgbuild.exists():
- logging.warning(f'.SRCINFO not in {str(item)}, try to parse')
- proc = await asyncio.create_subprocess_exec(
- f'{str(parse_pkgbuild.resolve())}',
- f'{str(pkgbuild_file)}',
- stdout=asyncio.subprocess.PIPE,
- )
- srcinfo = SRCINFO.parse((await proc.stdout.read()).decode())
- logging.info(f'extract .SRCINFO from {srcinfo.pkgbase}')
- else:
- logging.warning(f'.SRCINFO not in {str(item)}, no tool to parse')
- return
- else:
- async with aiofiles.open(srcinfo_file, 'r') as f:
- srcinfo = SRCINFO.parse(await f.read())
- version = srcinfo.get_version()
- async with session.get(f'https://aur.archlinux.org/cgit/aur.git/plain/.SRCINFO?h={srcinfo.pkgbase}') as resp:
- if resp.status != 200:
- logging.info(f'{item.name} not register in AUR')
- return
- new_srcinfo = SRCINFO.parse(await resp.text())
- new_version = new_srcinfo.get_version()
- result = vercmp(new_version, version)
- if result == 0:
- return
- elif result < 0:
- logging.warning(f'Warning: {item.name}({version}) is newer than AUR')
- else:
- if item.joinpath('.git').exists():
- if not dry_run:
- await asyncio.create_subprocess_exec('git', '-C', f'{str(item)}', 'pull', 'origin', 'master')
- logging.info(f'Upgrade {item.name} from {version} to {new_version}')
- else:
- logging.info(f'Found update {item.name}({new_version}) (local: {version})')
- if check_deps:
- deps = [*new_srcinfo.makedepends, *new_srcinfo.depends]
- async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session:
- tasks = [asyncio.create_task(check_pkg(session, n, pkg)) for n, pkg in enumerate(deps)]
- yay_dep_nums = []
- for task in asyncio.as_completed(tasks):
- n, r = await task
- if r:
- heapq.heappush(yay_dep_nums, n)
-
- yaydeps_dir = item.parent.joinpath('.yaydeps')
-
- if len(yay_dep_nums) > 0:
- async with aiofiles.open(yaydeps_dir.joinpath(item.stem), 'w', encoding='utf-8') as f:
- await f.write(
- '\n'.join([*(deps[heapq.heappop(yay_dep_nums)] for _ in range(len(yay_dep_nums))), '']))
-
-
-async def with_sem(sem: asyncio.Semaphore, coro: Coroutine):
- async with sem:
- return await coro
-
-
-async def main() -> int:
- parser = argparse.ArgumentParser(description='aur-check-update.py')
- parser.add_argument('--dry-run', dest='dry', help='dry run', action='store_true')
- parser.add_argument('--check-deps', dest='deps', help='check deps', action='store_true')
- parser.add_argument('--parse-pkgbuild', dest='script', metavar='SCRIPT',
- help='a script to parse pkgbuild',
- action='store')
- parser.add_argument('--miax-workers', dest='workers', metavar='MAX_WORKERS',
- help=f'aur check max workers, default: {MAX_WORKERS}',
- default=MAX_WORKERS, action='store', type=int)
- parser.add_argument(dest='repo_dir', metavar='REPO_DIR', help='repository directory', nargs='?')
-
- args = parser.parse_args()
-
- dry_run = args.dry
- check_deps = args.deps
- sem = asyncio.Semaphore(args.workers)
-
- if args.repo_dir is None:
- logging.error('repository directory not found')
- return 1
-
- script_file = Path(args.script)
- if args.script is None or not (script_file.is_file() and script_file.exists()):
- logging.info(f'Parse script {str(script_file)+" " if args.script is None else ""} not found')
- script_file = None
-
- repo_dir = Path(args.repo_dir)
-
- async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session:
- tasks = [asyncio.create_task(
- with_sem(sem, check_aur_update(
- session=session,
- item=item,
- dry_run=dry_run,
- parse_pkgbuild=script_file,
- check_deps=check_deps
- ))
- ) for item in repo_dir.iterdir()]
- await asyncio.gather(*tasks)
-
- return 0
-
-
-if __name__ == '__main__':
- logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s')
- asyncio.run(main())
diff --git a/utils/aur/requirements.txt b/utils/aur/requirements.txt
deleted file mode 100644
index 785d447..0000000
--- a/utils/aur/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-aiofiles~=0.7.0
-aiohttp~=3.8.0
diff --git a/utils/deploy.sh b/utils/deploy.sh
deleted file mode 100755
index bf2c695..0000000
--- a/utils/deploy.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# end up when failed
-set -Eeuo pipefail
-
-if [ $# -eq 0 ]; then
- echo 'Should provide least one argument'
- exit 1
-fi
-
-if [ -z ${SSH_CLIENT+x} ]; then
- ADDITIONAL_PARAM="--partial"
-else
- ADDITIONAL_PARAM="--partial -v"
-fi
-
-/usr/bin/date +%s > "packages/LASTSYNC"
-/usr/bin/rsync $ADDITIONAL_PARAM -r -c --update --links --delete --exclude=.gitignore "packages/" $1:/var/www/repo/
diff --git a/utils/parse-pkgbuild.sh b/utils/parse-pkgbuild.sh
deleted file mode 100755
index c0da48a..0000000
--- a/utils/parse-pkgbuild.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-# no makepkg version
-# https://github.com/archlinuxcn/repo/blob/master/parse-pkgbuild
-
-pushd $(dirname $@) > /dev/null
-makepkg --printsrcinfo
-popd > /dev/null
diff --git a/utils/pkgbuild.py b/utils/pkgbuild.py
deleted file mode 100755
index 5dfbd6a..0000000
--- a/utils/pkgbuild.py
+++ /dev/null
@@ -1,566 +0,0 @@
-#!/usr/bin/env python
-from __future__ import annotations
-import argparse
-import asyncio
-import dataclasses
-import lzma
-import logging
-import operator
-import os
-import pathlib
-import shutil
-import subprocess
-import time
-
-import aiofiles
-
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
-
-PKGBUILD_DIRECTORY_BASE = "repo"
-
-CI_COMMIT_BRANCH = os.getenv("CI_COMMIT_BRANCH", "")
-CI_DEFAULT_BRANCH = os.getenv("CI_DEFAULT_BRANCH", "")
-CI_COMMIT_TITLE = os.getenv("CI_COMMIT_TITLE", "")
-BUILD_OVERRIDE = os.getenv("BUILD_OVERRIDE", "")
-TRUST_GPG_SCRIPT = pathlib.Path("./utils/trust_gpg.sh").resolve()
-NO_UPLOAD = os.getenv("BUILD_NO_UPLOAD") is None
-FAIL_REPOS = []
-
-
-async def get_arch() -> str:
- p = await asyncio.create_subprocess_exec("uname", "-m", stdout=subprocess.PIPE)
- stdout, _ = await p.communicate()
- return stdout.decode().lower().strip()
-
-
-ARCH = asyncio.run(get_arch())
-
-
-class PackageVersionWithPath:
- pkg_version: PackageVersion
- path: pathlib.Path
- # For debug use
- spend_time: float
-
- @property
- def name(self) -> str:
- return self.pkg_version.name
-
- @property
- def version(self) -> str:
- return self.pkg_version.version
-
- @property
- def arch(self) -> bool:
- return self.pkg_version.arch_match
-
- def __str__(self) -> str:
- if self.spend_time < 0:
- return self.pkg_version.__str__()
- return f"{self.pkg_version.__str__()} (spend: {self.spend_time:.2f}s)"
-
-
-class PackageVersion:
- name: str
- version: str
- arch_match: bool
-
- def __eq__(self, other: PackageVersion) -> bool:
- return other.version == self.version
-
- def __gt__(self, other: PackageVersion) -> bool:
- return self.version > other.version
-
- def __lt__(self, other: PackageVersion) -> bool:
- return self.version < other.version
-
- def __le__(self, other: PackageVersion) -> bool:
- return self < other or self == other
-
- def __ge__(self, other: PackageVersion) -> bool:
- return self > other or self == other
-
- def __ne__(self, other: PackageVersion) -> bool:
- return not self == other
-
- def __str__(self) -> str:
- return f"{self.name} {self.version}"
-
-
-class MakepkgErrorReason:
- package: str
- code: int
- post_install_dependency: str | None
-
- def to_str(self) -> str:
- if self.post_install_dependency is not None:
- return f"{self.package}(dependency)[{self.post_install_dependency}]({self.code})"
- else:
- return f"{self.package}({self.code})"
-
-
-def fetch_database(pkg_db: str) -> dict[str, str]:
- name_read = False
- version_read = False
- pkg_name = ""
- package_storage = {}
- with lzma.open(pkg_db) as fin:
- for line in fin:
- line = line.strip()
- if name_read:
- pkg_name = line.decode()
- name_read = False
- elif version_read:
- package_storage.update({pkg_name: line.decode()})
- # logger.debug("%s %s", pkg_name, line.decode())
- version_read = False
- pkg_name = ""
- if line == b"%NAME%":
- assert not name_read
- name_read = True
- if line == b"%VERSION%":
- assert not version_read
- version_read = True
- assert (name_read is False and version_read is False) or (
- version_read != name_read
- )
- return package_storage
-
-
[email protected](init=False)
-class BuildEnvs:
- src_dest: str
- pkg_dest: str
- makepkg_conf: str
- signing_arg: list[str]
-
- def __init__(self):
- cwd = pathlib.Path(os.getcwd())
- self.pkg_dest = str(cwd.joinpath("packages", ARCH).resolve())
- self.src_dest = str(cwd.joinpath("build").resolve())
- self.makepkg_conf = str(cwd.joinpath("makepkg_current.conf").resolve())
- if CI_COMMIT_BRANCH == CI_DEFAULT_BRANCH and CI_COMMIT_BRANCH != "":
- self.signing_arg = ["--sign"]
- else:
- logger.info("Skip signing package")
-
- def get_dict(self) -> dict[str, str | list[str]]:
- return {
- "src_dest": self.src_dest,
- "pkg_dest": self.pkg_dest,
- "makepkg_conf": self.makepkg_conf,
- "signing_arg": self.signing_arg,
- }
-
-
-def get_src_info(s: str) -> PackageVersion:
- base = ""
- ver = ""
- rel = ""
- arch = []
- for line in s.splitlines(False):
- if (line := line.strip()).startswith("pkgver"):
- ver = line.split("=", maxsplit=2)[1]
- elif line.startswith("pkgrel"):
- rel = line.split("=", maxsplit=2)[1]
- elif line.startswith("pkgbase"):
- base = line.split("=", maxsplit=2)[1]
- elif line.startswith("arch"):
- arch.append(line.split("=", maxsplit=2)[1].strip().lower())
- return PackageVersion(
- base.strip(), f"{ver.strip()}-{rel.strip()}", ARCH in arch or "any" in arch
- )
-
-
-async def parse_srcinfo(path: pathlib.PurePath) -> PackageVersionWithPath:
- if (src_path := pathlib.Path(path.joinpath(".SRCINFO"))).is_file():
- async with aiofiles.open(str(src_path)) as fin:
- info = get_src_info(await fin.read())
- spend_time = -1.0
- else:
- start_time = time.time()
- p = await asyncio.create_subprocess_exec(
- "makepkg", "--printsrcinfo", cwd=src_path.parent, stdout=subprocess.PIPE
- )
- (stdout, stderr) = await p.communicate()
- info = get_src_info(stdout.decode())
- spend_time = time.time() - start_time
- return PackageVersionWithPath(info, pathlib.Path(path).resolve(), spend_time)
-
-
-async def fetch_packages_from_directory(
- repo_directory: pathlib.Path,
-) -> set[asyncio.Task[PackageVersionWithPath]]:
- tasks = []
-
- for folder in os.listdir(str(repo_directory)):
- if folder.startswith("."):
- continue
-
- if (pkg_dir := repo_directory.joinpath(folder).resolve()).is_file():
- continue
-
- tasks.append(asyncio.create_task(parse_srcinfo(pkg_dir)))
-
- finished, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
- return finished
-
-
-async def get_build_target(
- pkg_db: str,
- home_directory: pathlib.Path,
- build_target: list[str] | None,
- build_all: bool,
-) -> list[PackageVersionWithPath]:
- repo = fetch_database(pkg_db)
- pending_build = []
-
- finished = await fetch_packages_from_directory(
- home_directory.joinpath(PKGBUILD_DIRECTORY_BASE)
- )
-
- if len(BUILD_OVERRIDE):
- logger.warning("BUILD OVERRIDE: %s", BUILD_OVERRIDE)
- override = BUILD_OVERRIDE.split(";")
- return [
- future.result() for future in finished if future.result().name in override
- ]
-
- if build_all:
- return [future.result() for future in finished]
-
- if len(build_target):
- return [
- future.result()
- for future in finished
- if future.result().name in build_target
- ]
-
- for future in finished:
- pkg = future.result()
- if pkg.name in repo:
- # print(pkg.name, repo[pkg.name], pkg.version)
- if repo[pkg.name] < pkg.version:
- pending_build.append(pkg)
- else:
- pending_build.append(pkg)
-
- # print(pending_build)
- return pending_build
-
-
-async def run_hook(origin_path: pathlib.Path, target: str) -> None:
- if (
- hook_script := origin_path.parent.joinpath(".hook").joinpath(target).resolve()
- ).is_file():
- logger.info("Running %s hook script", target)
- p = await asyncio.create_subprocess_exec(
- str(hook_script), stdout=None, stderr=subprocess.STDOUT
- )
- await p.wait()
- if p.returncode != 0:
- logger.error(
- "Hook script return code: %d, skip other operation", p.returncode
- )
- return
-
-
-async def run_build_with_install(
- src_dest: str, pkg_dest: str, makepkg_conf: str, signing_arg: list[str]
-) -> int:
- return await run_build(src_dest, pkg_dest, makepkg_conf, signing_arg, "-i")
-
-
-async def run_build(
- src_dest: str,
- pkg_dest: str,
- makepkg_conf: str,
- signing_arg: list[str],
- *other_args: str,
-) -> int:
- env = os.environ
- env.update(
- {
- "SRCPKGDEST": src_dest,
- "SRCDEST": src_dest,
- "PKGDEST": pkg_dest,
- "MAKEPKG_CONF": makepkg_conf,
- }
- )
- p = await asyncio.create_subprocess_exec(
- "makepkg",
- "--clean",
- "-s",
- *other_args,
- *signing_arg,
- "--asdeps",
- "--noconfirm",
- "--needed",
- "--noprogressbar",
- env=env,
- stdout=None,
- )
- await p.wait()
- return p.returncode
-
-
-async def install_dependency_via_yay(dependency: str) -> int:
- await (
- p := await asyncio.create_subprocess_exec(
- "yay",
- "--noconfirm",
- "--asdeps",
- "--needed",
- "--noprogressbar",
- "-S",
- dependency,
- stdout=None,
- )
- ).wait()
- return p.returncode
-
-
-async def build_yay_dependency(target: PackageVersionWithPath, dependency: str) -> int:
- if (dep_dir := target.path.parent.joinpath(dependency).resolve()).is_dir():
- os.chdir(str(dep_dir))
- await run_hook(target.path, dependency)
- return await run_build_with_install(**BUILD_ENVS.get_dict())
- else:
- return await install_dependency_via_yay(dependency)
-
-
-async def do_build(target: PackageVersionWithPath) -> int:
- try:
- os.chdir(str(target.path))
- logger.info("Building %s (%s)", target.name, target.path.name)
- except OSError:
- logger.exception("Got exception while chdir to %s", target.path)
- raise
-
- base_dir_name = target.path.name
-
- await run_hook(target.path, base_dir_name)
-
- if (
- yay_deps := target.path.parent.joinpath(".yaydeps")
- .joinpath(base_dir_name)
- .resolve()
- ).is_file():
- async with aiofiles.open(str(yay_deps)) as fin:
- while dep := (await fin.readline()).strip():
- logger.debug("Find dependency %s, build first", dep)
- if ret := await build_yay_dependency(target, dep):
- logger.error(
- "Build dependencies package error, skipped next step (%d)",
- ret,
- )
- FAIL_REPOS.append(MakepkgErrorReason(target.name, ret, dep))
- return ret
-
- if (
- gpg_keys := target.path.parent.joinpath(".gpg_keys")
- .joinpath(base_dir_name)
- .resolve()
- ).is_file():
- await (
- await asyncio.create_subprocess_exec(
- "/bin/bash", str(gpg_keys), stdout=None
- )
- ).wait()
- await (
- await asyncio.create_subprocess_exec(str(TRUST_GPG_SCRIPT), stdout=None)
- ).wait()
-
- os.chdir(str(target.path))
- build_ret = await run_build(**BUILD_ENVS.get_dict())
-
- if build_ret != 0:
- if build_ret == 13:
- logger.warning("Already build %s, skipped.", base_dir_name)
- return 0
- FAIL_REPOS.append(MakepkgErrorReason(base_dir_name, build_ret, None))
- logger.warning("Build %s fail: %d", base_dir_name, build_ret)
-
- return build_ret
-
-
-async def clean_installed_packages() -> int:
- p = await asyncio.create_subprocess_exec(
- "/bin/bash",
- "-c",
- "pacman -Qdtq | ifne sudo pacman --noconfirm -Rcns -",
- stdout=None,
- )
- await p.wait()
- return p.returncode
-
-
-async def upload_packages() -> int:
- remote_path = os.getenv("REMOTE_PATH")
- upload_token = os.getenv("UPLOAD_TOKEN")
- if remote_path is None or upload_token is None:
- logger.error("$REMOTE_PATH OR $UPLOAD_TOKEN is None, skipped upload")
- await (
- p := await asyncio.create_subprocess_exec(
- "./utils/upload.py",
- remote_path,
- upload_token,
- ARCH,
- "--directory",
- BUILD_ENVS.pkg_dest,
- stdout=None,
- )
- ).wait()
- return p.returncode
-
-
-async def do_work(
- pkg_db: str,
- build_all: bool,
- build_target: list[str],
- fail_fast: bool = False,
- run_auto_remove: bool = True,
- dry_run: bool = False,
-) -> int:
- logger.info(
- "build_all: %s, fail_fast: %s, auto_remove: %s, dry_run: %s",
- build_all,
- fail_fast,
- run_auto_remove,
- dry_run,
- )
-
- home_directory = pathlib.Path(os.getcwd())
-
- build_target = sorted(
- await get_build_target(pkg_db, home_directory, build_target, build_all),
- key=operator.attrgetter("name"),
- )
- logger.debug(
- "Pending build summary (%d): %s",
- len(build_target),
- ", ".join(map(lambda x: x.name, build_target)),
- )
-
- for target in build_target:
- if not target.arch:
- logger.info("Skipped %s (Architecture not match)", target.name)
- continue
- if dry_run:
- logger.info("Skipped %s (--dry-run specified)", target.name)
- continue
-
- if (await do_build(target)) != 0:
- if fail_fast:
- break
- else:
- await asyncio.sleep(3)
-
- if run_auto_remove:
- await clean_installed_packages()
- for _ in range(3):
- logger.debug("---------------------------------------------------------")
-
- if dry_run:
- return 0
-
- if (src_dst := pathlib.Path(BUILD_ENVS.src_dest).resolve()).is_dir():
- shutil.rmtree(str(src_dst), ignore_errors=True)
-
- # Override last build timestamp
- async with aiofiles.open(
- str(pathlib.Path(BUILD_ENVS.pkg_dest).joinpath("LASTBUILD").resolve()), "w"
- ) as fout:
- await fout.write(f"{int(time.time())}")
-
- # Print all failed repos
- if len(FAIL_REPOS):
- logger.error("Build failed repositories (%d):", len(FAIL_REPOS))
- logger.error(
- "%s",
- ", ".join(repo.to_str() for repo in FAIL_REPOS),
- )
-
- os.chdir(str(home_directory))
- if ret := await upload_packages():
- logger.error("Got error in upload packages (%d)", ret)
- return ret
-
- if len(FAIL_REPOS):
- return 1
-
-
-def check_build_target(args: argparse.Namespace) -> bool:
- if args.build_all or len(args.build_target):
- return True
- if "REBUILD ALL" in CI_COMMIT_TITLE or "REBUILD_ALL" in CI_COMMIT_TITLE:
- return True
- return False
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(
- description="pkgbuild.py",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
- )
- parser.add_argument("db_file", help="Remote database file")
- parser.add_argument(
- "--build-all", help="Force build all package", action="store_true"
- )
- parser.add_argument(
- "--build-target", nargs="*", default=list(), help="Override build targets"
- )
- parser.add_argument(
- "--sign", default=False, action="store_true", help="Sign package after build"
- )
- parser.add_argument(
- "--fail-fast",
- default=False,
- action="store_true",
- help="If build failed, exit script",
- )
- parser.add_argument(
- "--disable-auto-remove",
- default=False,
- action="store_true",
- help="Disable auto remove after build package",
- )
- parser.add_argument(
- "--dry-run",
- default=False,
- action="store_true",
- help="Run but not actually build package",
- )
- args_ = parser.parse_args()
-
- try:
- import coloredlogs
-
- coloredlogs.install(
- logging.DEBUG,
- fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
- except ModuleNotFoundError:
- logging.basicConfig(
- level=logging.DEBUG,
- format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
-
- BUILD_ENVS = BuildEnvs()
- if args_.sign:
- BUILD_ENVS.signing_arg = ["--sign"]
- ret_ = asyncio.run(
- do_work(
- args_.db_file,
- check_build_target(args_),
- args_.build_target,
- args_.fail_fast,
- args_.disable_auto_remove,
- args_.dry_run,
- )
- )
- exit(ret_)
diff --git a/utils/pkgbuild_bootstrap b/utils/pkgbuild_bootstrap
deleted file mode 100755
index e0d23c6..0000000
--- a/utils/pkgbuild_bootstrap
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-
-if [ -z "$DOCKER_SETUP_SCRIPT" ]; then
- echo $GPG_PRIV_KEY | base64 -d | gpg --import
- cd "$(dirname "$0")"
- cd ../../
- echo "Current pwd: $(pwd)"
- sudo pacman -Sy
-fi
-
-set -Eeo pipefail
-trap cleanup SIGINT SIGTERM ERR EXIT
-
-cleanup() {
- trap - SIGINT SIGTERM ERR EXIT
- unset SRCDEST
- unset CONFDEST
- unset TMPCONF
- unset FLAGS
- unset ARCH
-}
-
-ARCH=$(uname -m)
-CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf"
-
-TMPCONF="makepkg_current.conf"
-cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF"
-cat "$CONFDEST" >> "$TMPCONF"
-
-FLAGS=""
-
-if [ "$EUID" -eq 0 ] ; then
- echo "\033[0;32mPlease run this script as non-root\033[0m"
- exit 3
-fi
-
-if [ -z "$REMOTE_DATABASE" ]; then
- echo "\033[0;32mYou should specify remote address\033[0m"
- exit 4
-fi
-
-if [ -n "$FORCE_BUILD_ALL" ]; then
- FLAGS+="--build-all "
-fi
-
-if [ -n "$BUILD_FAIL_FAST" ]; then
- FLAGS+="--fail-fast "
-fi
-
-curl -fL -o remote.db "$REMOTE_DATABASE"
-
-
-./utils/pkgbuild.py remote.db $FLAGS
diff --git a/utils/pkgman.py b/utils/pkgman.py
deleted file mode 100755
index 0e7388f..0000000
--- a/utils/pkgman.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import asyncio
-import os
-import pathlib
-
-try:
- import pkgbuild
-except ModuleNotFoundError:
- import utils.pkgbuild as pkgbuild
-
-
-async def fetch_and_get(_remote_addr: str):
- raise NotImplementedError("TODO!")
-
-
-async def show_diff(database: str, path: str) -> None:
- pkg_dir = pathlib.Path(os.getcwd()).joinpath(path).resolve()
- database_result = pkgbuild.fetch_database(database)
- pending = []
- for package in await pkgbuild.fetch_packages_from_directory(pkg_dir):
- if (package := package.result()).name in database_result:
- if database_result[package.name] < package.version:
- pending.append(package)
- else:
- pending.append(package)
- for package in pending:
- print(package.name, package.version)
-
-
-async def main(args: argparse.Namespace):
- if args.sub == "show":
- print(pkgbuild.fetch_database(args.database))
- elif args.sub == "show_local":
- pkg_dir = pathlib.Path(os.getcwd()).joinpath(args.path).resolve()
- for package in await pkgbuild.fetch_packages_from_directory(pkg_dir):
- print(package.result())
- elif args.sub == "show_internet":
- await fetch_and_get(args.database)
- elif args.sub == 'diff':
- await show_diff(args.database, args.path)
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser('pkgman.py')
- sub = parser.add_subparsers(title="subcommand", dest="sub")
- decmp_parser = sub.add_parser("show", description="Show databases")
- decmp_parser.add_argument("database")
-
- local_show_parser = sub.add_parser("show_local")
- local_show_parser.add_argument("path", default="./repo")
-
- decmp_online_parser = sub.add_parser("show_internet", description="Show remote databases")
- decmp_online_parser.add_argument("database")
-
- show_diff_parser = sub.add_parser("diff")
- show_diff_parser.add_argument("database")
- show_diff_parser.add_argument("path")
-
- args_ = parser.parse_args()
- asyncio.run(main(args_))
diff --git a/utils/systemd/update-repo-copy.service b/utils/systemd/update-repo-copy.service
deleted file mode 100644
index af79ee5..0000000
--- a/utils/systemd/update-repo-copy.service
+++ /dev/null
@@ -1,8 +0,0 @@
-[Unit]
-Description=Update repo copy service
-
-[Service]
-WorkingDirectory=/var/lib/update/packages/
-ExecStartPre=/var/lib/update/wait-services.sh
-ExecStart=/usr/bin/rsync -r --links -vct --update --delete . /var/www/packages \ No newline at end of file
diff --git a/utils/systemd/update-repo-notify.service b/utils/systemd/update-repo-notify.service
deleted file mode 100644
index d7e3c9a..0000000
--- a/utils/systemd/update-repo-notify.service
+++ /dev/null
@@ -1,5 +0,0 @@
-[Unit]
-Description=Update repo notify service
-
-[Service]
-ExecStart=/var/lib/update/trigger.py \ No newline at end of file
diff --git a/utils/systemd/update-repo-notify.socket b/utils/systemd/update-repo-notify.socket
deleted file mode 100644
index f14743c..0000000
--- a/utils/systemd/update-repo-notify.socket
+++ /dev/null
@@ -1,16 +0,0 @@
-[Unit]
-Description=Update repo notify socket
-
-[Socket]
-# If /var/run is not implemented as a symlink to /run, you may need to
-# specify ListenStream=/var/run/docker.sock instead.
-ListenStream=/run/update-repo-notify.sock
-SocketMode=0660
-SocketUser=update-database
-SocketGroup=http
-#Accept=yes
-MaxConnections=1
-FlushPending=yes
-
-[Install]
-WantedBy=sockets.target \ No newline at end of file
diff --git a/utils/systemd/[email protected] b/utils/systemd/[email protected]
deleted file mode 100644
index 6fa98da..0000000
--- a/utils/systemd/[email protected]
+++ /dev/null
@@ -1,7 +0,0 @@
-[Unit]
-Description=Update repo %i service
-
-[Service]
-User=update-database
-ExecStart=/var/lib/update/update-db.py /var/lib/update %i AAAABBBBCCCCDDDDDDDDCCCCBBBBAAAA
-#ExecStart=rsync \ No newline at end of file
diff --git a/utils/systemd/wait-services.sh b/utils/systemd/wait-services.sh
deleted file mode 100755
index be20239..0000000
--- a/utils/systemd/wait-services.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-echo "Checking is service running"
-while /usr/bin/systemctl status --no-page | grep update-repo@ | grep -v grep;
-do
- sleep 1
-done \ No newline at end of file
diff --git a/utils/trigger.py b/utils/trigger.py
deleted file mode 100755
index ca0cfd1..0000000
--- a/utils/trigger.py
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import logging
-import os
-import signal
-import socket
-import subprocess
-import threading
-from socketserver import StreamRequestHandler, TCPServer
-
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
-
-
-class Handler(StreamRequestHandler):
- def handle(self):
- while line := self.rfile.readline():
- if b"ARCH=" in line:
- (
- subprocess.Popen(
- [
- "systemctl",
- "--no-block",
- "start",
- f"update-repo@{line.split(b'=')[1].decode().strip()}",
- ]
- )
- ).wait()
- os.kill(os.getpid(), signal.SIGUSR1)
- return
- logger.info("Recv: %s", line)
-
-
-# source: https://gist.github.com/drmalex07/333d8a88c4918954e8e4
-class Server(TCPServer):
- # The constant would be better initialized by a systemd module
- SYSTEMD_FIRST_SOCKET_FD = 3
-
- def __init__(self, server_address, handler_cls):
- # Invoke base but omit bind/listen steps (performed by systemd activation!)
- TCPServer.__init__(self, server_address, handler_cls, bind_and_activate=False)
- # Override socket
- self.socket = socket.fromfd(
- self.SYSTEMD_FIRST_SOCKET_FD, self.address_family, self.socket_type
- )
-
-
-def main():
- def server_shutdown_wrapper(*_args) -> None:
- logger.debug("Shutdown the server")
- threading.Thread(
- target=server.shutdown,
- daemon=True,
- ).start()
-
- server = Server(("localhost", 0), Handler)
- signal.signal(signal.SIGUSR1, server_shutdown_wrapper)
-
- server.serve_forever()
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
-
- try:
- import coloredlogs
-
- coloredlogs.install(
- logging.DEBUG,
- fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
- except ModuleNotFoundError:
- logging.basicConfig(
- level=logging.DEBUG,
- format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
-
- main()
diff --git a/utils/trust_gpg.sh b/utils/trust_gpg.sh
deleted file mode 100755
index 5d4f8c3..0000000
--- a/utils/trust_gpg.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-gpg --list-keys --fingerprint | grep pub -A 1 | grep -Ev "pub|--" | tr -d ' ' \
- | awk 'BEGIN { FS = "\n" } ; { print $1":6:" } ' | gpg --import-ownertrust \ No newline at end of file
diff --git a/utils/update-db.py b/utils/update-db.py
deleted file mode 100755
index 3e318f5..0000000
--- a/utils/update-db.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import asyncio
-import logging
-import os
-import pathlib
-import shutil
-
-
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
-
-
-async def add_package(dest: pathlib.Path, pkg_name: str, key: str) -> None:
- # logger.debug("%s %s %s", dest, pkg_name, key)
- await (
- await asyncio.create_subprocess_exec(
- "/usr/bin/repo-add",
- str(dest),
- pkg_name,
- "-s",
- "-v",
- "-R",
- "-k",
- key,
- stdout=None,
- )
- ).wait()
-
-
-async def kill_gpg_agnet() -> None:
- await (await asyncio.create_subprocess_exec("/usr/bin/gpg-connect-agent", 'killagent', '/bye')).wait()
-
-
-async def main(remote_dest: str, repo_base_name: str, sign_key: str) -> None:
- cwd = pathlib.Path(os.getcwd()).resolve()
- remote_dest = pathlib.Path(remote_dest).resolve()
- database_dest = cwd.joinpath(repo_base_name).resolve()
-
- for file in os.listdir(str(remote_dest)):
- logger.debug("Copy file: %s", file)
- shutil.copy(str(remote_dest.joinpath(file).joinpath()), str(cwd))
-
- for file in os.listdir(str(remote_dest)):
- if ".pkg.tar.zst" in file and not file.endswith(".sig"):
- await add_package(database_dest, file, sign_key)
-
- await kill_gpg_agnet()
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser("update-db.py")
- parser.add_argument("cwd")
- parser.add_argument("basename")
- parser.add_argument("sign_key")
-
- args_ = parser.parse_args()
-
- try:
- import coloredlogs
-
- coloredlogs.install(
- logging.DEBUG,
- fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
- except ModuleNotFoundError:
- logging.basicConfig(
- level=logging.DEBUG,
- format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
- asyncio.run(main(args_.cwd, args_.basename, args_.sign_key))
diff --git a/utils/upload.php b/utils/upload.php
deleted file mode 100644
index 602ed69..0000000
--- a/utils/upload.php
+++ /dev/null
@@ -1,36 +0,0 @@
-<?PHP
- if (isset($_POST) && isset($_POST['token']) &&
- $_POST['token'] === 'be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09';
- ) {
- if (!isset($_POST['arch'])) {
- http_response_code(400);
- echo "400 Bad Request: Should specify arch field while post files";
- die();
- }
- if (!empty($_FILES['file'])) {
-
- $path = "/var/www/repo/". $_POST['arch'] . "/";
- $file_name = urldecode($_FILES['file']['name']);
- $path = $path . basename($file_name);
-
- if (move_uploaded_file($_FILES['file']['tmp_name'], $path)) {
- echo "The file " . basename($file_name) . " has been uploaded";
- } else {
- echo "There was an error uploading the file, please try again!";
- }
- } else if (isset($_POST['action'])) {
- if ($_POST['action'] === 'UPLOADED') {
- $sock = stream_socket_client('unix:///run/update-repo-notify.sock', $errno, $errst);
- fwrite($sock, "ARCH=". $_POST['arch'] ."\r\n");
- fclose($sock);
- echo "200 OK STARTED";
- } else if ($_POST['action'] === 'REQUIRE_CLEAN') {
- shell_exec("/usr/bin/rm -rf /var/www/repo/". $_POST['arch'] . "*");
- shell_exec("/usr/bin/mkdir /var/www/repo/". $_POST['arch']);
- echo "200 OK CLEANED";
- }
- }
- } else {
- http_response_code(403);
- }
-?>
diff --git a/utils/upload.py b/utils/upload.py
deleted file mode 100755
index 2fabd7c..0000000
--- a/utils/upload.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import asyncio
-import logging
-import os
-
-import aiohttp
-import aiohttp_socks
-
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
-
-
-def build_data(args: argparse.Namespace, action: str) -> dict[str, str]:
- return {"token": args.token, "arch": args.arch, "action": action}
-
-
-# source: https://stackoverflow.com/a/1094933
-def sizeof_fmt(num: int, suffix: str = "B"):
- for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
- if abs(num) < 1024.0:
- return f"{num:3.1f}{unit}{suffix}"
- num /= 1024.0
- return f"{num:.1f}Yi{suffix}"
-
-
-# TODO: use upload in chunks to big file
-async def send_file(
- session: aiohttp.ClientSession, args: argparse.Namespace, file: str
-) -> None:
- data = aiohttp.FormData()
- data.add_field("file", open(file, "rb"), filename=file)
- data.add_field("token", args.token)
- data.add_field("arch", args.arch)
- async with session.post(args.remote_address, data=data) as req:
- logger.debug("%s", (await req.text()).strip())
-
-
-async def main(args: argparse.Namespace) -> int:
- pending_upload = []
- for file in os.listdir("."):
- if ".pkg" in file:
- pending_upload.append(file)
-
- logger.debug("Pending upload (%d): %s", len(pending_upload), pending_upload)
-
- if not len(pending_upload):
- logger.info("Could not find any packaged packages, skip upload")
- return 0
-
- if proxy := os.getenv("http_proxy"):
- logger.debug("Using proxy %s", proxy)
- proxies = aiohttp_socks.ProxyConnector.from_url(proxy, rdns=False)
- else:
- proxies = None
-
- async with aiohttp.ClientSession(
- raise_for_status=True, timeout=aiohttp.ClientTimeout(30), connector=proxies
- ) as session:
- async with session.post(
- args.remote_address, data=build_data(args, "REQUIRE_CLEAN")
- ):
- pass
- for file in pending_upload:
- logger.info("Start upload %s (%s)", file, sizeof_fmt(os.stat(file).st_size))
- for retries in range(3):
- try:
- await send_file(session, args, file)
- break
- except asyncio.TimeoutError:
- if retries == 2:
- raise
- async with session.post(args.remote_address, data=build_data(args, "UPLOADED")):
- pass
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser("upload.py")
- parser.add_argument("remote_address")
- parser.add_argument("token")
- parser.add_argument("arch")
- parser.add_argument("--directory")
- args_ = parser.parse_args()
-
- try:
- import coloredlogs
-
- coloredlogs.install(
- logging.DEBUG,
- fmt="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
- except ModuleNotFoundError:
- logging.basicConfig(
- level=logging.DEBUG,
- format="%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s",
- )
-
- if args_.directory is not None:
- os.chdir(args_.directory)
-
- exit(asyncio.run(main(args_)))