From 254846d3da3f6c7fb70b316f5ebdf2ce4215309f Mon Sep 17 00:00:00 2001 From: KunoiSayami Date: Sun, 7 Nov 2021 13:48:41 +0800 Subject: refactor(script): Move all script to utils folder * feat(script): Build test package not request signing for now Signed-off-by: KunoiSayami --- .gitlab-ci.yml | 6 +- README.md | 5 +- ci-upload.sh | 33 --------- deploy.sh | 18 ----- docker/dockerfile | 12 +++- docker/onekey.sh | 9 ++- pkgbuild.sh | 145 --------------------------------------- repodb.sh | 34 ---------- utils/aur-check-update.py | 153 ------------------------------------------ utils/aur/aur-check-update.py | 153 ++++++++++++++++++++++++++++++++++++++++++ utils/aur/check-aur.py | 78 +++++++++++++++++++++ utils/check-aur.py | 78 --------------------- utils/ci-upload.sh | 38 +++++++++++ utils/deploy.sh | 18 +++++ utils/pkgbuild.sh | 152 +++++++++++++++++++++++++++++++++++++++++ utils/repodb.sh | 34 ++++++++++ 16 files changed, 495 insertions(+), 471 deletions(-) delete mode 100755 ci-upload.sh delete mode 100755 deploy.sh delete mode 100755 pkgbuild.sh delete mode 100755 repodb.sh delete mode 100755 utils/aur-check-update.py create mode 100755 utils/aur/aur-check-update.py create mode 100755 utils/aur/check-aur.py delete mode 100755 utils/check-aur.py create mode 100755 utils/ci-upload.sh create mode 100755 utils/deploy.sh create mode 100755 utils/pkgbuild.sh create mode 100755 utils/repodb.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 870262f..1086eea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ build: #- cd /home/build/$CI_PROJECT_TITLE - chown -R build:build ../$CI_PROJECT_TITLE - su -c 'echo $GPG_PRIV_KEY | base64 -d | gpg --import' build - - su -c './pkgbuild.sh' build + - su -c './utils/pkgbuild.sh' build - rm -rf build/ # only push to master master trigger push @@ -49,11 +49,9 @@ post: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH changes: - - .gitlab-ci.yml - - pkgbuild.sh - repo/**/* - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TITLE =~ /REBUILD/ script: - if [ -n "$TARGET_HOSTS" ]; then echo "$TARGET_HOSTS" | base64 -d >> /etc/hosts; fi; - - bash -c './ci-upload.sh' + - bash -c './utils/ci-upload.sh' diff --git a/README.md b/README.md index 3592f64..57ebe1a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,10 @@ sudo pacman-key --lsign-key 4A0F0C8BC709ACA4341767FB243975C8DB9656B9 ### For developers: -Use commit message starts with `fix(repo)` will let CI/CD rebuild all packages. +Use commit message starts with `fix(repo)` will let CI/CD rebuild all packages, or you can use `REBUILD`/`REBUILD_ALL` in commit title. +`REBUILD_ALL` will ignore [verified_packages](repo/.verified_repos). + +You need to open a pull request for every add package request. ## License diff --git a/ci-upload.sh b/ci-upload.sh deleted file mode 100755 index 307912e..0000000 --- a/ci-upload.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeuo pipefail - -# clean up when exit -trap cleanup SIGINT SIGTERM ERR EXIT -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset ARCH - unset PKGDEST -} - -ARCH=$(uname -m) -PKGDEST="${PWD}/packages/$ARCH" - -curl -d "token=$UPLOAD_TOKEN&action=REQUIRE_CLEAN&arch=$ARCH" -fsSL "$REMOTE_PATH" - -pushd $PKGDEST - -for file in *.pkg*; do - echo "Start upload package $file size: $(ls -lh $file | awk '{print $5}')..." - curl -F "file=@$file" -F "token=$UPLOAD_TOKEN" -F "arch=$ARCH" -fsSL "$REMOTE_PATH" - #echo "Stop upload package $file" -done - -popd - -curl -d "token=$UPLOAD_TOKEN&action=UPLOADED&arch=$ARCH" -fsSL "$REMOTE_PATH" - -if [ -r .fail ]; then - exit 2; -fi diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 462cfdf..0000000 --- a/deploy.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeuo pipefail - -if [ $# -eq 0 ]; then - echo 'Should provide least one argument' - exit 1 -fi - -if [ -z ${SSH_CLIENT+x} ]; then - ADDITIONAL_PARAM="--partial" -else - ADDITIONAL_PARAM="-Pv" -fi - -/usr/bin/date +%s > "packages/LASTSYNC" -/usr/bin/rsync $ADDITIONAL_PARAM -r -c --update --links --delete --exclude=.gitignore "packages/" $1:/var/www/repo/ diff --git a/docker/dockerfile b/docker/dockerfile index 26b7e49..879f1d4 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -1,12 +1,18 @@ FROM repobuildbase:latest -RUN echo -e '\n[archlinuxcn]\nServer = https://repo.archlinuxcn.org/$arch\n' >> /etc/pacman.conf +ARG REMOTE_ADDRESS + +RUN echo -e '\n[kunoisayami]\nServer = ${REMOTE_ADDRESS}\n' >> /etc/pacman.conf RUN pacman-key --init -RUN pacman --noconfirm --needed -Sy archlinuxcn-keyring +RUN curl -fL https://keys.openpgp.org/vks/v1/by-fingerprint/4A0F0C8BC709ACA4341767FB243975C8DB9656B9 | pacman-key --add - + +RUN pacman-key --finger 4A0F0C8BC709ACA4341767FB243975C8DB9656B9 + +RUN pacman-key --lsign-key 4A0F0C8BC709ACA4341767FB243975C8DB9656B9 -RUN pacman --noconfirm --needed -Su base-devel gnupg git yay +RUN pacman --noconfirm --needed -Syu base-devel gnupg git yay RUN useradd -m build diff --git a/docker/onekey.sh b/docker/onekey.sh index f27153a..99384db 100755 --- a/docker/onekey.sh +++ b/docker/onekey.sh @@ -3,7 +3,12 @@ # end up when failed set -Eeuo pipefail -pushd "$(uname -m)" || ( echo "Can't find your architecture" && exit 0 ) +if [ $# -eq 0 ] && [ -z "$1" ]; then + echo "Please specify REMOTE SERVER ADDRESS" + exit 1 +fi + +pushd "$(uname -m)" || ( echo "Can't find your architecture" && exit 1 ) docker build . -t repobuildbase popd -docker build . -t repobuild +docker build --build-arg "REMOTE_ADDRESS=$1" . -t repobuild diff --git a/pkgbuild.sh b/pkgbuild.sh deleted file mode 100755 index 7be46ab..0000000 --- a/pkgbuild.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeuo pipefail - -# clean up when exit -trap cleanup SIGINT SIGTERM ERR EXIT -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset UNSUCCESSFUL - unset SKIP_VERIFIED - unset FOLDER_NAME - unset PKGDEST - unset SRCDEST - unset CONFDEST - unset REPO_DEST - rm -f "$TMPCONF" - if ! { [ $# -gt 0 ] && [ "$1" = "--diff" ]; }; then - rm -f "$REPO_DIFF" - fi - unset TMPCONF - unset REPO_PENDING - unset REPO_DIFF - unset PKGBUILD_DIRECTORY_BASE - unset ARCH -} - -function hook { - if [ -r ../.hook/"$1" ]; then - echo "Running $1 Hook script" - . ../.hook/"$1" - fi -} - -ARCH=$(uname -m) -PKGBUILD_DIRECTORY_BASE="repo" -PKGDEST="${PWD}/packages/$ARCH" -SRCDEST="${PWD}/build" -CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf" -REPO_DIFF="${PWD}/packages/$ARCH/DIFF" -SKIP_VERIFIED=1 -UNSUCCESSFUL=0 - -TMPCONF=$(mktemp -t makepkg.$ARCH.conf.XXXXXXXXXX) || exit 1 -cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF" -cat "$CONFDEST" >> "$TMPCONF" - -touch "$REPO_DIFF" - -function get_diff_list { - git diff --name-only HEAD^ | while read line; do - if [[ $line =~ $PKGBUILD_DIRECTORY_BASE/\..+ ]] && [ "$line" != "$PKGBUILD_DIRECTORY_BASE/.verified_repos" ]; then - echo "$line" | cut -d'/' -f3 >> "$REPO_DIFF" - elif [[ $line =~ $PKGBUILD_DIRECTORY_BASE ]]; then - echo "$line" | cut -d'/' -f2 >> "$REPO_DIFF" - fi - done - cp "$REPO_DIFF" "${REPO_DIFF}_tmp" - sort -u "${REPO_DIFF}_tmp" > "$REPO_DIFF" - rm -f "${REPO_DIFF}_tmp" -} - -if [ $# -gt 0 ] && [ "$1" = "--diff" ]; then - get_diff_list - exit 0 -fi - -if [ "$EUID" -eq 0 ] ; then - echo "\033[0;32mPlease run this script as non-root\033[0m" - exit 3 -fi - -if [ $# -gt 0 ] && [ "$1" = "--all" ] || [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ fix\(repo\)|REBUILD ]]; then - if [ -z "${FORCE_ALL+x}" ] || [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ REBUILD(\ |_)ALL ]]; then - SKIP_VERIFIED=0 - fi - ls $PKGBUILD_DIRECTORY_BASE > "$REPO_DIFF" -else - get_diff_list - echo "Next" -fi - -pushd $PKGBUILD_DIRECTORY_BASE || ( echo -e "\033[0;31mError, can't switch to pkgbuild directory\033[0m" && exit 1 ) - -while read -r FOLDER_NAME ; do - if [ ! -d "$FOLDER_NAME" ]; then - echo -e "\033[0;32mSkip folder $FOLDER_NAME\033[0m" - continue - fi - echo -e "\033[0;32mProcessing $FOLDER_NAME\033[0m" - pushd "$FOLDER_NAME" || { echo -e "\033[1;33mWarning, can't chdir to $FOLDER_NAME, skipped\033[0m"; continue; } - - # hook must run before other checker - hook "$FOLDER_NAME" - - # Check PKGBUILD architecture - if ! ( pcregrep -M 'arch=\([^\)]*\)' PKGBUILD | grep -E "$ARCH|any" >/dev/null ); then - echo -e "\033[0;32mSkip folder $FOLDER_NAME (Architecture not match)\033[0m" - popd - continue - fi - - if [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ fix\(repo\)|REBUILD ]] && \ - [ $SKIP_VERIFIED -eq 1 ] && grep -Fxq "$FOLDER_NAME" "../.verified_repos"; then - echo -e "\033[0;32mSkip folder $FOLDER_NAME (verified repository)\033[0m" - popd - continue - fi - - if [ -r ../.yaydeps/"$FOLDER_NAME" ]; then - while read -r YAYDEP ; do - if [ -d "../$YAYDEP" ]; then - pushd "../$YAYDEP" - hook "$YAYDEP" - SRCPKGDEST=$SRCDEST SRCDEST=$SRCDEST PKGDEST=$PKGDEST MAKEPKG_CONF="$TMPCONF" makepkg --clean -si --asdeps --noconfirm --needed --noprogressbar - popd - else - yay -S --noconfirm --needed --asdeps "$YAYDEP" - fi - done < ../.yaydeps/"$FOLDER_NAME" - fi - - if [ -r ../.gpg_keys/"$FOLDER_NAME" ]; then - . ../.gpg_keys/"$FOLDER_NAME" - # source: https://stackoverflow.com/a/53886735 - gpg --list-keys --fingerprint | grep pub -A 1 | grep -Ev "pub|--" | tr -d ' ' \ - | awk 'BEGIN { FS = "\n" } ; { print $1":6:" } ' | gpg --import-ownertrust - fi - - SRCPKGDEST=$SRCDEST SRCDEST=$SRCDEST PKGDEST=$PKGDEST MAKEPKG_CONF="$TMPCONF" makepkg --clean -s --asdeps --noconfirm --needed --noprogressbar || { echo -e "\033[0;31mSkip folder $FOLDER_NAME\033[0m"; UNSUCCESSFUL=1; } - popd -done < "$REPO_DIFF" - -popd - -date +%s > "$PKGDEST/LASTBUILD" - -if [ $UNSUCCESSFUL -eq 1 ]; then - if [ -n "$CI_DEFAULT_BRANCH" ] && [ -n "$CI_DEFAULT_BRANCH" ] && [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - touch .fail - else - echo -e "\033[0;31mExit due makepkg failed\033[0m" - exit 2 - fi -fi diff --git a/repodb.sh b/repodb.sh deleted file mode 100755 index 212b137..0000000 --- a/repodb.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# end up when failed -set -Eeuo pipefail - -# clean up when exit -trap cleanup SIGINT SIGTERM ERR EXIT -cleanup() { - trap - SIGINT SIGTERM ERR EXIT - unset ARCH - unset PKGDEST - unset REPO_DEST - unset GPG_KEY - unset PACKAGE_NAME - unset LATEST -} - -if [ -z $ARCH ]; then - ARCH=$(uname -m) - echo "Warning: You should specify arch environment variable, use current platform: $ARCH" -fi - -REPO_BASE_NAME="kunoisayami" -#ARCH=$(uname -m) -REPO_DEST="${PWD}/packages/$ARCH/$REPO_BASE_NAME.db.tar.xz" -GPG_KEY="4A0F0C8BC709ACA4341767FB243975C8DB9656B9" - -while read PACKAGE_NAME; do - #LATEST=$(ls -t "$PKGDEST/$PACKAGE_NAME"* | grep -v ".sig" | head -n 1 | cut -d' ' -f1) - if [ -z $PACKAGE_NAME ]; then - continue - fi - repo-add "$REPO_DEST" "$PACKAGE_NAME" -s -v -R -k $GPG_KEY -done diff --git a/utils/aur-check-update.py b/utils/aur-check-update.py deleted file mode 100755 index 3188aea..0000000 --- a/utils/aur-check-update.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2021 KunoiSayami -# -# This module is part of KunoiSayami/repos and is released under -# the AGPL v3 License: https://www.gnu.org/licenses/agpl-3.0.txt -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -from __future__ import annotations - -import argparse -import asyncio -import logging -import re - -from collections.abc import Coroutine -from dataclasses import dataclass -from pathlib import Path - -import aiofiles -import aiohttp - -MAX_WORKERS = 32 - - -@dataclass -class Version: - pkgver: list[str] - pkgrel: list[str] - epoch: int - - @classmethod - def from_str(cls, s: str) -> Version: - match = re.search(r"pkgver=(?P.*)", s) - if not match: - raise ValueError('pkgver keyword not found') - else: - pkgver = match.group('VER').split('.') - match = re.search(r"pkgrel=(?P.*)", s) - if not match: - pkgrel = ['1'] - else: - pkgrel = match.group('REL').split('.') - match = re.search(r"epoch=(?P\d+)", s) - if not match: - epoch = 0 - else: - epoch = int(match.group('EPOCH')) - return cls(pkgver, pkgrel, epoch) - - @classmethod - async def from_file(cls, file: str | Path) -> Version: - async with aiofiles.open(file, 'r', encoding='utf-8') as fin: - return cls.from_str(await fin.read()) - - def __eq__(self, other: Version) -> bool: - return self.epoch == other.epoch \ - and self.pkgver == self.pkgver \ - and self.pkgrel == other.pkgrel - - def __gt__(self, other: Version) -> bool: - return self.epoch > other.epoch \ - or self.pkgver > other.pkgver \ - or (self.pkgver == other.pkgver and self.pkgrel > other.pkgrel) - - def __ne__(self, other: Version) -> bool: - return not self.__eq__(other) - - def __ge__(self, other: Version) -> bool: - return self.__gt__(other) or self.__eq__(other) - - def __lt__(self, other: Version) -> bool: - return not self.__ge__(other) - - def __str__(self) -> str: - prefix = f'{self.epoch}:' if self.epoch > 0 else '' - return f'{prefix}{self.pkgver}-{self.pkgrel}' - - -async def with_sem(sem: asyncio.Semaphore, coro: Coroutine): - async with sem: - return await coro - - -async def aur_check_update(session: aiohttp.ClientSession, item: Path, dry_run: bool) -> None: - if item.name.startswith('.') or not item.is_dir(): - return - pkgbuild_file = item.joinpath('PKGBUILD') - if not pkgbuild_file.exists(): - logging.warning(f'PKGBUILD not in {str(item)}, SKIP!') - return - version = await Version.from_file(pkgbuild_file) - async with session.get(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={item.name}') as resp: - if resp.status != 200: - logging.info(f'{item.name} not register in AUR') - return - new_version = Version.from_str(await resp.text()) - - if new_version == version: - return - elif new_version < version: - logging.warning(f'Warning: {version} is newer than AUR') - return - if item.joinpath('.git').exists(): - if not dry_run: - await asyncio.create_subprocess_exec('git', '-C', f'{str(item)}', 'pull') - logging.info(f'Upgrade {item.name} from {version} to {new_version}') - else: - logging.info(f'Found update {item.name}({version}) (local: {new_version})') - - -async def main() -> int: - parser = argparse.ArgumentParser(description='aur-check-update.py') - parser.add_argument('--dry', help='dry run', action='store_true') - parser.add_argument('--max-workers', dest='workers', metavar='MAX_WORKERS', - help=f'aur check max workers, default: {MAX_WORKERS}', - default=MAX_WORKERS, action='store', type=int) - parser.add_argument(dest='repo_dir', metavar='REPO_DIR', help='repository directory', nargs='?') - - args = parser.parse_args() - - dry_run = args.dry - sem = asyncio.Semaphore(args.workers) - - if args.repo_dir is None: - logging.error('repository directory not found') - return 1 - - repo_dir = Path(args.repo_dir) - - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: - tasks = [asyncio.create_task( - with_sem(sem, aur_check_update(session, item, dry_run)) - ) for item in repo_dir.iterdir()] - await asyncio.gather(*tasks) - - return 0 - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s') - loop = asyncio.get_event_loop() - exit(loop.run_until_complete(main())) diff --git a/utils/aur/aur-check-update.py b/utils/aur/aur-check-update.py new file mode 100755 index 0000000..3188aea --- /dev/null +++ b/utils/aur/aur-check-update.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 KunoiSayami +# +# This module is part of KunoiSayami/repos and is released under +# the AGPL v3 License: https://www.gnu.org/licenses/agpl-3.0.txt +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from __future__ import annotations + +import argparse +import asyncio +import logging +import re + +from collections.abc import Coroutine +from dataclasses import dataclass +from pathlib import Path + +import aiofiles +import aiohttp + +MAX_WORKERS = 32 + + +@dataclass +class Version: + pkgver: list[str] + pkgrel: list[str] + epoch: int + + @classmethod + def from_str(cls, s: str) -> Version: + match = re.search(r"pkgver=(?P.*)", s) + if not match: + raise ValueError('pkgver keyword not found') + else: + pkgver = match.group('VER').split('.') + match = re.search(r"pkgrel=(?P.*)", s) + if not match: + pkgrel = ['1'] + else: + pkgrel = match.group('REL').split('.') + match = re.search(r"epoch=(?P\d+)", s) + if not match: + epoch = 0 + else: + epoch = int(match.group('EPOCH')) + return cls(pkgver, pkgrel, epoch) + + @classmethod + async def from_file(cls, file: str | Path) -> Version: + async with aiofiles.open(file, 'r', encoding='utf-8') as fin: + return cls.from_str(await fin.read()) + + def __eq__(self, other: Version) -> bool: + return self.epoch == other.epoch \ + and self.pkgver == self.pkgver \ + and self.pkgrel == other.pkgrel + + def __gt__(self, other: Version) -> bool: + return self.epoch > other.epoch \ + or self.pkgver > other.pkgver \ + or (self.pkgver == other.pkgver and self.pkgrel > other.pkgrel) + + def __ne__(self, other: Version) -> bool: + return not self.__eq__(other) + + def __ge__(self, other: Version) -> bool: + return self.__gt__(other) or self.__eq__(other) + + def __lt__(self, other: Version) -> bool: + return not self.__ge__(other) + + def __str__(self) -> str: + prefix = f'{self.epoch}:' if self.epoch > 0 else '' + return f'{prefix}{self.pkgver}-{self.pkgrel}' + + +async def with_sem(sem: asyncio.Semaphore, coro: Coroutine): + async with sem: + return await coro + + +async def aur_check_update(session: aiohttp.ClientSession, item: Path, dry_run: bool) -> None: + if item.name.startswith('.') or not item.is_dir(): + return + pkgbuild_file = item.joinpath('PKGBUILD') + if not pkgbuild_file.exists(): + logging.warning(f'PKGBUILD not in {str(item)}, SKIP!') + return + version = await Version.from_file(pkgbuild_file) + async with session.get(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={item.name}') as resp: + if resp.status != 200: + logging.info(f'{item.name} not register in AUR') + return + new_version = Version.from_str(await resp.text()) + + if new_version == version: + return + elif new_version < version: + logging.warning(f'Warning: {version} is newer than AUR') + return + if item.joinpath('.git').exists(): + if not dry_run: + await asyncio.create_subprocess_exec('git', '-C', f'{str(item)}', 'pull') + logging.info(f'Upgrade {item.name} from {version} to {new_version}') + else: + logging.info(f'Found update {item.name}({version}) (local: {new_version})') + + +async def main() -> int: + parser = argparse.ArgumentParser(description='aur-check-update.py') + parser.add_argument('--dry', help='dry run', action='store_true') + parser.add_argument('--max-workers', dest='workers', metavar='MAX_WORKERS', + help=f'aur check max workers, default: {MAX_WORKERS}', + default=MAX_WORKERS, action='store', type=int) + parser.add_argument(dest='repo_dir', metavar='REPO_DIR', help='repository directory', nargs='?') + + args = parser.parse_args() + + dry_run = args.dry + sem = asyncio.Semaphore(args.workers) + + if args.repo_dir is None: + logging.error('repository directory not found') + return 1 + + repo_dir = Path(args.repo_dir) + + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: + tasks = [asyncio.create_task( + with_sem(sem, aur_check_update(session, item, dry_run)) + ) for item in repo_dir.iterdir()] + await asyncio.gather(*tasks) + + return 0 + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s') + loop = asyncio.get_event_loop() + exit(loop.run_until_complete(main())) diff --git a/utils/aur/check-aur.py b/utils/aur/check-aur.py new file mode 100755 index 0000000..6749f69 --- /dev/null +++ b/utils/aur/check-aur.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 KunoiSayami and contributors +# +# This module is part of KunoiSayami/repos and is released under +# the AGPL v3 License: https://www.gnu.org/licenses/agpl-3.0.txt +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import argparse +import asyncio +import logging +import re +from pathlib import Path +from typing import Tuple + +import aiofiles +import aiohttp + + +async def check_pkg(session: aiohttp.ClientSession, num: int, pkg: str) -> Tuple[int, bool]: + logging.info(f'Checking {pkg}') + ret = await session.head(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={pkg}') + return num, ret.status == 200 + + +async def main() -> int: + parser = argparse.ArgumentParser(description='check-aur.py') + parser.add_argument('paths', metavar='sub_path', type=str, nargs='*', help='sub-path for repo root') + + args = parser.parse_args() + repo_dir = Path(*args.paths) + pkgbuild_file = repo_dir.joinpath('PKGBUILD') + if not pkgbuild_file.exists(): + logging.error("Can't locate PKGBUILD file, please specify directory in arguments or chdir to folder") + return 1 + + async with aiofiles.open(pkgbuild_file, 'r', encoding='utf-8') as f: + pkgbuild_text = await f.read() + + deps_match = re.search(r"makedepends=\((?P([^)]|\n)+)\)", pkgbuild_text, re.M) + if not deps_match: + logging.error("Can't found `makedepends' keyword, if you think this is mistake, please report issue") + return 2 + + deps = deps_match.group('DEPS').split() + + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: + tasks = [asyncio.create_task(check_pkg(session, n, pkg)) for n, pkg in enumerate(deps)] + yay_dep_nums = [] + for task in asyncio.as_completed(tasks): + n, r = await task + if r: + yay_dep_nums.append(n) + + yaydeps_dir = repo_dir.parent.joinpath('.yaydeps') + + if len(yay_dep_nums) > 0: + async with aiofiles.open(yaydeps_dir.joinpath(repo_dir.stem), 'w', encoding='utf-8') as f: + await f.write('\n'.join([*(deps[n] for n in sorted(yay_dep_nums)), ''])) + + return 0 + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s') + loop = asyncio.get_event_loop() + exit(loop.run_until_complete(main())) diff --git a/utils/check-aur.py b/utils/check-aur.py deleted file mode 100755 index 6749f69..0000000 --- a/utils/check-aur.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2021 KunoiSayami and contributors -# -# This module is part of KunoiSayami/repos and is released under -# the AGPL v3 License: https://www.gnu.org/licenses/agpl-3.0.txt -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -import argparse -import asyncio -import logging -import re -from pathlib import Path -from typing import Tuple - -import aiofiles -import aiohttp - - -async def check_pkg(session: aiohttp.ClientSession, num: int, pkg: str) -> Tuple[int, bool]: - logging.info(f'Checking {pkg}') - ret = await session.head(f'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h={pkg}') - return num, ret.status == 200 - - -async def main() -> int: - parser = argparse.ArgumentParser(description='check-aur.py') - parser.add_argument('paths', metavar='sub_path', type=str, nargs='*', help='sub-path for repo root') - - args = parser.parse_args() - repo_dir = Path(*args.paths) - pkgbuild_file = repo_dir.joinpath('PKGBUILD') - if not pkgbuild_file.exists(): - logging.error("Can't locate PKGBUILD file, please specify directory in arguments or chdir to folder") - return 1 - - async with aiofiles.open(pkgbuild_file, 'r', encoding='utf-8') as f: - pkgbuild_text = await f.read() - - deps_match = re.search(r"makedepends=\((?P([^)]|\n)+)\)", pkgbuild_text, re.M) - if not deps_match: - logging.error("Can't found `makedepends' keyword, if you think this is mistake, please report issue") - return 2 - - deps = deps_match.group('DEPS').split() - - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session: - tasks = [asyncio.create_task(check_pkg(session, n, pkg)) for n, pkg in enumerate(deps)] - yay_dep_nums = [] - for task in asyncio.as_completed(tasks): - n, r = await task - if r: - yay_dep_nums.append(n) - - yaydeps_dir = repo_dir.parent.joinpath('.yaydeps') - - if len(yay_dep_nums) > 0: - async with aiofiles.open(yaydeps_dir.joinpath(repo_dir.stem), 'w', encoding='utf-8') as f: - await f.write('\n'.join([*(deps[n] for n in sorted(yay_dep_nums)), ''])) - - return 0 - - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s - %(levelname)s - %(funcName)s - %(lineno)d - %(message)s') - loop = asyncio.get_event_loop() - exit(loop.run_until_complete(main())) diff --git a/utils/ci-upload.sh b/utils/ci-upload.sh new file mode 100755 index 0000000..69f19cd --- /dev/null +++ b/utils/ci-upload.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# end up when failed +set -Eeuo pipefail + +# clean up when exit +trap cleanup SIGINT SIGTERM ERR EXIT +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + unset ARCH + unset PKGDEST +} + +ARCH=$(uname -m) +PKGDEST="${PWD}/packages/$ARCH" + +if [ $(find $PKGDEST -name \*.pkg\* | wc -l) -eq 0 ]; then + echo -e "\033[0;31mCould not find any packaged packages, skip upload\033[0m" + exit 0 +fi + +curl -d "token=$UPLOAD_TOKEN&action=REQUIRE_CLEAN&arch=$ARCH" -fsSL "$REMOTE_PATH" + +pushd $PKGDEST + +for file in *.pkg*; do + echo "Start upload package $file size: $(ls -lh $file | awk '{print $5}')..." + curl -F "file=@$file" -F "token=$UPLOAD_TOKEN" -F "arch=$ARCH" -fsSL "$REMOTE_PATH" + #echo "Stop upload package $file" +done + +popd + +curl -d "token=$UPLOAD_TOKEN&action=UPLOADED&arch=$ARCH" -fsSL "$REMOTE_PATH" + +if [ -r .fail ]; then + exit 2; +fi diff --git a/utils/deploy.sh b/utils/deploy.sh new file mode 100755 index 0000000..462cfdf --- /dev/null +++ b/utils/deploy.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# end up when failed +set -Eeuo pipefail + +if [ $# -eq 0 ]; then + echo 'Should provide least one argument' + exit 1 +fi + +if [ -z ${SSH_CLIENT+x} ]; then + ADDITIONAL_PARAM="--partial" +else + ADDITIONAL_PARAM="-Pv" +fi + +/usr/bin/date +%s > "packages/LASTSYNC" +/usr/bin/rsync $ADDITIONAL_PARAM -r -c --update --links --delete --exclude=.gitignore "packages/" $1:/var/www/repo/ diff --git a/utils/pkgbuild.sh b/utils/pkgbuild.sh new file mode 100755 index 0000000..86965dd --- /dev/null +++ b/utils/pkgbuild.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# end up when failed +set -Eeuo pipefail + +# clean up when exit +trap cleanup SIGINT SIGTERM ERR EXIT +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + unset SIGNING_ARG + unset UNSUCCESSFUL + unset SKIP_VERIFIED + unset FOLDER_NAME + unset PKGDEST + unset SRCDEST + unset CONFDEST + unset REPO_DEST + rm -f "$TMPCONF" + if ! { [ $# -gt 0 ] && [ "$1" = "--diff" ]; }; then + rm -f "$REPO_DIFF" + fi + unset TMPCONF + unset REPO_PENDING + unset REPO_DIFF + unset PKGBUILD_DIRECTORY_BASE + unset ARCH +} + +function hook { + if [ -r ../.hook/"$1" ]; then + echo "Running $1 Hook script" + . ../.hook/"$1" + fi +} + +ARCH=$(uname -m) +PKGBUILD_DIRECTORY_BASE="repo" +PKGDEST="${PWD}/packages/$ARCH" +SRCDEST="${PWD}/build" +CONFDEST="${PWD}/makepkg.d/makepkg.$ARCH.conf" +REPO_DIFF="${PWD}/packages/$ARCH/DIFF" +SKIP_VERIFIED=1 +UNSUCCESSFUL=0 +SIGNING_ARG="" + +TMPCONF=$(mktemp -t makepkg.$ARCH.conf.XXXXXXXXXX) || exit 1 +cat "${PWD}/makepkg.d/makepkg.base.conf" > "$TMPCONF" +cat "$CONFDEST" >> "$TMPCONF" + +touch "$REPO_DIFF" + +function get_diff_list { + git diff --name-only HEAD^ | while read line; do + if [[ $line =~ $PKGBUILD_DIRECTORY_BASE/\..+ ]] && [ "$line" != "$PKGBUILD_DIRECTORY_BASE/.verified_repos" ]; then + echo "$line" | cut -d'/' -f3 >> "$REPO_DIFF" + elif [[ $line =~ $PKGBUILD_DIRECTORY_BASE ]]; then + echo "$line" | cut -d'/' -f2 >> "$REPO_DIFF" + fi + done + cp "$REPO_DIFF" "${REPO_DIFF}_tmp" + sort -u "${REPO_DIFF}_tmp" > "$REPO_DIFF" + rm -f "${REPO_DIFF}_tmp" +} + +if [ $# -gt 0 ] && [ "$1" = "--diff" ]; then + get_diff_list + exit 0 +fi + +if [ "$EUID" -eq 0 ] ; then + echo "\033[0;32mPlease run this script as non-root\033[0m" + exit 3 +fi + +if [ $# -gt 0 ] && [ "$1" = "--all" ] || [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ fix\(repo\)|REBUILD ]]; then + if [ -z "${FORCE_ALL+x}" ] || [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ REBUILD(\ |_)ALL ]]; then + SKIP_VERIFIED=0 + fi + ls $PKGBUILD_DIRECTORY_BASE > "$REPO_DIFF" +else + get_diff_list +fi + +if [ -n "$CI_DEFAULT_BRANCH" ] && [ -n "$CI_DEFAULT_BRANCH" ] && [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + SIGNING_ARG="-s" +else + echo -e "\033[0;32mSkip signing package\033[0m" +fi + +pushd $PKGBUILD_DIRECTORY_BASE || ( echo -e "\033[0;31mError, can't switch to pkgbuild directory\033[0m" && exit 1 ) + +while read -r FOLDER_NAME ; do + if [ ! -d "$FOLDER_NAME" ]; then + echo -e "\033[0;32mSkip folder $FOLDER_NAME\033[0m" + continue + fi + echo -e "\033[0;32mProcessing $FOLDER_NAME\033[0m" + pushd "$FOLDER_NAME" || { echo -e "\033[1;33mWarning, can't chdir to $FOLDER_NAME, skipped\033[0m"; continue; } + + # hook must run before other checker + hook "$FOLDER_NAME" + + # Check PKGBUILD architecture + if ! ( pcregrep -M 'arch=\([^\)]*\)' PKGBUILD | grep -E "$ARCH|any" >/dev/null ); then + echo -e "\033[0;32mSkip folder $FOLDER_NAME (Architecture not match)\033[0m" + popd + continue + fi + + if [ -n "$CI_COMMIT_TITLE" ] && [[ $CI_COMMIT_TITLE =~ fix\(repo\)|REBUILD ]] && \ + [ $SKIP_VERIFIED -eq 1 ] && grep -Fxq "$FOLDER_NAME" "../.verified_repos"; then + echo -e "\033[0;32mSkip folder $FOLDER_NAME (verified repository)\033[0m" + popd + continue + fi + + if [ -r ../.yaydeps/"$FOLDER_NAME" ]; then + while read -r YAYDEP ; do + if [ -d "../$YAYDEP" ]; then + pushd "../$YAYDEP" + hook "$YAYDEP" + SRCPKGDEST=$SRCDEST SRCDEST=$SRCDEST PKGDEST=$PKGDEST MAKEPKG_CONF="$TMPCONF" makepkg --clean -i $SIGNING_ARG --asdeps --noconfirm --needed --noprogressbar + popd + else + yay -S --noconfirm --needed --asdeps "$YAYDEP" + fi + done < ../.yaydeps/"$FOLDER_NAME" + fi + + if [ -r ../.gpg_keys/"$FOLDER_NAME" ]; then + . ../.gpg_keys/"$FOLDER_NAME" + # source: https://stackoverflow.com/a/53886735 + gpg --list-keys --fingerprint | grep pub -A 1 | grep -Ev "pub|--" | tr -d ' ' \ + | awk 'BEGIN { FS = "\n" } ; { print $1":6:" } ' | gpg --import-ownertrust + fi + + SRCPKGDEST=$SRCDEST SRCDEST=$SRCDEST PKGDEST=$PKGDEST MAKEPKG_CONF="$TMPCONF" makepkg --clean $SIGNING_ARG --asdeps --noconfirm --needed --noprogressbar || { echo -e "\033[0;31mSkip folder $FOLDER_NAME\033[0m"; UNSUCCESSFUL=1; } + popd +done < "$REPO_DIFF" + +popd + +date +%s > "$PKGDEST/LASTBUILD" + +if [ $UNSUCCESSFUL -eq 1 ]; then + if [ -n "$CI_DEFAULT_BRANCH" ] && [ -n "$CI_DEFAULT_BRANCH" ] && [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + touch .fail + else + echo -e "\033[0;31mExit due makepkg failed\033[0m" + exit 2 + fi +fi diff --git a/utils/repodb.sh b/utils/repodb.sh new file mode 100755 index 0000000..212b137 --- /dev/null +++ b/utils/repodb.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# end up when failed +set -Eeuo pipefail + +# clean up when exit +trap cleanup SIGINT SIGTERM ERR EXIT +cleanup() { + trap - SIGINT SIGTERM ERR EXIT + unset ARCH + unset PKGDEST + unset REPO_DEST + unset GPG_KEY + unset PACKAGE_NAME + unset LATEST +} + +if [ -z $ARCH ]; then + ARCH=$(uname -m) + echo "Warning: You should specify arch environment variable, use current platform: $ARCH" +fi + +REPO_BASE_NAME="kunoisayami" +#ARCH=$(uname -m) +REPO_DEST="${PWD}/packages/$ARCH/$REPO_BASE_NAME.db.tar.xz" +GPG_KEY="4A0F0C8BC709ACA4341767FB243975C8DB9656B9" + +while read PACKAGE_NAME; do + #LATEST=$(ls -t "$PKGDEST/$PACKAGE_NAME"* | grep -v ".sig" | head -n 1 | cut -d' ' -f1) + if [ -z $PACKAGE_NAME ]; then + continue + fi + repo-add "$REPO_DEST" "$PACKAGE_NAME" -s -v -R -k $GPG_KEY +done -- cgit v1.3.1