diff --git a/README.md b/README.md index 2d8abf3..44d31ab 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Use python version > 3.8 - -h, --help -> ```Show help message and exit``` -- -g GROUP, --group GROUP -> ```Add GROUP id it can be found under group name. Id must be integer``` +- -g GROUP, --group GROUP -> ```Add repository to GROUP name. It also named Organisation``` - -u URL [URL ...], --urls URL [URL ...] @@ -19,9 +19,13 @@ Use python version > 3.8 ```Add file with urls. Each url on new line. Can be combined with --url option. Names will generate automatically from links``` - -t TOKEN, --token TOKEN -```Access token to gitlab API. More information:``` [gitlab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) +```Access token to gitea API. More information:``` [gitea docs](https://docs.gitea.io/en-us/api-usage/#authentication) -- -l GITLAB, --gitlab GITLAB ```Provide gitlab url. Default link``` https://git.do.x5.ru +- T GITHUBTOKEN, --githubtoken GITHUBTOKEN +- ```Please provide github token to get access to private repositories``` [github tokens](https://github.com/settings/tokens) + + +- -l GITURL, --giturl GITURL ```Provide git url. Default link``` https://git.mywistr.com ## Usage @@ -33,12 +37,12 @@ python3 github_mirror.py [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN ## Examples: - python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" -g 2059 -t "git-QwertY1245kde" + python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" -g "GitHub" -t "gtb-QwertY1245kde" - python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "git-QwertY1245kde" + python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "gtb-QwertY1245kde" - python3 github_mirror -f github_mirrors.txt -g 59563 -t "git-QwertY1245kde" + python3 github_mirror.py -f github_mirrors.txt -g "Public" -t "gtb-QwertY1245kde" - python3 github_mirror -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "git-QwertY1245kde" + python3 github_mirror.py -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "gtb-QwertY125kde" - python3 github_mirror.py --gitlab "https://gitlab.company.ru" -t "git-QwertY1245kde" -g 2059 -u "https://github.com/s3rius/FastAPI-template.git" \ No newline at end of file + python3 github_mirror.py --giturl "https://gitea.company.ru" -t "gtb-QwertY1245kde" -u "https://github.com/Balshgit/sonar-scanner.git" -g "Personal" -T "ghb-Qwerty321ldf" diff --git a/core/argument_parser.py b/core/argument_parser.py index 3dffb13..29a80aa 100644 --- a/core/argument_parser.py +++ b/core/argument_parser.py @@ -1,19 +1,19 @@ from argparse import ArgumentParser -GITLAB_URL = 'https://git.do.x5.ru' +GIT_URL = 'https://git.mywistr.com' -USAGE = '''github_mirror [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN +USAGE = '''github_mirror [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN [-T GitHubTOKEN] -------------------------------------------------- -python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" -g 2059 -t "git-QwertY1245kde" +python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" -g "GitHub" -t "gtb-QwertY1245kde" -python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "git-QwertY1245kde" +python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "gtb-QwertY1245kde" -python3 github_mirror -f github_mirrors.txt -g 59563 -t "git-QwertY1245kde" +python3 github_mirror.py -f github_mirrors.txt -g "Public" -t "gtb-QwertY1245kde" -python3 github_mirror -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "git-QwertY1245kde" +python3 github_mirror.py -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "gtb-QwertY125kde" -python3 github_mirror.py --gitlab "https://gitlab.company.ru" -t "git-QwertY1245kde" -g 2059 +python3 github_mirror.py --giturl "https://gitea.company.ru" -t "gtb-QwertY1245kde" -u "https://github.com/Balshgit/sonar-scanner.git" -g "Personal" -T "ghb-Qwerty321ldf" -------------------------------------------------- @@ -28,14 +28,14 @@ def create_parser() -> ArgumentParser: """ parser = ArgumentParser( prog='github_mirror', - description='''Script to add mirror repo into gitlab''', + description='''Script to add mirror repo into gitea''', epilog='''the developer is not responsible for the operation of the script :)''', add_help=True, usage=USAGE ) - parser.add_argument('-g', '--group', required=False, type=int, - help='Add group id it can be found under group name. Id must be integer') + parser.add_argument('-g', '--group', required=False, type=str, + help='Add repository to GROUP name. It also named Organisation') parser.add_argument('-u', '--urls', nargs='+', help='Provide url or urls to mirror with it in format: ' @@ -46,10 +46,13 @@ def create_parser() -> ArgumentParser: '--url option. Names will generate automatically from links') parser.add_argument('-t', '--token', required=True, - help='Access token to gitlab API. More information: https://docs.gitlab.com/ee/user/profile/' - 'personal_access_tokens.html#create-a-personal-access-token') + help='Access token to gitea API. More information: https://docs.gitea.io/en-us/api-usage/' + '#authentication') - parser.add_argument('-l', '--gitlab', required=False, default=GITLAB_URL, - help=f'Provide gitlab url. Default link {GITLAB_URL}') + parser.add_argument('-T', '--githubtoken', required=False, help='Please provide github token to get access ' + 'to private repositories') + + parser.add_argument('-l', '--giturl', required=False, default=GIT_URL, + help=f'Provide git url where you want store your repositories. Default link {GIT_URL}') return parser diff --git a/core/repo_creator.py b/core/repo_creator.py index c897d79..80bb51a 100644 --- a/core/repo_creator.py +++ b/core/repo_creator.py @@ -1,3 +1,4 @@ +import random from typing import Union import requests @@ -23,83 +24,55 @@ class RepositoryCreator: :param data: Provide request data :return: Response object on None """ + try: request = requests.request(method, url, headers=self.headers, json=data, verify=False) return request except Exception as err: - logger.error(f'Connection not established. Check vpn is connected! \n{err}') + logger.error(f'Connection not established. \n{err}') - def __create_new_project(self, url: str, group_id: int = None) -> Union[str, None]: + def __create_new_project(self, url: str, group_name: str = None, auth_token: str = None) -> None: """ Create new project in gitlab with name based on provided url - :param url: github url to mirror with: - :param group_id: namespace in gitlab to combine repos + :param url: GitHub url to mirror with: + :param group_name: namespace in gitlab to combine repos + :param auth_token: GitHub token to access private repositories :return: repo_id as string or None if any error """ # name of repository will generate automatically from link name = url.split('/')[-1].replace('.git', '') - git_data = {'name': name} - if group_id: - git_data['namespace_id'] = group_id + update_time = random.randint(120, 580) # prevent update all repos at the same time + git_data = {'repo_name': name, "wiki": True, "private": False, 'mirror_interval': f'{update_time}h0m0s', + "mirror": True, "lfs": True, "clone_addr": url} + if group_name: + git_data['repo_owner'] = group_name + if auth_token: + git_data['auth_token'] = auth_token - request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects', git_data) + request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v1/repos/migrate', git_data) try: if request.status_code == self.HTTP_201_CREATED: repo_data = request.json() - name_with_namespace = repo_data.get('name_with_namespace', None) + name_with_namespace = repo_data.get('full_name', None) if name_with_namespace: logger.info(f'Repository {name_with_namespace} has been created') else: logger.info(f'Repository {repo_data["name"]} has been created') return repo_data['id'] else: - logger.error(f'Cant create new project. Status code: {request.status_code}. Reason: {request.text}') + logger.error(f'Cant create {name} project. Status code: {request.status_code}. Reason: {request.text}') except AttributeError: pass - def __add_pull_mirror(self, url: str, repo_id: str) -> Union[str, None]: - """ - Add pull mirror to Settings -> Repository -> Mirroring repositories - - :param url: github url to mirror with - :param repo_id: id of repository which will be updated - :return: github url which will be mirrored - """ - - if repo_id: - git_data = {"mirror": True, "import_url": url} - request = self.__gitlab_request('PUT', f'{self.gitlab_url}/api/v4/projects/{repo_id}', git_data) - if request and request.status_code == self.HTTP_200_OK: - return url - elif request.status_code != self.HTTP_200_OK: - logger.error(f'Cant add mirror url to project. Status code: {request.status_code}. ' - f'Reason: {request.text}') - - def __pull_github_repo(self, url: str, repo_id: str): - """ - Initiate pull request for gitlab repository - - :param url: github url to mirror with - :param repo_id: id of repository which will be updated - """ - - if repo_id: - request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects/{repo_id}/mirror/pull') - if request and request.status_code == self.HTTP_200_OK: - logger.info(f'Repository: {url} has been pulled') - elif request.status_code != self.HTTP_200_OK: - logger.error(f'Error pull repository. Status code: {request.status_code}. Reason: {request.text}') - - def create_repository_mirror(self, github_url: str, group_id: int): + def create_repository_mirror(self, github_url: str, group_id: str, auth_token: str): """ Base action for one thread. Creates repository, add mirror url and triggers pull at te end - :param github_url: Github url which will be mirrored + :param github_url: GitGub url which will be mirrored :param group_id: Gitlab group id which contains created repository + :param auth_token: GitGub token to access private repositories """ - repo_id = self.__create_new_project(github_url, group_id) - url = self.__add_pull_mirror(github_url, repo_id) - if url: - self.__pull_github_repo(url, repo_id) + + self.__create_new_project(github_url, group_id, auth_token) diff --git a/core/utils.py b/core/utils.py index 4a0e1ce..ab6abb1 100644 --- a/core/utils.py +++ b/core/utils.py @@ -38,7 +38,7 @@ def threads_ready_statistic(threads: List[Thread]): statistic = Counter(threads_statistic) ready_count = statistic.get(False, 0) percent = int(ready_count / len(threads) * 100) - time.sleep(1) + time.sleep(5) if 0 < percent < 100: logger.info(f'Ready: {percent}%') if not any(threads_statistic): diff --git a/github-repositories.txt b/github-repositories.txt index 8f6e25b..aebcc2f 100644 --- a/github-repositories.txt +++ b/github-repositories.txt @@ -1,5 +1,4 @@ https://github.com/30-seconds/30-seconds-of-python.git -https://github.com/Balshgit/sonar-scanner.git https://github.com/Delgan/loguru.git https://github.com/EbookFoundation/free-programming-books.git https://github.com/KristianOellegaard/django-health-check.git @@ -8,7 +7,6 @@ https://github.com/PyCQA/flake8.git https://github.com/SeleniumHQ/selenium.git https://github.com/SmileyChris/easy-thumbnails.git https://github.com/TheAlgorithms/Python.git -https://github.com/TheAlgorithms/Python.git https://github.com/TvoroG/pytest-lazy-fixture.git https://github.com/aio-libs/aiohttp.git https://github.com/aiogram/aiogram.git @@ -51,7 +49,6 @@ https://github.com/kamranahmedse/developer-roadmap.git https://github.com/kvesteri/validators.git https://github.com/lepture/captcha.git https://github.com/lorien/awesome-web-scraping.git -https://github.com/lorien/awesome-web-scraping.git https://github.com/marshmallow-code/marshmallow.git https://github.com/mher/flower.git https://github.com/numpy/numpy.git @@ -86,5 +83,4 @@ https://github.com/talkpython/modern-apis-with-fastapi.git https://github.com/talkpython/web-applications-with-fastapi-course.git https://github.com/vinta/awesome-python.git https://github.com/wemake-services/django-split-settings.git -https://github.com/wemake-services/django-split-settings.git https://github.com/wemake-services/wemake-django-template.git \ No newline at end of file diff --git a/github_mirror.py b/github_mirror.py index 841460b..6953aa7 100644 --- a/github_mirror.py +++ b/github_mirror.py @@ -1,5 +1,5 @@ import sys -from threading import Thread +from threading import Thread, Semaphore from core.argument_parser import create_parser from core.repo_creator import RepositoryCreator @@ -16,7 +16,7 @@ def main(): # parse urls if args.file: with open(f'{args.file}', mode='r') as file: - lines = [repo.strip() for repo in file] + lines = [line.replace('\n', '').strip() for line in file.readlines()] mirror_urls.extend(lines) if args.urls: mirror_urls.extend(args.urls) @@ -24,20 +24,24 @@ def main(): # parse gitlab group of repositories if it exists group_id = args.group if args.group else None - gitlab_url = args.gitlab # if not provided used default value https://git.do.x5.ru - headers = {'PRIVATE-TOKEN': args.token} # gitlab users token must be provided + git_url = args.giturl # if not provided used default value https://git.mywistr.com + headers = {'Authorization': f'token {args.token}'} # git user token must be provided - repository_creator = RepositoryCreator(gitlab_url=gitlab_url, headers=headers) + repository_creator = RepositoryCreator(gitlab_url=git_url, headers=headers) + + github_token = args.githubtoken if args.githubtoken else None # used for access to personal GitHub repositories threads = [] if mirror_urls: for url in set(mirror_urls): # github urls must be unique thread = Thread(target=repository_creator.create_repository_mirror, - kwargs={'github_url': url, 'group_id': group_id, }) - threads.append(thread) + kwargs={'github_url': url, 'group_id': group_id, 'auth_token': github_token, } + ) - for thread in threads: - thread.start() + threads.append(thread) + with Semaphore(10): + for thread in threads: + thread.start() threads_ready_statistic(threads) # add threads ready status to log output else: @@ -47,4 +51,3 @@ def main(): if __name__ == '__main__': main() -