PersonalPublic/github_mirror
1
0
Fork 0
mirror of https://github.com/Balshgit/github_mirror.git synced 2025-09-10 23:20:41 +03:00
Code Issues Projects Releases Wiki Activity

gitea personal git script version

Browse Source
This commit is contained in:
Dmitry Afanasyev 2022-03-14 00:22:25 +03:00
parent 920e37b9ba
commit 77f515efe9
4 changed files with 72 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
core/argument_parser.py
View File

@ -1,6 +1,6 @@
from argparse import ArgumentParser
GITLAB_URL = 'https://git.do.x5.ru'
GIT_URL = 'https://git.mywistr.com'
USAGE = '''github_mirror [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN
--------------------------------------------------
@ -34,7 +34,7 @@ def create_parser() -> ArgumentParser:
usage=USAGE
)
parser.add_argument('-g', '--group', required=False, type=int,
parser.add_argument('-g', '--group', required=False, type=str,
help='Add group id it can be found under group name. Id must be integer')
parser.add_argument('-u', '--urls', nargs='+',
@ -49,7 +49,10 @@ def create_parser() -> ArgumentParser:
help='Access token to gitlab API. More information: https://docs.gitlab.com/ee/user/profile/'
'personal_access_tokens.html#create-a-personal-access-token')
parser.add_argument('-l', '--gitlab', required=False, default=GITLAB_URL,
help=f'Provide gitlab url. Default link {GITLAB_URL}')
parser.add_argument('-T', '--githubtoken', required=False, help='Please provide github token to get access '
'to private repositories')
parser.add_argument('-l', '--giturl', required=False, default=GIT_URL,
help=f'Provide gitlab url. Default link {GIT_URL}')
return parser

97
core/repo_creator.py
View File

@ -1,3 +1,4 @@
import random
from typing import Union
import requests
@ -23,32 +24,38 @@ class RepositoryCreator:
:param data: Provide request data
:return: Response object on None
"""
try:
request = requests.request(method, url, headers=self.headers, json=data, verify=False)
return request
except Exception as err:
logger.error(f'Connection not established. Check vpn is connected! \n{err}')
logger.error(f'Connection not established. \n{err}')
def __create_new_project(self, url: str, group_id: int = None) -> Union[str, None]:
def __create_new_project(self, url: str, group_name: str = None, auth_token: str = None) -> None:
"""
Create new project in gitlab with name based on provided url
:param url: github url to mirror with:
:param group_id: namespace in gitlab to combine repos
:param group_name: namespace in gitlab to combine repos
:param auth_token: github token to access private repositories
:return: repo_id as string or None if any error
"""
# name of repository will generate automatically from link
name = url.split('/')[-1].replace('.git', '')
git_data = {'name': name}
if group_id:
git_data['namespace_id'] = group_id
update_time = random.randint(48, 96)
git_data = {'repo_name': name, "wiki": True, "private": False, 'mirror_interval': f'{update_time}h0m0s',
"mirror": True, "lfs": True, "clone_addr": url}
if group_name:
git_data['repo_owner'] = group_name
if auth_token:
git_data['auth_token'] = auth_token
request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects', git_data)
request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v1/repos/migrate', git_data)
try:
if request.status_code == self.HTTP_201_CREATED:
repo_data = request.json()
name_with_namespace = repo_data.get('name_with_namespace', None)
name_with_namespace = repo_data.get('full_name', None)
if name_with_namespace:
logger.info(f'Repository {name_with_namespace} has been created')
else:
@ -59,47 +66,47 @@ class RepositoryCreator:
except AttributeError:
pass
def __add_pull_mirror(self, url: str, repo_id: str) -> Union[str, None]:
"""
Add pull mirror to Settings -> Repository -> Mirroring repositories
# def __add_pull_mirror(self, url: str, repo_id: str) -> Union[str, None]:
# """
# Add pull mirror to Settings -> Repository -> Mirroring repositories
#
# :param url: github url to mirror with
# :param repo_id: id of repository which will be updated
# :return: github url which will be mirrored
# """
#
# if repo_id:
# git_data = {"mirror": True, "import_url": url}
# request = self.__gitlab_request('PUT', f'{self.gitlab_url}/api/v4/projects/{repo_id}', git_data)
# if request and request.status_code == self.HTTP_200_OK:
# return url
# elif request.status_code != self.HTTP_200_OK:
# logger.error(f'Cant add mirror url to project. Status code: {request.status_code}. '
# f'Reason: {request.text}')
#
# def __pull_github_repo(self, url: str, repo_id: str):
# """
# Initiate pull request for gitlab repository
#
# :param url: github url to mirror with
# :param repo_id: id of repository which will be updated
# """
#
# if repo_id:
# request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects/{repo_id}/mirror/pull')
# if request and request.status_code == self.HTTP_200_OK:
# logger.info(f'Repository: {url} has been pulled')
# elif request.status_code != self.HTTP_200_OK:
# logger.error(f'Error pull repository. Status code: {request.status_code}. Reason: {request.text}')
:param url: github url to mirror with
:param repo_id: id of repository which will be updated
:return: github url which will be mirrored
"""
if repo_id:
git_data = {"mirror": True, "import_url": url}
request = self.__gitlab_request('PUT', f'{self.gitlab_url}/api/v4/projects/{repo_id}', git_data)
if request and request.status_code == self.HTTP_200_OK:
return url
elif request.status_code != self.HTTP_200_OK:
logger.error(f'Cant add mirror url to project. Status code: {request.status_code}. '
f'Reason: {request.text}')
def __pull_github_repo(self, url: str, repo_id: str):
"""
Initiate pull request for gitlab repository
:param url: github url to mirror with
:param repo_id: id of repository which will be updated
"""
if repo_id:
request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects/{repo_id}/mirror/pull')
if request and request.status_code == self.HTTP_200_OK:
logger.info(f'Repository: {url} has been pulled')
elif request.status_code != self.HTTP_200_OK:
logger.error(f'Error pull repository. Status code: {request.status_code}. Reason: {request.text}')
def create_repository_mirror(self, github_url: str, group_id: int):
def create_repository_mirror(self, github_url: str, group_id: str, auth_token: str):
"""
Base action for one thread. Creates repository, add mirror url and triggers pull at te end
:param github_url: Github url which will be mirrored
:param group_id: Gitlab group id which contains created repository
:param auth_token: Github token to access private repositories
"""
repo_id = self.__create_new_project(github_url, group_id)
url = self.__add_pull_mirror(github_url, repo_id)
if url:
self.__pull_github_repo(url, repo_id)
self.__create_new_project(github_url, group_id, auth_token)

2
core/utils.py
View File

@ -38,7 +38,7 @@ def threads_ready_statistic(threads: List[Thread]):
statistic = Counter(threads_statistic)
ready_count = statistic.get(False, 0)
percent = int(ready_count / len(threads) * 100)
time.sleep(1)
time.sleep(5)
if 0 < percent < 100:
logger.info(f'Ready: {percent}%')
if not any(threads_statistic):

20
github_mirror.py
View File

@ -1,5 +1,5 @@
import sys
from threading import Thread
from threading import Thread, Semaphore
from core.argument_parser import create_parser
from core.repo_creator import RepositoryCreator
@ -24,20 +24,24 @@ def main():
# parse gitlab group of repositories if it exists
group_id = args.group if args.group else None
gitlab_url = args.gitlab # if not provided used default value https://git.do.x5.ru
headers = {'PRIVATE-TOKEN': args.token} # gitlab users token must be provided
git_url = args.giturl # if not provided used default value https://git.mywistr.com
headers = {'Authorization': f'token {args.token}'} # gitlab users token must be provided
repository_creator = RepositoryCreator(gitlab_url=gitlab_url, headers=headers)
repository_creator = RepositoryCreator(gitlab_url=git_url, headers=headers)
github_token = args.githubtoken if args.githubtoken else None
threads = []
if mirror_urls:
for url in set(mirror_urls): # github urls must be unique
thread = Thread(target=repository_creator.create_repository_mirror,
kwargs={'github_url': url, 'group_id': group_id, })
threads.append(thread)
kwargs={'github_url': url, 'group_id': group_id, 'auth_token': github_token, }
)
for thread in threads:
thread.start()
threads.append(thread)
with Semaphore(10):
for thread in threads:
thread.start()
threads_ready_statistic(threads) # add threads ready status to log output
else: