Merge branch 'gitea'

README.md

@@ -8,7 +8,7 @@ Use python version > 3.8
 - -h, --help -> ```Show help message and exit```
 
 
-- -g GROUP, --group GROUP -> ```Add GROUP id it can be found under group name. Id must be integer```
+- -g GROUP, --group GROUP -> ```Add repository to GROUP name. It also named Organisation```
 
 
 - -u URL [URL ...], --urls URL [URL ...] 
@@ -19,9 +19,13 @@ Use python version > 3.8
 ```Add file with urls. Each url on new line. Can be combined with --url option. Names will generate automatically from links```
 
 - -t TOKEN, --token TOKEN
-```Access token to gitlab API. More information:``` [gitlab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)
+```Access token to gitea API. More information:``` [gitea docs](https://docs.gitea.io/en-us/api-usage/#authentication)
 
-- -l GITLAB, --gitlab GITLAB ```Provide gitlab url. Default link``` https://git.do.x5.ru
+- T GITHUBTOKEN, --githubtoken GITHUBTOKEN 
+- ```Please provide github token to get access to private repositories``` [github tokens](https://github.com/settings/tokens)
+
+
+- -l GITURL, --giturl GITURL ```Provide git url. Default link``` https://git.mywistr.com
 
 
 ## Usage 
@@ -33,12 +37,12 @@ python3 github_mirror.py [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN
 
 ## Examples:
 
-    python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" -g 2059 -t "git-QwertY1245kde"
+    python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" -g "GitHub" -t "gtb-QwertY1245kde"
     
-    python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "git-QwertY1245kde"
+    python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "gtb-QwertY1245kde"
     
-    python3 github_mirror -f github_mirrors.txt -g 59563 -t "git-QwertY1245kde"
+    python3 github_mirror.py -f github_mirrors.txt -g "Public" -t "gtb-QwertY1245kde"
     
-    python3 github_mirror -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "git-QwertY1245kde"
+    python3 github_mirror.py -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "gtb-QwertY125kde"
     
-    python3 github_mirror.py --gitlab "https://gitlab.company.ru" -t "git-QwertY1245kde" -g 2059 -u "https://github.com/s3rius/FastAPI-template.git"
+    python3 github_mirror.py --giturl "https://gitea.company.ru" -t "gtb-QwertY1245kde" -u "https://github.com/Balshgit/sonar-scanner.git" -g "Personal" -T "ghb-Qwerty321ldf"

core/argument_parser.py

@@ -1,19 +1,19 @@
 from argparse import ArgumentParser
 
-GITLAB_URL = 'https://git.do.x5.ru'
+GIT_URL = 'https://git.mywistr.com'
 
-USAGE = '''github_mirror [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN
+USAGE = '''github_mirror [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN [-T GitHubTOKEN]
 --------------------------------------------------
 
-python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" -g 2059 -t "git-QwertY1245kde"
+python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" -g "GitHub" -t "gtb-QwertY1245kde"
 
-python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "git-QwertY1245kde"
+python3 github_mirror.py -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git" -t "gtb-QwertY1245kde"
 
-python3 github_mirror -f github_mirrors.txt -g 59563 -t "git-QwertY1245kde"
+python3 github_mirror.py -f github_mirrors.txt -g "Public" -t "gtb-QwertY1245kde"
 
-python3 github_mirror -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "git-QwertY1245kde"
+python3 github_mirror.py -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git" -t "gtb-QwertY125kde"
 
-python3 github_mirror.py --gitlab "https://gitlab.company.ru" -t "git-QwertY1245kde" -g 2059
+python3 github_mirror.py --giturl "https://gitea.company.ru" -t "gtb-QwertY1245kde" -u "https://github.com/Balshgit/sonar-scanner.git" -g "Personal" -T "ghb-Qwerty321ldf"
 
 
 --------------------------------------------------
@@ -28,14 +28,14 @@ def create_parser() -> ArgumentParser:
     """
     parser = ArgumentParser(
         prog='github_mirror',
-        description='''Script to add mirror repo into gitlab''',
+        description='''Script to add mirror repo into gitea''',
         epilog='''the developer is not responsible for the operation of the script :)''',
         add_help=True,
         usage=USAGE
     )
 
-    parser.add_argument('-g', '--group', required=False, type=int,
+    parser.add_argument('-g', '--group', required=False, type=str,
-                        help='Add group id it can be found under group name. Id must be integer')
+                        help='Add repository to GROUP name. It also named Organisation')
 
     parser.add_argument('-u', '--urls', nargs='+',
                         help='Provide url or urls to mirror with it in format: '
@@ -46,10 +46,13 @@ def create_parser() -> ArgumentParser:
                                              '--url option. Names will generate automatically from links')
 
     parser.add_argument('-t', '--token', required=True,
-                        help='Access token to gitlab API. More information: https://docs.gitlab.com/ee/user/profile/'
+                        help='Access token to gitea API. More information: https://docs.gitea.io/en-us/api-usage/'
-                             'personal_access_tokens.html#create-a-personal-access-token')
+                             '#authentication')
 
-    parser.add_argument('-l', '--gitlab', required=False, default=GITLAB_URL,
+    parser.add_argument('-T', '--githubtoken', required=False, help='Please provide github token to get access '
-                        help=f'Provide gitlab url. Default link {GITLAB_URL}')
+                                                                    'to private repositories')
+
+    parser.add_argument('-l', '--giturl', required=False, default=GIT_URL,
+                        help=f'Provide git url where you want store your repositories. Default link {GIT_URL}')
 
     return parser

core/repo_creator.py

@@ -1,3 +1,4 @@
+import random
 from typing import Union
 
 import requests
@@ -23,83 +24,55 @@ class RepositoryCreator:
         :param data: Provide request data
         :return: Response object on None
         """
+
         try:
             request = requests.request(method, url, headers=self.headers, json=data, verify=False)
             return request
         except Exception as err:
-            logger.error(f'Connection not established. Check vpn is connected! \n{err}')
+            logger.error(f'Connection not established. \n{err}')
 
-    def __create_new_project(self, url: str, group_id: int = None) -> Union[str, None]:
+    def __create_new_project(self, url: str, group_name: str = None, auth_token: str = None) -> None:
         """
         Create new project in gitlab with name based on provided url
 
-        :param url: github url to mirror with:
+        :param url: GitHub url to mirror with:
-        :param group_id: namespace in gitlab to combine repos
+        :param group_name: namespace in gitlab to combine repos
+        :param auth_token: GitHub token to access private repositories
         :return: repo_id as string or None if any error
         """
 
         # name of repository will generate automatically from link
         name = url.split('/')[-1].replace('.git', '')
-        git_data = {'name': name}
+        update_time = random.randint(120, 580)  # prevent update all repos at the same time
-        if group_id:
+        git_data = {'repo_name': name, "wiki": True, "private": False, 'mirror_interval': f'{update_time}h0m0s',
-            git_data['namespace_id'] = group_id
+                    "mirror": True, "lfs": True, "clone_addr": url}
+        if group_name:
+            git_data['repo_owner'] = group_name
+        if auth_token:
+            git_data['auth_token'] = auth_token
 
-        request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects', git_data)
+        request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v1/repos/migrate', git_data)
         try:
             if request.status_code == self.HTTP_201_CREATED:
                 repo_data = request.json()
-                name_with_namespace = repo_data.get('name_with_namespace', None)
+                name_with_namespace = repo_data.get('full_name', None)
                 if name_with_namespace:
                     logger.info(f'Repository {name_with_namespace} has been created')
                 else:
                     logger.info(f'Repository {repo_data["name"]} has been created')
                 return repo_data['id']
             else:
-                logger.error(f'Cant create new project. Status code: {request.status_code}. Reason: {request.text}')
+                logger.error(f'Cant create {name} project. Status code: {request.status_code}. Reason: {request.text}')
         except AttributeError:
             pass
 
-    def __add_pull_mirror(self, url: str, repo_id: str) -> Union[str, None]:
+    def create_repository_mirror(self, github_url: str, group_id: str, auth_token: str):
-        """
-        Add pull mirror to Settings -> Repository -> Mirroring repositories
-
-        :param url: github url to mirror with
-        :param repo_id: id of repository which will be updated
-        :return: github url which will be mirrored
-        """
-
-        if repo_id:
-            git_data = {"mirror": True, "import_url": url}
-            request = self.__gitlab_request('PUT', f'{self.gitlab_url}/api/v4/projects/{repo_id}', git_data)
-            if request and request.status_code == self.HTTP_200_OK:
-                return url
-            elif request.status_code != self.HTTP_200_OK:
-                logger.error(f'Cant add mirror url to project. Status code: {request.status_code}. '
-                             f'Reason: {request.text}')
-
-    def __pull_github_repo(self, url: str, repo_id: str):
-        """
-        Initiate pull request for gitlab repository
-
-        :param url: github url to mirror with
-        :param repo_id: id of repository which will be updated
-        """
-
-        if repo_id:
-            request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects/{repo_id}/mirror/pull')
-            if request and request.status_code == self.HTTP_200_OK:
-                logger.info(f'Repository: {url} has been pulled')
-            elif request.status_code != self.HTTP_200_OK:
-                logger.error(f'Error pull repository. Status code: {request.status_code}. Reason: {request.text}')
-
-    def create_repository_mirror(self, github_url: str, group_id: int):
         """
         Base action for one thread. Creates repository, add mirror url and triggers pull at te end
 
-        :param github_url: Github url which will be mirrored
+        :param github_url: GitGub url which will be mirrored
         :param group_id: Gitlab group id which contains created repository
+        :param auth_token: GitGub token to access private repositories
         """
-        repo_id = self.__create_new_project(github_url, group_id)
+
-        url = self.__add_pull_mirror(github_url, repo_id)
+        self.__create_new_project(github_url, group_id, auth_token)
-        if url:
-            self.__pull_github_repo(url, repo_id)

core/utils.py

@@ -38,7 +38,7 @@ def threads_ready_statistic(threads: List[Thread]):
         statistic = Counter(threads_statistic)
         ready_count = statistic.get(False, 0)
         percent = int(ready_count / len(threads) * 100)
-        time.sleep(1)
+        time.sleep(5)
         if 0 < percent < 100:
             logger.info(f'Ready: {percent}%')
         if not any(threads_statistic):

github-repositories.txt

@@ -1,5 +1,4 @@
 https://github.com/30-seconds/30-seconds-of-python.git
-https://github.com/Balshgit/sonar-scanner.git
 https://github.com/Delgan/loguru.git
 https://github.com/EbookFoundation/free-programming-books.git
 https://github.com/KristianOellegaard/django-health-check.git
@@ -8,7 +7,6 @@ https://github.com/PyCQA/flake8.git
 https://github.com/SeleniumHQ/selenium.git
 https://github.com/SmileyChris/easy-thumbnails.git
 https://github.com/TheAlgorithms/Python.git
-https://github.com/TheAlgorithms/Python.git
 https://github.com/TvoroG/pytest-lazy-fixture.git
 https://github.com/aio-libs/aiohttp.git
 https://github.com/aiogram/aiogram.git
@@ -51,7 +49,6 @@ https://github.com/kamranahmedse/developer-roadmap.git
 https://github.com/kvesteri/validators.git
 https://github.com/lepture/captcha.git
 https://github.com/lorien/awesome-web-scraping.git
-https://github.com/lorien/awesome-web-scraping.git
 https://github.com/marshmallow-code/marshmallow.git
 https://github.com/mher/flower.git
 https://github.com/numpy/numpy.git
@@ -86,5 +83,4 @@ https://github.com/talkpython/modern-apis-with-fastapi.git
 https://github.com/talkpython/web-applications-with-fastapi-course.git
 https://github.com/vinta/awesome-python.git
 https://github.com/wemake-services/django-split-settings.git
-https://github.com/wemake-services/django-split-settings.git
 https://github.com/wemake-services/wemake-django-template.git

github_mirror.py

@@ -1,5 +1,5 @@
 import sys
-from threading import Thread
+from threading import Thread, Semaphore
 
 from core.argument_parser import create_parser
 from core.repo_creator import RepositoryCreator
@@ -16,7 +16,7 @@ def main():
     # parse urls
     if args.file:
         with open(f'{args.file}', mode='r') as file:
-            lines = [repo.strip() for repo in file]
+            lines = [line.replace('\n', '').strip() for line in file.readlines()]
         mirror_urls.extend(lines)
     if args.urls:
         mirror_urls.extend(args.urls)
@@ -24,18 +24,22 @@ def main():
     # parse gitlab group of repositories if it exists
     group_id = args.group if args.group else None
 
-    gitlab_url = args.gitlab  # if not provided used default value https://git.do.x5.ru
+    git_url = args.giturl  # if not provided used default value https://git.mywistr.com
-    headers = {'PRIVATE-TOKEN': args.token}  # gitlab users token must be provided
+    headers = {'Authorization': f'token {args.token}'}  # git user token must be provided
 
-    repository_creator = RepositoryCreator(gitlab_url=gitlab_url, headers=headers)
+    repository_creator = RepositoryCreator(gitlab_url=git_url, headers=headers)
+
+    github_token = args.githubtoken if args.githubtoken else None  # used for access to personal GitHub repositories
 
     threads = []
     if mirror_urls:
         for url in set(mirror_urls):  # github urls must be unique
             thread = Thread(target=repository_creator.create_repository_mirror,
-                            kwargs={'github_url': url, 'group_id': group_id, })
+                            kwargs={'github_url': url, 'group_id': group_id, 'auth_token': github_token, }
-            threads.append(thread)
+                            )
 
+            threads.append(thread)
+        with Semaphore(10):
             for thread in threads:
                 thread.start()
 
@@ -47,4 +51,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-