initial commit

.gitignore

@@ -0,0 +1,246 @@
+#### joe made this: https://goel.io/joe
+
+# Git style-guide:
+# https://github.com/agis-/git-style-guide
+
+# enviroment
+.venv/
+venv/
+#####=== OSX ===#####
+.DS_Store
+.AppleDouble
+.LSOverride
+
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear on external disk
+.Spotlight-V100
+.Trashes
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+#####=== Windows ===#####
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+#####=== Python ===#####
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+#####=== Sass ===#####
+
+.sass-cache
+*.css.map
+
+#####=== Yeoman ===#####
+
+node_modules/
+bower_components/
+
+build/
+
+
+#####=== Vagrant ===#####
+.vagrant/
+
+#####=== Node ===#####
+
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git-
+node_modules
+
+# Debug log from npm
+npm-debug.log
+
+#### jetbrains ####
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+# You can uncomment these lines to enable configuration sharing between
+# team members, or you can restrict `.idea/` folder at all (default).
+
+# User-specific stuff:
+# .idea/**/workspace.xml
+# .idea/**/tasks.xml
+# .idea/dictionaries
+
+# # Sensitive or high-churn files:
+# .idea/**/dataSources/
+# .idea/**/dataSources.ids
+# .idea/**/dataSources.xml
+# .idea/**/dataSources.local.xml
+# .idea/**/sqlDataSources.xml
+# .idea/**/dynamic.xml
+# .idea/**/uiDesigner.xml
+
+# # Gradle:
+# .idea/**/gradle.xml
+# .idea/**/libraries
+
+# # Mongo Explorer plugin:
+# .idea/**/mongoSettings.xml
+
+# # Cursive Clojure plugin
+# .idea/replstate.xml
+
+# Restrict `.idea/` folder at all:
+.idea/
+
+# CMake
+cmake-build-debug/
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+
+#####=== Custom ===#####
+# Directories:
+media/
+.static/
+/static/
+
+# File types:
+*.sqlite3
+*.db
+
+# Configuration file with private data:
+*.env
+.env
+
+
+# Deploy files for Docker:
+docker-compose.deploy.yml
+
+# Certificates:
+docker/caddy/certs/
+
+# Artifacts:
+.gitlab/.svn/
+artifacts/
+
+# mypy:
+.mypy_cache/
+
+# pytest:
+.pytest_cache/
+
+# ipython:
+.ipython/

.python-version

@@ -0,0 +1 @@
+3.9.1

README.md

@@ -0,0 +1,44 @@
+# Github mirror creator
+
+Use python version > 3.8
+
+
+## Argumetns:
+
+- -h, --help -> ```Show help message and exit```
+
+
+- -g GROUP, --group GROUP -> ```Add GROUP id it can be found under group name. Id must be integer```
+
+
+- -u URL [URL ...], --urls URL [URL ...] 
+```Provide url or urls to mirror with it in format: https://github.com/s3rius/FastAPI-template.git You can provide multiple urls separated by space. Names will generate automatically from links```
+
+
+- -f FILE, --file FILE
+```Add file with urls. Each url on new line. Can be combined with --url option. Names will generate automatically from links```
+
+- -t TOKEN, --token TOKEN
+```Access token to gitlab API. More information:``` [gitlab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)
+
+- -l GITLAB, --gitlab GITLAB ```Provide gitlab url. Default link``` https://git.do.x5.ru
+
+
+## Usage 
+
+```python
+python3 github_mirror.py [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN
+```
+
+
+## Examples:
+
+    python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" -g 2059
+    
+    python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git"
+    
+    python3 github_mirror -f github_mirrors.txt -g 59563
+    
+    python3 github_mirror -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git"
+    
+    python3 gitlab_mirror.py --gitlab "https://gitlab.company.ru" -t "git-QwertY1245kde" -g 2059

core/__init__.py

@@ -0,0 +1,4 @@
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+requests.packages.urllib3.disable_warnings(InsecureRequestWarning)  # disable ssl warning

core/argument_parser.py

@@ -0,0 +1,55 @@
+from argparse import ArgumentParser
+
+GITLAB_URL = 'https://git.do.x5.ru'
+
+USAGE = '''github_mirror [-h] [-g GROUP] (-u URLS [URLS ...] | -f FILE) -t TOKEN
+--------------------------------------------------
+
+python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" -g 2059
+
+python3 github_mirror -u "https://github.com/s3rius/FastAPI-template.git" "https://github.com/sqlalchemy/sqlalchemy.git"
+
+python3 github_mirror -f github_mirrors.txt -g 59563
+
+python3 github_mirror -f github_mirrors.txt -u "https://github.com/s3rius/FastAPI-template.git"
+
+python3 gitlab_mirror.py --gitlab "https://gitlab.company.ru" -t "git-QwertY1245kde" -g 2059
+
+
+--------------------------------------------------
+'''
+
+
+def create_parser() -> ArgumentParser:
+    """
+    Create argparse parser
+
+    :return: command parser
+    """
+    parser = ArgumentParser(
+        prog='github_mirror',
+        description='''Script to add mirror repo into gitlab''',
+        epilog='''the developer is not responsible for the operation of the script :)''',
+        add_help=True,
+        usage=USAGE
+    )
+
+    parser.add_argument('-g', '--group', required=False, type=int,
+                        help='Add group id it can be found under group name. Id must be integer')
+
+    parser.add_argument('-u', '--urls', nargs='+', help='Provide url or urls to mirror with it in format: '
+                                                        'https://github.com/s3rius/FastAPI-template.git. '
+                                                        'You can provide multiple urls separated by space. '
+                                                        'Names will generate automatically from links')
+
+    parser.add_argument('-f', '--file', help='Add file with urls. Each url on new line. Can be combined with '
+                                             '--url option. Names will generate automatically from links')
+
+    parser.add_argument('-t', '--token', required=True,
+                        help='Access token to gitlab API. More information: https://docs.gitlab.com/ee/user/profile/'
+                             'personal_access_tokens.html#create-a-personal-access-token')
+
+    parser.add_argument('-l', '--gitlab', required=False, default=GITLAB_URL,
+                        help=f'Provide gitlab url. Default link {GITLAB_URL}')
+
+    return parser

core/repo_creator.py

@@ -0,0 +1,104 @@
+from typing import Union
+
+import requests
+from requests import Response
+
+from core.utils import logger
+
+
+class RepositoryCreator:
+
+    def __init__(self, gitlab_url: str, headers: dict):
+        self.gitlab_url = gitlab_url
+        self.headers = headers
+        self.HTTP_201_CREATED = 201
+        self.HTTP_200_OK = 200
+
+    def __gitlab_request(self, method: str, url: str, data: dict = None) -> Union[Response, None]:
+        """
+        Create request to gitlab
+
+        :param method: Request method can be changed
+        :param url: Url to request
+        :param data: Provide request data
+        :return: Response object on None
+        """
+        try:
+            request = requests.request(method, url, headers=self.headers, json=data, verify=False)
+            return request
+        except Exception as err:
+            logger.error(f'Connection not established. Check vpn is connected! \n{err}')
+
+    def __create_new_project(self, url: str, group_id: int = None) -> Union[str, None]:
+        """
+        Create new project in gitlab with name based on provided url
+
+        :param url: github url to mirror with:
+        :param group_id: namespace in gitlab to combine repos
+        :return: repo_id as string or None if any error
+        """
+
+        # name of repository will generate automatically from link
+        name = url.split('/')[-1].replace('.git', '')
+        git_data = {'name': name}
+        if group_id:
+            git_data['namespace_id'] = group_id
+
+        request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects', git_data)
+        try:
+            if request.status_code == self.HTTP_201_CREATED:
+                repo_data = request.json()
+                name_with_namespace = repo_data.get('name_with_namespace', None)
+                if name_with_namespace:
+                    logger.info(f'Repository {name_with_namespace} has been created')
+                else:
+                    logger.info(f'Repository {repo_data["name"]} has been created')
+                return repo_data['id']
+            else:
+                logger.error(f'Cant create new project. Status code: {request.status_code}. Reason: {request.text}')
+        except AttributeError:
+            pass
+
+    def __add_pull_mirror(self, url: str, repo_id: str) -> Union[str, None]:
+        """
+        Add pull mirror to Settings -> Repository -> Mirroring repositories
+
+        :param url: github url to mirror with
+        :param repo_id: id of repository which will be updated
+        :return: github url which will be mirrored
+        """
+
+        if repo_id:
+            git_data = {"mirror": True, "import_url": url}
+            request = self.__gitlab_request('PUT', f'{self.gitlab_url}/api/v4/projects/{repo_id}', git_data)
+            if request and request.status_code == self.HTTP_200_OK:
+                return url
+            elif request.status_code != self.HTTP_200_OK:
+                logger.error(f'Cant add mirror url to project. Status code: {request.status_code}. '
+                             f'Reason: {request.text}')
+
+    def __pull_github_repo(self, url: str, repo_id: str):
+        """
+        Initiate pull request for gitlab repository
+
+        :param url: github url to mirror with
+        :param repo_id: id of repository which will be updated
+        """
+
+        if repo_id:
+            request = self.__gitlab_request('POST', f'{self.gitlab_url}/api/v4/projects/{repo_id}/mirror/pull')
+            if request and request.status_code == self.HTTP_200_OK:
+                logger.info(f'Repository: {url} has been pulled')
+            elif request.status_code != self.HTTP_200_OK:
+                logger.error(f'Error pull repository. Status code: {request.status_code}. Reason: {request.text}')
+
+    def create_repository_mirror(self, **kwargs):
+        """
+        Base action for one thread. Creates repository, add mirror url and triggers pull at te end
+
+        :param kwargs: Can contain github url to mirror of, gitlab group ID
+        """
+        repo_id = self.__create_new_project(kwargs['url'], kwargs['group_id'])
+        url = self.__add_pull_mirror(kwargs['url'], repo_id)
+        if url:
+            self.__pull_github_repo(url, repo_id)

core/utils.py

@@ -0,0 +1,46 @@
+import importlib.util
+import logging
+import sys
+import time
+from collections import Counter
+from threading import Thread
+from typing import List
+
+# use loguru if it is possible for color output
+if importlib.util.find_spec('loguru') is not None:
+    from loguru import logger
+    logger.remove()
+    logger.add(sink=sys.stdout, colorize=True, level='DEBUG',
+               format="<cyan>{time:DD.MM.YYYY HH:mm:ss}</cyan> | <level>{level}</level> | "
+                      "<magenta>{message}</magenta>")
+
+# use standard logging
+else:
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.INFO)
+    log_formatter = logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
+    console_handler.setFormatter(log_formatter)
+
+    logger.addHandler(console_handler)
+
+
+def threads_ready_statistic(threads: List[Thread]):
+    """
+    Getting information how many threads are running right now
+
+    :param threads: List of active threads
+    """
+    while True:
+        threads_statistic = [thread.is_alive() for thread in threads]
+        statistic = Counter(threads_statistic)
+        ready_count = statistic.get(False, 0)
+        percent = int(ready_count / len(threads) * 100)
+        time.sleep(1)
+        if 0 < percent < 100:
+            logger.info(f'Ready: {percent}%')
+        if not any(threads_statistic):
+            logger.info(f'Ready: 100%')
+            break

github-repositories.txt

@@ -0,0 +1,76 @@
+https://github.com/s3rius/FastAPI-template.git
+https://github.com/sqlalchemy/sqlalchemy.git
+https://github.com/sqlalchemy/alembic.git
+https://github.com/jazzband/django-defender.git
+https://github.com/python-telegram-bot/python-telegram-bot.git
+https://github.com/sickcodes/Docker-OSX.git
+https://github.com/LonamiWebs/Telethon.git
+https://github.com/TheAlgorithms/Python.git
+https://github.com/wemake-services/wemake-django-template.git
+https://github.com/selfedu-rus/django-lessons.git
+https://github.com/vinta/awesome-python.git
+https://github.com/Balshgit/sonar-scanner.git
+https://github.com/kvesteri/validators.git
+https://github.com/jiaaro/pydub.git
+https://github.com/lorien/awesome-web-scraping.git
+https://github.com/TheAlgorithms/Python.git
+https://github.com/30-seconds/30-seconds-of-python.git
+https://github.com/spinda/Destroy-Windows-10-Spying.git
+https://github.com/talkpython/web-applications-with-fastapi-course.git
+https://github.com/talkpython/modern-apis-with-fastapi.git
+https://github.com/ohld/django-telegram-bot.git
+https://github.com/chrisk314/django-celery-docker-example.git
+https://github.com/danistefanovic/build-your-own-x.git
+https://github.com/EbookFoundation/free-programming-books.git
+https://github.com/kamranahmedse/developer-roadmap.git
+https://github.com/github/gitignore.git
+https://github.com/jlevy/the-art-of-command-line.git
+https://github.com/public-apis/public-apis.git
+https://github.com/wemake-services/django-split-settings.git
+https://github.com/sandix90/sqlalchemy_basics.git
+https://github.com/lorien/awesome-web-scraping.git
+https://github.com/aiogram/aiogram.git
+https://github.com/gildasio/h2t.git
+https://github.com/jazzband/django-debug-toolbar.git
+https://github.com/jazzband/django-axes.git
+https://github.com/psycopg/psycopg2.git
+https://github.com/samuelcolvin/pydantic.git
+https://github.com/python-pillow/Pillow.git
+https://github.com/asweigart/pyautogui.git
+https://github.com/Delgan/loguru.git
+https://github.com/aio-libs/aiohttp.git
+https://github.com/python-poetry/poetry.git
+https://github.com/SeleniumHQ/selenium.git
+https://github.com/henriquebastos/python-decouple.git
+https://github.com/pytest-dev/pytest.git
+https://github.com/pytest-dev/pytest-cov.git
+https://github.com/pytest-dev/pytest-mock.git
+https://github.com/pytest-dev/pytest-django.git
+https://github.com/pytest-dev/pytest-bdd.git
+https://github.com/pytest-dev/pytest-asyncio.git
+https://github.com/cookiecutter/cookiecutter.git
+https://github.com/proofit404/stories.git
+https://github.com/ansible/ansible.git
+https://github.com/lepture/captcha.git
+https://github.com/encode/starlette.git
+https://github.com/donnemartin/interactive-coding-challenges.git
+https://github.com/ipython/ipython.git
+https://github.com/benoitc/gunicorn.git
+https://github.com/encode/uvicorn.git
+https://github.com/python/mypy.git
+https://github.com/encode/django-rest-framework.git
+https://github.com/pyupio/safety.git
+https://github.com/jmcarp/nplusone.git
+https://github.com/celery/celery.git
+https://github.com/celery/django-celery.git
+https://github.com/celery/django-celery-beat.git
+https://github.com/celery/django-celery-results.git
+https://github.com/pika/pika.git
+https://github.com/pyca/bcrypt.git
+https://github.com/axnsan12/drf-yasg.git
+https://github.com/carltongibson/django-filter.git
+https://github.com/django/django.git
+https://github.com/KristianOellegaard/django-health-check.git
+https://github.com/wemake-services/django-split-settings.git
+https://github.com/jazzband/django-constance.git
+https://github.com/numpy/numpy.git

github_mirror.py

@@ -0,0 +1,51 @@
+import sys
+from threading import Thread, Semaphore
+
+from core.argument_parser import create_parser
+from core.repo_creator import RepositoryCreator
+from core.utils import logger, threads_ready_statistic
+
+
+def main():
+
+    parser = create_parser()
+    args = parser.parse_args(sys.argv[1:])
+
+    mirror_urls = []
+
+    # parse urls
+    if args.file:
+        with open(f'{args.file}', mode='r') as file:
+            lines = [repo.strip() for repo in file]
+        mirror_urls.extend(lines)
+    if args.urls:
+        mirror_urls.extend(args.urls)
+
+    # parse gitlab group of repositories if it exists
+    group_id = args.group if args.group else None
+
+    gitlab_url = args.gitlab  # if not provided used default value https://git.do.x5.ru
+    headers = {'PRIVATE-TOKEN': args.token}  # gitlab users token must be provided
+
+    repository_creator = RepositoryCreator(gitlab_url=gitlab_url, headers=headers)
+
+    threads = []
+    if mirror_urls:
+        for url in set(mirror_urls):  # github urls must be unique
+            thread = Thread(target=repository_creator.create_repository_mirror,
+                            kwargs={'url': url, 'group_id': group_id, })
+            threads.append(thread)
+
+        for thread in threads:
+            with Semaphore(50):
+                thread.start()
+
+        threads_ready_statistic(threads)  # add threads ready status to log output
+
+    else:
+        logger.info('You must provide urls to mirror')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()