dockerhub.py

@@ -2,7 +2,7 @@ import asyncio
 import re
 import sys
 from logging import Logger
-from multiprocessing import Process, Lock
+from multiprocessing import Process
 from typing import Any
 
 import httpx
@@ -10,47 +10,19 @@ from httpx import AsyncHTTPTransport, AsyncClient
 from packaging.version import parse as parse_version
 from termcolor import colored
 
-SERVICES: dict[str: dict[str, Any]] = {
+SERVICES = {
-    'general': {
+    'nextcloud': '25.0.4',
-        'components': [
+    'gitea/gitea': '1.18.5',
-            {'name': 'caddy', 'version': '2.7.5'},
+    'caddy': '2.6.4',
-            {'name': 'python', 'version': '3.11.5'},
+    'mediawiki': '1.39.2',
-        ]
+    'bitwarden/server': '2023.2.0',
-    },
+    'redis': '7.0.8',
-    'nextcloud': {
+    'nginx': '1.23.3',
-        'components': [
+    'mariadb': '10.11.2',
-            {'name': 'nextcloud', 'version': '27.1.2'},
+    'postgres': '15.2',
-            {'name': 'mysql', 'version': '8.1.0'},
+    'mysql': '8.0.32',
-            {'name': 'redis', 'version': '7.2.1'},
+    'selenoid/firefox': '110.0',
-            {'name': 'nginx', 'version': '1.25.2'},
+    'python': '3.11.1',
-            {'name': 'onlyoffice/documentserver', 'version': '7.3.3.50'},
-        ],
-    },
-    'gitea': {
-        'components': [
-            {'name': 'gitea/gitea', 'version': '1.20.5'},
-            {'name': 'postgres', 'version': '15.4'},
-        ],
-    },
-    'mediawiki': {
-        'components': [
-            {'name': 'mediawiki', 'version': '1.40.1'},
-            {'name': 'mariadb', 'version': '11.1.2'},
-        ],
-    },
-    'bitwarden': {
-        'components': [
-            {'name': 'bitwarden/web', 'version': '2023.9.1'},
-            {'name': 'bitwarden/server', 'version': '2023.9.0'},
-        ],
-    },
-    'mosgortrans': {
-        'deprecated': True,
-        'components': [
-            {'name': 'selenoid/chrome', 'version': '111.0'},
-            {'name': 'aerokube/selenoid', 'version': '1.10.10'},
-        ],
-    },
 }
 
 
@@ -85,106 +57,16 @@ logger = configure_logger()
 
 
 class DockerHubScanner:
-    """Url server examples:
-    bitwarden/server
-    https://hub.docker.com/v2/namespaces/bitwarden/repositories/server/tags?page=2
 
-    caddy
+    # bitwarden/server
-    https://registry.hub.docker.com/v2/repositories/library/caddy/tags?page=1
+    # https://hub.docker.com/v2/namespaces/bitwarden/repositories/server/tags?page=2
-    """
+
+    # caddy
+    # https://registry.hub.docker.com/v2/repositories/library/caddy/tags?page=1
 
     DOCKERHUB_REGISTRY_API = 'https://registry.hub.docker.com/v2/repositories/library'
     DOCKERHUB_API = 'https://hub.docker.com/v2/namespaces'
 
-    async def get_tags(self, service_name: str, service_component: dict[str, str]) -> dict[str, list[str]]:
-        """
-        To make method really async it should be rewritten on pages not by get next page each time.
-        Also, dockerhub protected from bruteforce requests.
-        Better with getting next page each time
-        """
-        component_name = service_component['name']
-        component_version = service_component['version']
-
-        url = self._docker_hub_api_url(component_name)
-        all_tags = []
-        transport = AsyncHTTPTransport(retries=1)
-        async with AsyncClient(transport=transport) as client:
-            payload = await self._async_request(client=client, url=url)
-
-            if not payload:
-                return {component_name: all_tags}
-
-            next_page, tags = self._get_next_page_and_tags_from_payload(payload)
-
-            all_tags.extend(tags)
-
-            while component_version not in all_tags:
-                if not next_page:
-                    break
-                payload = await self._async_request(client=client, url=next_page)
-                next_page, tags = self._get_next_page_and_tags_from_payload(payload)
-                all_tags.extend(tags)
-
-            # filter tags contains versions 1.18.3 and not contains letters 1.18.3-fpm-alpine. Sort by version number
-            tags = sorted(
-                list(
-                    filter(
-                        lambda t: re.search(r'\d+\.\d', t) and not re.search(r'[a-z]', t),
-                        all_tags,
-                    )
-                ),
-                reverse=True,
-                key=parse_version,
-            )
-
-            # Do not show older versions than current in tags
-            try:
-                tags = tags[:tags.index(component_version) + 1]
-            except ValueError:
-                tags = tags[:3]
-                logger.error(
-                    f"cant find tag {component_version} for service {service_name}"
-                    f"for component {component_name}"
-                )
-        return {component_name: tags}
-
-    def get_data(self, service_name: str, service_component: dict[str, str]) -> dict[str, list[str]]:
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        services_tags = loop.run_until_complete(self.get_tags(service_name, service_component))
-
-        return services_tags
-
-    def print_data(self, service_name: str, service_component: dict[str, str], lock: Lock) -> None:
-
-        component_name = service_component['name']
-        component_version = service_component['version']
-
-        data = self.get_data(service_name, service_component)
-
-        with lock:
-            print(
-                f"Service: {colored(service_name, color='light_grey')}",
-                f"\nComponent: {colored(component_name, color='light_blue')}",
-                f"\nLatest tags: {colored(str(data[component_name]), color='magenta')}",
-                f"\nCurrent version: {colored(component_version, color='cyan')}",
-            )
-
-            if data[component_name][0] > component_version:
-                print(f"New version of {component_name}: {colored(data[component_name][0], color='yellow')}")
-            print()
-
-    async def _async_request(self, client: AsyncClient, url: str) -> dict[str, Any] | None:
-
-        response = await client.get(url)
-        status = response.status_code
-        if status == httpx.codes.OK:
-            return response.json()
-        else:
-            uri = url.replace(self.DOCKERHUB_REGISTRY_API, '').replace(self.DOCKERHUB_API, '')
-            logger.info(f'got response status: {status} for uri: {uri}')
-        return None
-
     def _docker_hub_api_url(self, service_name: str) -> str:
 
         if '/' in service_name:
@@ -194,35 +76,89 @@ class DockerHubScanner:
             url = f'{self.DOCKERHUB_REGISTRY_API}/{service_name}/tags'
         return url
 
+    @staticmethod
+    async def _async_request(client: AsyncClient, url: str) -> dict[str, Any] | None:
+
+        response = await client.get(url)
+        status = response.status_code
+        if status == httpx.codes.OK:
+            return response.json()
+        return None
+
     @staticmethod
     def _get_next_page_and_tags_from_payload(payload: dict[str, Any]) -> tuple[str | None, list[str]]:
         next_page = payload['next']
         names = [release['name'] for release in payload['results']]
         return next_page, names
 
+    async def get_tags(self, service_name: str) -> dict[str, list[str]]:
+        """
+        To make method really async it should be rewritten on pages not by get next page each time.
+        Also, dockerhub protected from bruteforce requests.
+        Better with getting next page each time
+        """
+
+        tags = []
+        url = self._docker_hub_api_url(service_name)
+        transport = AsyncHTTPTransport(retries=1)
+        async with AsyncClient(transport=transport) as client:
+            payload = await self._async_request(client=client, url=url)
+
+            if not payload:
+                return {service_name: tags}
+
+            next_page, names = self._get_next_page_and_tags_from_payload(payload)
+
+            tags.extend(names)
+
+            while SERVICES[service_name] not in tags:
+                payload = await self._async_request(client=client, url=next_page)
+                next_page, names = self._get_next_page_and_tags_from_payload(payload)
+                tags.extend(names)
+
+            # filter tags contains versions 1.18.3 and not contains letters 1.18.3-fpm-alpine. Sort by version number
+            tags = sorted(
+                list(filter(lambda t: re.search(r'\d+\.\d', t) and not re.search(r'[a-z]', t), tags)),
+                reverse=True,
+                key=parse_version,
+            )
+
+            # Do not show older versions than current in tags
+            tags = tags[:tags.index(SERVICES[service_name]) + 1]
+        return {service_name: tags}
+
+    def get_data(self, service_name: str) -> dict[str, list[str]]:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        services_tags = loop.run_until_complete(self.get_tags(service_name))
+
+        return services_tags
+
+    def print_data(self, service_name: str) -> None:
+
+        data = self.get_data(service_name)
+        print(
+            f"Service: {colored(service_name, color='light_grey')}",
+            f"\nTags: {colored(str(data[service_name]), color='magenta')}",
+            f"\nCurrent version: {colored(SERVICES[service_name], color='cyan')}"
+        )
+
+        if data[service_name][0] > SERVICES[service_name]:
+            print(f"New version of {service_name}: {colored(data[service_name][0], color='yellow')}")
+        print()
+
 
 if __name__ == '__main__':
 
-    print(colored('Services'.center(50, '-', ), color='white'), '\n')
+    print('Services'.center(50, '-'), '\n')
-
-    global_lock = Lock()
 
     dockerhub_scanner = DockerHubScanner()
     processes = []
 
-    for service, service_details in SERVICES.items():
+    for service in SERVICES:
-        for component in service_details['components']:
+        process = Process(target=dockerhub_scanner.print_data, kwargs={'service_name': service})
-            if service_details.get('deprecated', False):
-                continue
-            process = Process(
-                target=dockerhub_scanner.print_data,
-                kwargs={'service_name': service, 'service_component': component, 'lock': global_lock}
-            )
         processes.append(process)
         process.start()
 
     for process in processes:
         process.join()
-
-    print(colored("All jobs done", color='white'))
-

goodgame.py

@@ -166,15 +166,15 @@ class GoodGame:
     def sync_counter(self) -> str:
         page = 1
 
-        response = requests.get(f'{self.BASE_URL}?page={page}', timeout=2)
+        resp = requests.get(f'{self.BASE_URL}?page={page}')
-        streams = response.json()['streams']
+        streams = resp.json()['streams']
         for stream in streams:
             self.all_streams.update({stream['id']: stream})
         max_current_viewers = streams[0]['viewers']
         while streams:
             page += 1
-            response = requests.get(f'{self.BASE_URL}?page={page}')
+            resp = requests.get(f'{self.BASE_URL}?page={page}')
-            streams = response.json()['streams']
+            streams = resp.json()['streams']
             for stream in streams:
                 self.all_streams.update({stream['id']: stream})
         return self.__prepare_result(max_current_viewers)
@@ -184,18 +184,17 @@ if __name__ == '__main__':
     print("-" * 76)
     good_game = GoodGame()
     start = time.time()
-    good_game.async_counter()
+    async_process = Process(
-    # async_process = Process(
+        target=good_game.async_counter, args=(), kwargs={}, name='async_process'
-    #     target=good_game.async_counter, args=(), kwargs={}, name='async_process'
+    )
-    # )
+    sync_process = Process(
-    # sync_process = Process(
+        target=good_game.sync_counter, args=(), kwargs={}, name='sync_process'
-    #     target=good_game.sync_counter, args=(), kwargs={}, name='sync_process'
+    )
-    # )
 
-    # sync_process.start()
+    async_process.start()
-    # async_process.start()
+    sync_process.start()
-    # sync_process.join()
-    # async_process.join()
 
+    async_process.join()
+    sync_process.join()
     stop = time.time()
     logger.info(f'End all processes. Execution time: {round(stop-start, 2)} seconds')