initial commit

This commit is contained in:
2021-07-28 02:15:48 +03:00
commit 735633853a
6607 changed files with 1084121 additions and 0 deletions

View File

@@ -0,0 +1,8 @@
# from django.contrib import admin
#
# from server.apps.main.models import BlogPost
#
#
# @admin.register(BlogPost)
# class BlogPostAdmin(admin.ModelAdmin[BlogPost]):
# """Admin panel example for ``BlogPost`` model."""

View File

@@ -0,0 +1,28 @@
from celery import Celery
# from server.settings.components import config
from pathlib import Path
from decouple import AutoConfig
BASE_DIR = Path.cwd().parent.parent.parent.parent
config = AutoConfig(search_path=BASE_DIR.joinpath('config'))
RABBITMQ_DEFAULT_USER = config('RABBITMQ_DEFAULT_USER')
RABBITMQ_DEFAULT_PASS = config('RABBITMQ_DEFAULT_PASS')
RABBITMQ_PORT = config('RABBITMQ_PORT', cast=int, default=5672)
RABBITMQ_HOST = config('RABBITMQ_HOST')
celery_app = Celery(
'tasks',
broker='amqp://{login}:{password}@{host}:{port}'.format(
login=RABBITMQ_DEFAULT_USER,
password=RABBITMQ_DEFAULT_PASS,
host=RABBITMQ_HOST,
port=RABBITMQ_PORT,
),
backend='rpc://',
)
celery_app.autodiscover_tasks()

View File

@@ -0,0 +1,102 @@
import requests
from requests.models import Response
from requests.auth import HTTPBasicAuth
import re
import time
from functools import lru_cache
from typing import Dict, Optional
from server.apps.main.celery_config import celery_app
from server.settings.components.common import GIT_API_URL
from celery_progress.backend import ProgressRecorder
from celery import shared_task
from server.settings.components import config
def current_page(response: Response, link: str) -> int:
url = str(response.links[f'{link}']['url'])
page_count = int(str(re.findall(pattern=r'page=\d+', string=url)[1])
.replace('page=', ''))
return page_count
def github_request(url: str) -> Response:
auth = HTTPBasicAuth(config('GITHUB_USERNAME'), config('GITHUB_PASSWORD'))
counter = 0
while True:
try:
counter += 1
if auth == HTTPBasicAuth('', ''):
response = requests.get(url)
else:
response = requests.get(url, auth=auth)
return response
except ConnectionError as connection_error:
if counter < 5:
time.sleep(10)
else:
raise connection_error
@shared_task(bind=True)
def get_github_stars(self, username: str) -> Dict[str, Optional[int]]:
url = f'{GIT_API_URL}/{username}/repos?per_page=100&page=1'
print(url)
progress_recorder = ProgressRecorder(self)
response = github_request(url)
if response.status_code >= 400:
result = {}
else:
repos = response.json()
try:
page_count = current_page(response, 'last')
repos_count = (page_count - 1) * 100 + \
len(github_request(response.links['last']['url']).json())
except KeyError as e:
page_count = 1
repos_count = len(repos)
i = 0
while 'next' in response.links.keys():
i += 1
response = github_request(response.links['next']['url'])
repos.extend(response.json())
current = i * 100 + len(response.json())
# Progress bar
percent = round(100 / page_count * i)
progress_recorder.set_progress(current, repos_count,
description=f'Processing: {percent}%')
# Fetching repos and stars in dict
data: Dict[str, int] = {}
try:
for item in repos:
data[item['name']] = int(item['stargazers_count'])
result = dict(sorted(data.items(), key=lambda x: x[1], reverse=True))
except TypeError:
result = {}
return result
@shared_task(bind=True)
def process_download(self):
print('Task started')
# Create the progress recorder instance
# which we'll use to update the web page
progress_recorder = ProgressRecorder(self)
print('Start')
for i in range(5):
# Sleep for 1 second
time.sleep(1)
# Print progress in Celery task output
print(i + 1)
# Update progress on the web page
progress_recorder.set_progress(i + 1, 5, description='Downloading')
print('End')
return 'Task Complete'

View File

@@ -0,0 +1,8 @@
from django import forms
class GithubForm(forms.Form):
search_field = forms.CharField(max_length=20, required=True,
help_text='search github stars',
label='github_search',)

View File

@@ -0,0 +1,11 @@
"""
This package is a place for your business logic.
Please, do not create any other files inside your app package.
Place all files here, including: logic, forms, serializers.
Decoupling is a good thing. We need more of that.
Try using https://github.com/dry-python/
It is awesome for writing business logic!
"""

View File

@@ -0,0 +1,19 @@
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
class Command(BaseCommand):
def handle(self, *args, **options):
if not User.objects.filter(username='admin').exists():
username = 'admin'
email = 'admin@admin.ru'
password = 'admin'
admin = User.objects.create_superuser(username=username,
password=password,
email=email)
admin.is_active = True
admin.is_admin = True
admin.save()
else:
print('Admin accounts can only be initialized if no Accounts exist')

View File

@@ -0,0 +1,35 @@
# Generated by Django 2.2.7 on 2019-11-24 11:01
from django.db import migrations, models
class Migration(migrations.Migration):
"""Initial migration that creates the example BlogPost model."""
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name='BlogPost',
fields=[
(
'id',
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name='ID',
),
),
('title', models.CharField(max_length=80)),
('body', models.TextField()),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
],
options={
'verbose_name': 'BlogPost',
'verbose_name_plural': 'BlogPosts',
},
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.5 on 2021-07-16 10:35
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='blogpost',
name='id',
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.5 on 2021-07-16 11:11
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0002_alter_blogpost_id'),
]
operations = [
migrations.AlterField(
model_name='blogpost',
name='id',
field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.5 on 2021-07-16 11:14
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0003_alter_blogpost_id'),
]
operations = [
migrations.AlterField(
model_name='blogpost',
name='id',
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.5 on 2021-07-16 11:15
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0004_alter_blogpost_id'),
]
operations = [
migrations.AlterField(
model_name='blogpost',
name='id',
field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.5 on 2021-07-16 11:23
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0005_alter_blogpost_id'),
]
operations = [
migrations.AlterField(
model_name='blogpost',
name='id',
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.5 on 2021-07-16 11:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0006_alter_blogpost_id'),
]
operations = [
migrations.AlterField(
model_name='blogpost',
name='id',
field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
]

View File

@@ -0,0 +1,33 @@
import textwrap
from typing import Final, final
from django.db import models
#: That's how constants should be defined.
_POST_TITLE_MAX_LENGTH: Final = 80
@final
class BlogPost(models.Model):
"""
This model is used just as an example.
With it we show how one can:
- Use fixtures and factories
- Use migrations testing
"""
title = models.CharField(max_length=_POST_TITLE_MAX_LENGTH)
body = models.TextField()
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta(object):
verbose_name = 'BlogPost' # You can probably use `gettext` for this
verbose_name_plural = 'BlogPosts'
def __str__(self) -> str:
"""All django models should have this method."""
return textwrap.wrap(self.title, _POST_TITLE_MAX_LENGTH // 4)[0]

View File

@@ -0,0 +1,167 @@
class CeleryProgressBar {
constructor(progressUrl, options) {
this.progressUrl = progressUrl;
options = options || {};
let progressBarId = options.progressBarId || 'progress-bar';
let progressBarMessage = options.progressBarMessageId || 'progress-bar-message';
this.progressBarElement = options.progressBarElement || document.getElementById(progressBarId);
this.progressBarMessageElement = options.progressBarMessageElement || document.getElementById(progressBarMessage);
this.onProgress = options.onProgress || this.onProgressDefault;
this.onSuccess = options.onSuccess || this.onSuccessDefault;
this.onError = options.onError || this.onErrorDefault;
this.onTaskError = options.onTaskError || this.onTaskErrorDefault;
this.onDataError = options.onDataError || this.onError;
this.onRetry = options.onRetry || this.onRetryDefault;
this.onIgnored = options.onIgnored || this.onIgnoredDefault;
let resultElementId = options.resultElementId || 'celery-result';
this.resultElement = options.resultElement || document.getElementById(resultElementId);
this.onResult = options.onResult || this.onResultDefault;
// HTTP options
this.onNetworkError = options.onNetworkError || this.onError;
this.onHttpError = options.onHttpError || this.onError;
this.pollInterval = options.pollInterval || 500;
// Other options
let barColorsDefault = {
success: '#76ce60',
error: '#dc4f63',
progress: '#68a9ef',
ignored: '#7a7a7a'
}
this.barColors = Object.assign({}, barColorsDefault, options.barColors);
}
onSuccessDefault(progressBarElement, progressBarMessageElement, result) {
result = this.getMessageDetails(result);
progressBarElement.style.backgroundColor = this.barColors.success;
progressBarMessageElement.textContent = "Success! Please refresh page.";
}
onResultDefault(resultElement, result) {
if (resultElement) {
resultElement.textContent = result;
}
}
/**
* Default handler for all errors.
* @param data - A Response object for HTTP errors, undefined for other errors
*/
onErrorDefault(progressBarElement, progressBarMessageElement, excMessage, data) {
progressBarElement.style.backgroundColor = this.barColors.error;
excMessage = excMessage || '';
progressBarMessageElement.textContent = "Uh-Oh, something went wrong! " + excMessage;
}
onTaskErrorDefault(progressBarElement, progressBarMessageElement, excMessage) {
let message = this.getMessageDetails(excMessage);
this.onError(progressBarElement, progressBarMessageElement, message);
}
onRetryDefault(progressBarElement, progressBarMessageElement, excMessage, retryWhen) {
retryWhen = new Date(retryWhen);
let message = 'Retrying in ' + Math.round((retryWhen.getTime() - Date.now())/1000) + 's: ' + excMessage;
this.onError(progressBarElement, progressBarMessageElement, message);
}
onIgnoredDefault(progressBarElement, progressBarMessageElement, result) {
progressBarElement.style.backgroundColor = this.barColors.ignored;
progressBarMessageElement.textContent = result || 'Task result ignored!'
}
onProgressDefault(progressBarElement, progressBarMessageElement, progress) {
progressBarElement.style.backgroundColor = this.barColors.progress;
progressBarElement.style.width = progress.percent + "%";
var description = progress.description || "";
if (progress.current == 0) {
if (progress.pending === true) {
progressBarMessageElement.textContent = 'Waiting for task to start...';
} else {
progressBarMessageElement.textContent = 'Task started...';
}
} else {
progressBarMessageElement.textContent = progress.current + ' of ' + progress.total + ' processed. ' + description;
}
}
getMessageDetails(result) {
if (this.resultElement) {
return ''
} else {
return result || '';
}
}
/**
* Process update message data.
* @return true if the task is complete, false if it's not, undefined if `data` is invalid
*/
onData(data) {
let done = false;
if (data.progress) {
this.onProgress(this.progressBarElement, this.progressBarMessageElement, data.progress);
}
if (data.complete === true) {
done = true;
if (data.success === true) {
this.onSuccess(this.progressBarElement, this.progressBarMessageElement, data.result);
} else if (data.success === false) {
if (data.state === 'RETRY') {
this.onRetry(this.progressBarElement, this.progressBarMessageElement, data.result.message, data.result.when);
done = false;
delete data.result;
} else {
this.onTaskError(this.progressBarElement, this.progressBarMessageElement, data.result);
}
} else {
if (data.state === 'IGNORED') {
this.onIgnored(this.progressBarElement, this.progressBarMessageElement, data.result);
delete data.result;
} else {
done = undefined;
this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Data Error");
}
}
if (data.hasOwnProperty('result')) {
this.onResult(this.resultElement, data.result);
}
} else if (data.complete === undefined) {
done = undefined;
this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Data Error");
}
return done;
}
async connect() {
let response;
try {
response = await fetch(this.progressUrl);
} catch (networkError) {
this.onNetworkError(this.progressBarElement, this.progressBarMessageElement, "Network Error");
throw networkError;
}
if (response.status === 200) {
let data;
try {
data = await response.json();
} catch (parsingError) {
this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Parsing Error")
throw parsingError;
}
const complete = this.onData(data);
if (complete === false) {
setTimeout(this.connect.bind(this), this.pollInterval);
}
} else {
this.onHttpError(this.progressBarElement, this.progressBarMessageElement, "HTTP Code " + response.status, response);
}
}
static initProgressBar(progressUrl, options) {
const bar = new this(progressUrl, options);
bar.connect();
}
}

View File

@@ -0,0 +1,32 @@
class CeleryWebSocketProgressBar extends CeleryProgressBar {
constructor(progressUrl, options) {
super(progressUrl, options);
}
async connect() {
var ProgressSocket = new WebSocket((location.protocol === 'https:' ? 'wss' : 'ws') + '://' +
window.location.host + this.progressUrl);
ProgressSocket.onopen = function (event) {
ProgressSocket.send(JSON.stringify({'type': 'check_task_completion'}));
};
const bar = this;
ProgressSocket.onmessage = function (event) {
let data;
try {
data = JSON.parse(event.data);
} catch (parsingError) {
bar.onDataError(bar.progressBarElement, bar.progressBarMessageElement, "Parsing Error")
throw parsingError;
}
const complete = bar.onData(data);
if (complete === true || complete === undefined) {
ProgressSocket.close();
}
};
}
}

View File

@@ -0,0 +1,29 @@
body {
text-align: center;
}
img {
max-width: 100%;
height: auto;
}
.wemake-services-body {
height: 95vh;
display: flex;
flex-wrap: wrap;
flex-direction: column;
justify-content: center;
align-items: center;
}
.wemake-services-logo {
max-width: 250px;
margin: 0 auto;
}
.github-corner img {
position: absolute;
top: 0;
right: 0;
border: 0;
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.6 KiB

View File

@@ -0,0 +1,31 @@
{% load static %}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>Celery Progress Demo</title>
<meta name="Celery Progress Demo" content="" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<!-- Bootstrap CSS -->
<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css" rel="stylesheet">
</head>
<body>
{% block demo %}{% endblock %}
<!-- JQuery -->
<script src="https://code.jquery.com/jquery-3.5.1.min.js" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script>
<!-- Bootstrap JS -->
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/js/bootstrap.min.js"></script>
<!-- Celery Progress -->
<script src="{% static 'celery_progress/celery_progress.js' %}"></script>
{% block progress_bar_js %}{% endblock progress_bar_js %}
<div class="container text-center" style="padding-top: 20px;">
{{ message }}
{% for repo, stars in data.items %}
<br>
{{ repo }} {{ stars }}
{% endfor %}
</div>
</body>
</html>

View File

@@ -0,0 +1,17 @@
{% block content %}
<h1>Регистрация</h1>
<form method="post" action="">
{% csrf_token %}
<dl class="register">
{% for field in form %}
<dt>{{ field.label_tag }}</dt>
<dd class="clearfix">{{ field }}
{% if field.help_text %}<div class="clearfix">{{ field.help_text }}</div>{% endif %}
{% if field.errors %}<div class="myerrors clearfix">{{ field.errors }}</div>{% endif %}
</dd>
{% endfor %}
</dl>
<input type="submit" value="Зарегистрироваться" class="clearfix">
</form>
{% endblock %}

View File

@@ -0,0 +1,40 @@
{% extends "base.html" %}
{% load static %}
{% block demo %}
<!-- Reset Form -->
<div class="container text-center" style="padding-top: 20px;">
<form action="{% url 'demo' %}" method="GET" style="display: inline;">
{% csrf_token %}
<button class="btn btn-primary" type="submit" style="width:120px; background-color:#23e841; color:black">
<strong>REFRESH PAGE</strong>
</button>
</form>
</div>
<!-- Download Form -->
<div class="container text-center" style="padding-top: 20px;">
<form action="{% url 'demo' %}" method="post" style="display: inline;">
{% csrf_token %}
<dl class="register">
{% for field in form %}
<dt>{{ field.label_tag }}</dt>
<dd class="clearfix">{{ field }}
{% if field.help_text %}<div class="clearfix">{{ field.help_text }}</div>{% endif %}
{% if field.errors %}<div class="myerrors clearfix">{{ field.errors }}</div>{% endif %}
</dd>
{% endfor %}
</dl>
<button class="btn btn-primary" type="submit" style="width:120px; background-color:#23e841; color:black;">
<strong>Get Github stars</strong>
</button>
</form>
</div>
<!-- Download Status -->
<div class="container" style="padding-top: 20px;">
<div class="card" style="height: 120px;">
{% block progress %}{% endblock progress %}
</div>
</div>
{% endblock %}

View File

@@ -0,0 +1,25 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
</head>
<body>
{% block content %}
<h1>Github stars on repos</h1>
<form method="post" action="">
{% csrf_token %}
<dl class="register">
{% for field in form %}
<dt>{{ field.label_tag }}</dt>
<dd class="clearfix">{{ field }}
{% if field.help_text %}<div class="clearfix">{{ field.help_text }}</div>{% endif %}
{% if field.errors %}<div class="myerrors clearfix">{{ field.errors }}</div>{% endif %}
</dd>
{% endfor %}
</dl>
<input type="submit" value="Подсчёт звёзд" class="clearfix">
</form>
{% endblock %}
</body>
</html>

View File

@@ -0,0 +1,14 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Github results</title>
</head>
<body>
{{ message }}
{% for repo, stars in data.items %}
<br>
{{ repo }} {{ stars }}
{% endfor %}
</body>
</html>

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,26 @@
{% extends "download.html" %}
{% load static %}
{% block progress %}
<div class="text-center" style="font-size: 14px">
<div id="progress-bar-message">
Click the "Get Github stars" button
</div>
</div>
<div class='progress-wrapper' style="padding-top: 10px;">
<div id='progress-bar' class='progress-bar progress-bar-striped' role='progressbar' style="height:30px; width: 0%; border-radius: 5px">&nbsp;</div>
</div>
{% endblock progress %}
{% block progress_bar_js %}
{% if task_id %}
<script type="text/javascript">
// Progress Bar (JQuery)
$(function () {
var progressUrl = "{% url 'celery_progress:task_status' task_id %}";
CeleryProgressBar.initProgressBar(progressUrl, {})
});
</script>
{% endif %}
{% endblock progress_bar_js %}

View File

@@ -0,0 +1,14 @@
# The humans responsible & technology colophon
# http://humanstxt.org/
## balsh
Team:
## Technologies
Language: English
Doctype: HTML5
Technologies: Python, Django

View File

@@ -0,0 +1,2 @@
User-agent: *
Disallow:

View File

@@ -0,0 +1,9 @@
from django.urls import path
from server.apps.main.views import index
app_name = 'main'
urlpatterns = [
path('hello/', index, name='hello'),
]

View File

@@ -0,0 +1,107 @@
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render, redirect
from django.urls import reverse
from .forms import GithubForm
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from .commands import get_github_stars, process_download
from django.views.decorators.http import require_http_methods
from celery.result import AsyncResult
task_id = {}
def index(request: HttpRequest) -> HttpResponse:
"""
Main (or index) view.
Returns rendered default page to the user.
Typed with the help of ``django-stubs`` project.
"""
return render(request, 'main/index.html')
@login_required
def github(request: HttpRequest) -> HttpResponse:
username = str(request.user.username)
try:
email = getattr((User.objects.get(username=username)),
'email', 'default@email.ru')
except ObjectDoesNotExist as e:
error = 'That user doesnt exists or not log on'
print(error, e)
if request.method == 'POST':
github_username = str(request.POST.get('search_field'))
result = get_github_stars.delay(github_username)
task_id[username] = result.task_id
return redirect(reverse('github_result'))
form = GithubForm
return render(request, 'main/github.html',
context={'form': form})
@login_required
@require_http_methods(['GET'])
def github_result(request: HttpRequest) -> HttpResponse:
username = str(request.user.username)
data = AsyncResult(task_id[username])
if data.ready():
message = "Result Ready"
result = data.get()
print('result ready')
else:
print('result not ready')
return render(request, 'main/github_result.html',
context={'data': result,
'message': message})
def demo_view(request: HttpRequest) -> HttpResponse:
username = str(request.user.username)
form = GithubForm
result = {}
message = ''
if request.method == 'GET':
try:
data = AsyncResult(task_id[username])
if data.ready():
result = data.get()
message = f'Total repos: {len(result)}\n'
if len(result) == 0:
result = {'Error': 'User has no repositories!'}
print('Result ready! Please refresh page')
else:
print('result not ready')
except KeyError as e:
print(e)
finally:
# Return demo view
return render(request, 'progress.html',
context={'data': result, 'form': form,
'message': message})
elif request.method == 'POST':
message = 'Please wait'
github_username = str(request.POST.get('search_field'))
# Create Task
result = get_github_stars.delay(username=github_username)
# Get ID
task_id[username] = result.task_id
# Print Task ID
print(f'Celery Task ID: {task_id[username]}')
# Return demo view with Task ID
return render(request, 'progress.html',
context={'task_id': task_id[username],
'message': message,
'data': {}})