From 491ccd1cac15de19972daeb6da286ae470f47d14 Mon Sep 17 00:00:00 2001 From: Dmitry Afanasyev Date: Sat, 4 Mar 2023 18:21:14 +0300 Subject: [PATCH] initial commit --- .gitignore | 146 +++++++++++++++++++++++++++++++++ README.md | 53 ++++++++++++ conf/Caddyfile | 15 ++++ conf/install-conf.yml | 33 ++++++++ delete.yml | 123 ++++++++++++++++++++++++++++ docker-compose.yml | 182 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 552 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 conf/Caddyfile create mode 100644 conf/install-conf.yml create mode 100644 delete.yml create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ecbc9cf --- /dev/null +++ b/.gitignore @@ -0,0 +1,146 @@ +### Python template + +.idea/ +.vscode/ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal +*.db + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# my staff +delete/ +delete.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..7961377 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# HA setup + +Simple docker-compose deployment to experiment with Cells v4 Clustering model. +It uses `pydio/cells:unstable` docker image, use whatever image by editing the docker-compose.yml file. + +## Preparing dependencies + +HA deployments relies on external dependencies to make Cells image fully stateless. +This sample creates the following images : MySQL, MongoDB, NATS.io, ETCD, Hashicorp Vault and Redis. + +This Vault requires a manual preparation for a specific key/value store (see below) + +```sh +cd +# start all third-party services +docker-compose up -d mysql mongo nats etcd vault redis minio caddy + +# create buckets in minio +docker-compose up createbuckets + +# Create a dedicated kvstore for certificates in Vault (configured in DEV mode with a preset VAULT_TOKEN, this should not be the case in production) +docker-compose exec -e VAULT_ADDR=http://localhost:8200 -e VAULT_TOKEN=secret_vault_token vault vault secrets enable -version=2 -path=caddycerts kv +``` + +## Starting Cells Nodes + +```sh +# Start one node, then open https://localhost:8080 to perform the install, it will read the conf/install-conf.yaml file +docker-compose up -d cells1; docker-compose logs -f cells1 +``` + +Now you can spin more cells nodes: +```sh +# Once install is finished, start other nodes +docker-compose up -d cells2 cells3; docker-compose logs -f cells2 cells3 +``` + +## Caddy LoadBalancer Access + +Caddy load balancer is configured in self-signed mode. +This requires adding localhost => caddy domain name to your local /etc/hosts file. + +Once started, it will monitor cells instances on /pprofs endpoint to automatically enable/disable upstreams. + +Access https://caddy:8585/ to access Cells. Enjoy! + +## Stopping cluster + +```sh +# To clean everything +docker-compose down -v --remove-orphan +``` + diff --git a/conf/Caddyfile b/conf/Caddyfile new file mode 100644 index 0000000..12cbb3c --- /dev/null +++ b/conf/Caddyfile @@ -0,0 +1,15 @@ +{ + https_port 8085 + skip_install_trust + local_certs +} +localhost { + reverse_proxy { + to https://cells1:8080 + transport http { + tls + tls_insecure_skip_verify + } + } + tls internal +} \ No newline at end of file diff --git a/conf/install-conf.yml b/conf/install-conf.yml new file mode 100644 index 0000000..7edbae7 --- /dev/null +++ b/conf/install-conf.yml @@ -0,0 +1,33 @@ +# This is the minimal configuration to directly start a pre-configured server in sandbox mode. +# Simply run 'docker-compose up -d', you can log in with admin/admin at https://localhost:8080 +# After accepting the self signed certificate + +# Adapt to your convenience and report to the documentation to explore further possibilities. + +# WebUI Admin definition +frontendlogin: admin +frontendpassword: admin + +# DB connection +dbconnectiontype: tcp +dbtcphostname: mysql +dbtcpport: 3306 +dbtcpname: cells +dbtcpuser: {$INSTALL_MYSQL_DB_USER} +dbtcppassword: {$INSTALL_MYSQL_DB_PASSWORD} + +# Mongo Setup +documentsdsn: mongodb://mongo:27017/cells +usedocumentsdsn: true + +# Advanced Datasource Setup +dstype: S3 +dss3custom: http://minio:9000 +dss3apikey: {$INSTALL_MINIO_ROOT_USER} +dss3apisecret: {$INSTALL_MINIO_ROOT_PASSWORD} +dss3bucketdefault: pydiods1 +dss3bucketpersonal: personal +dss3bucketcells: cellsdata +dss3bucketbinaries: binaries +dss3bucketthumbs: thumbs +dss3bucketversions: versions \ No newline at end of file diff --git a/delete.yml b/delete.yml new file mode 100644 index 0000000..375edc0 --- /dev/null +++ b/delete.yml @@ -0,0 +1,123 @@ +version: '3.9' + +# Common variables +x-cells: &cells-common + image: pydio/cells:4.1 + restart: unless-stopped + environment: + - CELLS_WORKING_DIR=/var/cells + - CELLS_BROKER=nats://nats:4222 + - CELLS_CACHE=redis://redis:6379 + - CELLS_BIND_ADDRESS=0.0.0.0 + +volumes: + cells_data: {} + mysql_data: {} + caddy_data: {} + caddy_logs: {} + caddy_config: {} + +services: + + # MySQL DB + mysql: + image: mysql:8.0.32 + restart: unless-stopped + volumes: + - mysql_data:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=cells + - MYSQL_DATABASE=cells + - MYSQL_USER=${MYSQL_DB_USER} + - MYSQL_PASSWORD=${MYSQL_DB_PASSWORD} + command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci] + + # Mongo DB + mongo: + image: mongo:6.0.4 + restart: unless-stopped + expose: [27017] + + # Nats events broker + nats: + image: nats:2.9.15 + expose: + - 4222 + - 8222 + #for verbose, use "--http_port 8222 -V" + command: "--http_port 8222" + + + redis: + image: redis:7.0.9 + hostname: redis + expose: [6379] + + minio: + image: quay.io/minio/minio + command: server --console-address ":9001" /data + hostname: minio + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + expose: + - 9000 + ports: + - "9001:9001" + volumes: + - cells_data:/data + + createbuckets: + image: minio/mc:RELEASE.2023-02-28T00-12-59Z + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add myminio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}; + /usr/bin/mc rm -r --force myminio/pydiods1; + /usr/bin/mc rm -r --force myminio/personal; + /usr/bin/mc rm -r --force myminio/cellsdata; + /usr/bin/mc rm -r --force myminio/thumbs; + /usr/bin/mc rm -r --force myminio/versions; + /usr/bin/mc rm -r --force myminio/binaries; + /usr/bin/mc mb myminio/pydiods1; + /usr/bin/mc mb myminio/personal; + /usr/bin/mc mb myminio/cellsdata; + /usr/bin/mc mb myminio/thumbs; + /usr/bin/mc mb myminio/versions; + /usr/bin/mc mb myminio/binaries; + exit 0; + " + # + # CELLS MULTIPLE NODES + # cells1 overrides env and volumes to access the install YAML + # + cells1: + <<: *cells-common + hostname: cells1 + expose: [8080] + environment: + - CELLS_WORKING_DIR=/var/cells + - CELLS_BROKER=nats://nats:4222 + - CELLS_CACHE=redis://redis:6379 + - CELLS_BIND_ADDRESS=0.0.0.0 + + - CELLS_INSTALL_YAML=/pydio/config/install.yml + - INSTALL_MYSQL_DB_USER=${MYSQL_DB_USER} + - INSTALL_MYSQL_DB_PASSWORD=${MYSQL_DB_PASSWORD} + - INSTALL_MINIO_ROOT_USER=${MINIO_ROOT_USER} + - INSTALL_MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + volumes: + - ./conf/install-conf.yml:/pydio/config/install.yml:ro + + + # Caddy reverse proxy, exposed as self-signed on port 445 + caddy: + image: caddy:2.6.4-alpine + ports: ["8085:8085"] + restart: unless-stopped + volumes: + - ./conf/Caddyfile:/etc/caddy/Caddyfile + - caddy_logs:/var/caddy/logs + - caddy_data:/data + - caddy_config:/config \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..db2a812 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,182 @@ +version: '3.9' + +# Common variables +x-cells: &cells-common + image: pydio/cells:4.1 + restart: unless-stopped + environment: + - CELLS_WORKING_DIR=/var/cells +# - CELLS_CONFIG=etcd://etcd:2379 +# - CELLS_REGISTRY=etcd://etcd:2379 + - CELLS_BROKER=nats://nats:4222 +# - CELLS_KEYRING=vault://vault:8200/secret?key=master +# - CELLS_CERTS_STORE=vault://vault:8200/caddycerts + - CELLS_CACHE=redis://redis:6379 +# - CELLS_ENABLE_PPROF=true + - CELLS_BIND_ADDRESS=0.0.0.0 +# - VAULT_TOKEN=${VAULT_ROOT_TOKEN} + +volumes: + cells_data: {} + mysql_data: {} + etcd_data: {} + caddy_data: {} + caddy_logs: {} + caddy_config: {} + +services: + + # MySQL DB + mysql: + image: mysql:8.0.32 + restart: unless-stopped + volumes: + - mysql_data:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=cells + - MYSQL_DATABASE=cells + - MYSQL_USER=${MYSQL_DB_USER} + - MYSQL_PASSWORD=${MYSQL_DB_PASSWORD} + command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci] + + # Mongo DB + mongo: + image: mongo:6.0.4 + restart: unless-stopped + expose: [27017] + +# # ETCD Single-Node for registry and configs +# etcd: +# image: quay.io/coreos/etcd:v3.5.2 +# entrypoint: /usr/local/bin/etcd +# ports: +# - 4001:4001 +# - 2380:2380 +# - 2379:2379 +# volumes: +# - etcd_data:/etcd_data +# - /usr/share/ca-certificates/:/etc/ssl/certs +# command: +# - '--name=etcd-0' +# - '--advertise-client-urls=http://etcd:2379,http://etcd:4001' +# - '--listen-client-urls=http://0.0.0.0:2379,http://0.0.0.0:4001' +# - '--initial-advertise-peer-urls=http://etcd:2380' +# - '--listen-peer-urls=http://0.0.0.0:2380' +# - '--initial-cluster-token=etcd-cluster-1' +# - '--initial-cluster=etcd-0=http://etcd:2380' +# - '--initial-cluster-state=new' + + # Nats events broker + nats: + image: nats:2.9.15 + expose: + - 4222 + - 8222 + #for verbose, use "--http_port 8222 -V" + command: "--http_port 8222" + +# # Hashicorp vault for keyring and certificates +# vault: +# image: vault +# expose: [8200] +# environment: +# - VAULT_DEV_ROOT_TOKEN_ID=${VAULT_ROOT_TOKEN} +# cap_add: +# - IPC_LOCK + + redis: + image: redis:7.0.9 + hostname: redis + expose: [6379] + + minio: + image: quay.io/minio/minio + command: server --console-address ":9001" /data + hostname: minio + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + expose: + - 9000 + ports: + - "9001:9001" + volumes: + - cells_data:/data + +# createbuckets: +# image: minio/mc:RELEASE.2023-02-28T00-12-59Z +# depends_on: +# - minio +# entrypoint: > +# /bin/sh -c " +# /usr/bin/mc config host add myminio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}; +# /usr/bin/mc rm -r --force myminio/pydiods1; +# /usr/bin/mc rm -r --force myminio/personal; +# /usr/bin/mc rm -r --force myminio/cellsdata; +# /usr/bin/mc rm -r --force myminio/thumbs; +# /usr/bin/mc rm -r --force myminio/versions; +# /usr/bin/mc rm -r --force myminio/binaries; +# /usr/bin/mc mb myminio/pydiods1; +# /usr/bin/mc mb myminio/personal; +# /usr/bin/mc mb myminio/cellsdata; +# /usr/bin/mc mb myminio/thumbs; +# /usr/bin/mc mb myminio/versions; +# /usr/bin/mc mb myminio/binaries; +# exit 0; +# " + # + # CELLS MULTIPLE NODES + # cells1 overrides env and volumes to access the install YAML + # + cells1: + <<: *cells-common + hostname: cells1 + expose: [8080] + environment: + - CELLS_WORKING_DIR=/var/cells +# - CELLS_CONFIG=etcd://etcd:2379 +# - CELLS_REGISTRY=etcd://etcd:2379 + - CELLS_BROKER=nats://nats:4222 +# - CELLS_KEYRING=vault://vault:8200/secret?key=master +# - CELLS_CERTS_STORE=vault://vault:8200/caddycerts + - CELLS_CACHE=redis://redis:6379 +# - CELLS_ENABLE_PPROF=true + - CELLS_BIND_ADDRESS=0.0.0.0 +# - VAULT_TOKEN=${VAULT_ROOT_TOKEN} + - CELLS_INSTALL_YAML=/pydio/config/install.yml + - INSTALL_MYSQL_DB_USER=${MYSQL_DB_USER} + - INSTALL_MYSQL_DB_PASSWORD=${MYSQL_DB_PASSWORD} + - INSTALL_MINIO_ROOT_USER=${MINIO_ROOT_USER} + - INSTALL_MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + volumes: + - ./conf/install-conf.yml:/pydio/config/install.yml:ro +# ports: +# - "8080:8080" # We expose that port externally for first configuration step (access to https://localhost:8080) +# - "8002:8002" # uncomment if you want to browse the internal registry of this node using cells-ctl + +# cells2: +# <<: *cells-common +# hostname: cells2 +# expose: [8080] +## uncomment if you want to open this instance directly on localhost:8082 +## ports: +## - 8082:8080 + +# cells3: +# <<: *cells-common +# hostname: cells3 +# expose: [8080] +## uncomment if you want to open this instance directly on localhost:8083 +## ports: +## - 8083:8080 + + # Caddy reverse proxy, exposed as self-signed on port 445 + caddy: + image: caddy:2.6.4-alpine + ports: ["8085:8085"] + restart: unless-stopped + volumes: + - ./conf/Caddyfile:/etc/caddy/Caddyfile + - caddy_logs:/var/caddy/logs + - caddy_data:/data + - caddy_config:/config \ No newline at end of file