diff --git a/Dockerfile b/Dockerfile index a810420..f8fdd2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ +# See docker/README for how to use this. + FROM debian:stretch LABEL maintainer="Michael Halstead " -EXPOSE 80 ENV PYTHONUNBUFFERED=1 \ LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 \ @@ -30,7 +31,6 @@ RUN apt-get install -y --no-install-recommends \ libjpeg-dev \ libmariadbclient-dev \ locales \ - rabbitmq-server \ netcat-openbsd \ curl \ git-core \ @@ -49,7 +49,8 @@ RUN apt-get purge -y autoconf g++ make python3-dev libjpeg-dev libmariadbclient- && rm -rf /var/lib/apt/lists/* \ && apt-get clean COPY . /opt/layerindex -COPY settings.py /opt/layerindex/settings.py +COPY docker/settings.py /opt/layerindex/settings.py +COPY docker/refreshlayers.sh /opt/refreshlayers.sh COPY docker/updatelayers.sh /opt/updatelayers.sh COPY docker/migrate.sh /opt/migrate.sh diff --git a/Dockerfile.web b/Dockerfile.web index e73974a..e83aa04 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,4 +1,3 @@ FROM nginx:latest LABEL maintainer="Michael Halstead " COPY docker/nginx.conf /etc/nginx/nginx.conf -COPY layerindex/static /usr/share/nginx/html/static diff --git a/docker/README b/docker/README index 05a27ed..9c24421 100644 --- a/docker/README +++ b/docker/README @@ -1,26 +1,85 @@ -## This is set up to make a cluster of three containers. First we build two from the root of the repo. +## Layerindex example docker instructions + +## This is set up to make a cluster of 5 containers: +## - layersapp: the application +## - layersdb: the database +## - layersweb: NGINX web server (as a proxy and for serving static content) +## - layerscelery: Celery (for running background jobs) +## - layersrabbit: RabbitMQ (required by Celery) + +## First, find and replace layers.openembedded.org below with your hostname +## You'll probably also want to replace the database password "testingpw". +## If you are using a version of Docker older than 17.06 then you'll need to replace --mount src=,dst= with -v : + +## If you want to change any of the application configuration, edit docker/settings.py as desired. +## Some settings have been set so that values can be passed in via environment variables. +## You will definitely need to set SECRET_KEY. + +## If you are on a network that requires a proxy to get out to the internet, then you'll need to: +## - Uncomment several lines in Dockerfile (search for "proxy") +## - Edit docker/.gitconfig and docker/git-proxy + +## Build the main container from the root of the repo. docker build -t halstead/layerindex-app . + +## Build the static web server container +## (for SSL, first move docker/nginx-ssl.conf to docker/nginx.conf and edit as needed.) docker build -t halstead/layerindex-web -f Dockerfile.web . -## Start a database server. We use MariaDB in production. -## In order to configure your settings.py file to use this database server, use: -## 'ENGINE': 'django.db.backends.mysql', -## 'NAME': 'layersdb', -## 'USER': 'root', -## 'PASSWORD': 'testingpw', -## 'HOST': 'layersdb', -## 'PORT': '', -docker run -d --name layerdb -e MYSQL_ROOT_PASSWORD=testingpw -e MYSQL_DATABASE=layersdb mariadb --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci +## Add a network for our containers +docker network create layerindex -## If you have a copy of the the production data now is the time to insert it. -## If not you can skip the next step for a clean install. -xzcat ./layerdb.sql.xz | docker run -i --link layerdb:layersdb --rm mariadb sh -c 'exec mysql -hlayersdb -uroot -p"testingpw" layersdb' +## Start a database server - here we use MariaDB, though you can obviously use something else and change docker/settings.py as appropriate +# run one of the following. +## To use an existing dump run the following and wait for startup and import +docker run --detach --name layersdb --network layerindex --mount src=layers-database-dump.sql,dst=/docker-entrypoint-initdb.d/layerdb.sql -e MYSQL_ROOT_PASSWORD=testingpw mariadb:10.2 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci +## Or to start fresh +docker run --detach --name layersdb --network layerindex -e MYSQL_DATABASE=layersdb -e MYSQL_ROOT_PASSWORD=testingpw mariadb:10.2 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci -docker run -d --link layerdb:layersdb --name layersapp halstead/layerindex-app -docker run -d --link layersapp:layersapp --name layersweb -p 49153:80 halstead/layerindex-web +## Start the RabbitMQ container +docker run --detach --network layerindex --name layersrabbit rabbitmq:alpine -## To apply layerindex migration -docker run --rm --link layerdb:layersdb halstead/layerindex-app /opt/migrate.sh +## Start the Celery container +docker run --detach --network layerindex --name layerscelery -e DATABASE_PASSWORD=testingpw -e DATABASE_HOST=layersdb halstead/layerindex-app /usr/local/bin/celery -A layerindex.tasks worker --loglevel=info --workdir=/opt/layerindex -## To update the layer info we can run the job in a temporary container. -docker run --rm --link layerdb:layersdb halstead/layerindex-app /opt/updatelayers.sh + +## Apply any pending layerindex migrations / initialize the database +docker run --rm --network layerindex --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app /opt/migrate.sh + +## For a fresh database, create an admin account +docker run --rm -it --network layerindex --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app /opt/layerindex/manage.py createsuperuser + + +## Start the layerindex application +docker run --detach --network layerindex --name layersapp --hostname layers.openembedded.org -e DATABASE_PASSWORD=testingpw -e DATABASE_HOST=layersdb halstead/layerindex-app + +## Create a volume for static assets +docker volume create layersstatic + +## Set the volume permissions using debian:stretch since we recently fetched it +docker run --mount src=layersstatic,dst=/usr/share/nginx/html debian:stretch chown 500 /usr/share/nginx/html + +## Generate static assets. Run this command again to regenerate at any time (when static assets in the code are updated) +docker run --env STATIC_ROOT=/usr/share/nginx/html -ti --rm --network layerindex --hostname layers.openembedded.org --name generatestatic --mount src=layersstatic,dst=/usr/share/nginx/html --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app python3 /opt/layerindex/manage.py collectstatic + +## Start the reverse proxy +## run one of the following: +## A) for local/test use forward port 8080: +docker run --detach --network layerindex -p 8080:80 --mount src=layersstatic,dst=/usr/share/nginx/html --name layersweb --hostname layers.openembedded.org halstead/layerindex-web +## B) with SSL for production: +# Make sure your DNS is setup and then run the following to get the certs +docker run -it --rm -p 80:80 -p 443:443 --name certbot --mount src=layerscerts,dst=/etc/letsencrypt --mount src=certbotvar,dst=/var/lib/letsencrypt certbot/certbot certonly #renew +# then start the proxy with ssl +docker run --detach --network layerindex -p 80:80 -p 443:443 --mount src=layersstatic,dst=/usr/share/nginx/html --mount src=layerscerts,dst=/etc/letsencrypt --name layersweb --hostname layers.openembedded.org halstead/layerindex-web + +## Create a workdir to prevent downloading repos fresh each time +docker volume create update-workdir + +## Set the volume permissions using debian:stretch since we recently fetched it +docker run --mount src=update-workdir,dst=/opt/workdir debian:stretch chown 500 /opt/workdir + +## Run the layer updates +docker run --rm --network layerindex --hostname updatelayers.openembedded.org --name updatelayers-throwaway --mount src=update-workdir,dst=/opt/workdir --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app python3 /opt/layerindex/layerindex/update.py + +## Or do a full refresh +docker run --rm --network layerindex --hostname updatelayers.openembedded.org --name updatelayers-throwaway --mount src=update-workdir,dst=/opt/workdir --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app python3 /opt/layerindex/layerindex/update.py -r diff --git a/docker/nginx-ssl.conf b/docker/nginx-ssl.conf new file mode 100644 index 0000000..46cfde3 --- /dev/null +++ b/docker/nginx-ssl.conf @@ -0,0 +1,116 @@ +#daemon off; ##Included in CMD +error_log /dev/stdout info; +worker_processes 1; + +# user nobody nogroup; +pid /tmp/nginx.pid; + +events { + worker_connections 1024; + accept_mutex off; +} + +http { + include mime.types; + default_type application/octet-stream; + access_log /dev/stdout combined; + sendfile on; + + upstream app_server { + # For a TCP configuration: + server layersapp:5000 fail_timeout=0; + } + + server { + listen 80 default; + client_max_body_size 4G; + server_name _; + + keepalive_timeout 5; + + # path for static files + root /usr/share/nginx/html; + + return 301 https://layers.openembedded.org$request_uri; + } + + server { + listen 80; + client_max_body_size 4G; + server_name layers.openembedded.org; + + keepalive_timeout 5; + + # path for static files + root /usr/share/nginx/html; + + location /favicon.ico { + return 301 http://layers.openembedded.org/static/img/favicon.ico; + } + + location /admin { + return 301 https://layers.openembedded.org$request_uri; + } + + location /accounts/login { + return 301 https://layers.openembedded.org$request_uri; + } + + location / { + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + + proxy_pass http://app_server; + } + + } + + server { + listen 443 ssl default; + server_name _; + ssl_certificate /etc/letsencrypt/live/layers.openembedded.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/layers.openembedded.org/privkey.pem; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + keepalive_timeout 5; + + # path for static files + root /usr/share/nginx/html; + + return 301 https://layers.openembedded.org$request_uri; + } + + server { + listen 443 ssl; + server_name layers.openembedded.org; + ssl_certificate /etc/letsencrypt/live/layers.openembedded.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/layers.openembedded.org/privkey.pem; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + # path for static files + root /usr/share/nginx/html; + + location /favicon.ico { + return 301 https://layers.openembedded.org/static/img/favicon.ico; + } + + location / { + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + + proxy_pass http://app_server; + } + } +} diff --git a/docker/refreshlayers.sh b/docker/refreshlayers.sh new file mode 100755 index 0000000..0c550bd --- /dev/null +++ b/docker/refreshlayers.sh @@ -0,0 +1,3 @@ +#!/bin/bash +update=/opt/layerindex/layerindex/update.py +$update -q -r diff --git a/docker/settings.py b/docker/settings.py new file mode 100644 index 0000000..3d2c4c2 --- /dev/null +++ b/docker/settings.py @@ -0,0 +1,248 @@ +# Django settings for layerindex project. +# +# Based on settings.py from the Django project template +# Copyright (c) Django Software Foundation and individual contributors. + +import os + +DEBUG = os.getenv('DEBUG', False) + +ADMINS = ( + ('Paul Eggleton', 'paul.eggleton@linux.intel.com'), + ('Michael Halstead', 'mhalstead@linuxfoundation.org'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'layersdb', + 'USER': 'root', + 'PASSWORD': os.getenv('DATABASE_PASSWORD', 'testingpw'), + 'HOST': os.getenv('DATABASE_HOST', 'layersdb'), + 'PORT': '', + } +} + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# On Unix systems, a value of None will cause Django to use the same +# timezone as the operating system. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'Etc/UTC' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# If you set this to False, Django will not format dates, numbers and +# calendars according to the current locale +USE_L10N = True + +# Avoid specific paths +BASE_DIR = os.path.dirname(__file__) + +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/home/media/media.lawrence.com/media/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" +MEDIA_URL = '' + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/home/media/media.lawrence.com/static/" +STATIC_ROOT = '/usr/share/nginx/html/static/' + +# URL prefix for static files. +# Example: "http://media.lawrence.com/static/" +STATIC_URL = '/static/' + +# URL prefix for admin static files -- CSS, JavaScript and images. +# Make sure to use a trailing slash. +# Examples: "http://foo.com/static/admin/", "/static/admin/". +ADMIN_MEDIA_PREFIX = '/static/admin/' + +# Additional locations of static files +STATICFILES_DIRS = ( + # Put strings here, like "/home/html/static" or "C:/www/django/static". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +# 'django.contrib.staticfiles.finders.DefaultStorageFinder', +) + +# Make this unique, and don't share it with anybody. +SECRET_KEY = os.getenv('SECRET_KEY', '') + +MIDDLEWARE_CLASSES = ( + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'reversion.middleware.RevisionMiddleware', +) + +# We allow CORS calls from everybody +CORS_ORIGIN_ALLOW_ALL = True +# for the API pages +CORS_URLS_REGEX = r'.*/api/.*'; + + +# Clickjacking protection +X_FRAME_OPTIONS = 'DENY' + +ROOT_URLCONF = 'urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + BASE_DIR + "/templates", + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.contrib.messages.context_processors.messages', + 'django.template.context_processors.request', + 'layerindex.context_processors.layerindex_context', + ], + }, + }, +] + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'layerindex', + 'registration', + 'reversion', + 'reversion_compare', + 'captcha', + 'rest_framework', + 'corsheaders', + 'django_nvd3' +) + +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': ( + 'layerindex.restperm.ReadOnlyPermission', + ), + 'DATETIME_FORMAT': '%Y-%m-%dT%H:%m:%S+0000', +} + +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'class': 'django.utils.log.AdminEmailHandler' + } + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins'], + 'level': 'ERROR', + 'propagate': True, + }, + } +} + +# Set bootstrap alert CSS styles for each message level +from django.contrib.messages import constants as messages +MESSAGE_TAGS = { + messages.SUCCESS: 'alert-success', + messages.INFO: 'alert-info', + messages.WARNING: '', + messages.ERROR: 'alert-error', +} + +# Registration settings +ACCOUNT_ACTIVATION_DAYS = 2 +EMAIL_HOST = os.getenv('EMAIL_HOST', 'layers.test') +EMAIL_PORT = os.getenv('EMAIL_PORT', '25') +DEFAULT_FROM_EMAIL = 'noreply@' + os.getenv('HOSTNAME', 'layers.test') +LOGIN_REDIRECT_URL = '/layerindex' + +# Full path to directory where layers should be fetched into by the update script +LAYER_FETCH_DIR = "/opt/workdir" + +# Base temporary directory in which to create a directory in which to run BitBake +TEMP_BASE_DIR = "/tmp" + +# Fetch URL of the BitBake repository for the update script +BITBAKE_REPO_URL = "git://git.openembedded.org/bitbake" + +# Core layer to be used by the update script for basic BitBake configuration +CORE_LAYER_NAME = "openembedded-core" + +# Update records older than this number of days will be deleted every update +UPDATE_PURGE_DAYS = 30 + +# Remove layer dependencies that are not specified in conf/layer.conf +REMOVE_LAYER_DEPENDENCIES = False + +# Always use https:// for review URLs in emails (since it may be redirected to +# the login page) +FORCE_REVIEW_HTTPS = True + +# Settings for layer submission feature +SUBMIT_EMAIL_FROM = 'noreply@' + os.getenv('HOSTNAME', 'layers.test') +SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission' + +# Send email to maintainer(s) when their layer is published +SEND_PUBLISH_EMAIL = True + +# RabbitMQ settings +RABBIT_BROKER = 'amqp://guest:guest@layersrabbit:5672/' +RABBIT_BACKEND = 'rpc://layersrabbit/' + +# Used for fetching repo +PARALLEL_JOBS = "4" + +# Full path to directory where rrs tools stores logs +TOOLS_LOG_DIR = "" + +USE_X_FORWARDED_HOST = True +ALLOWED_HOSTS = [os.getenv('HOSTNAME', 'layers.test')] +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')