docker: enhance example setup

* Put NGINX, Celery, and RabbitMQ into their own separate containers * Use a docker network instead of the deprecated --link * Allow for collecting the static files properly * Create a copy of settings.py specifically for the docker setup. This will need to be kept in sync with the main example settings.py, but it avoids the user having to edit it too much. * Add optional SSL configuration using letsencrypt certificate * Create some volumes for static files / fetched repos * Add some more helpful setup instructions Largely based upon work by Michael Halstead <michael@yoctoproject.org>. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
2025-07-05 05:04:46 +02:00 · 2018-07-10 11:56:10 +02:00 · 2018-07-10 11:56:10 +02:00 · 272f0eded2
commit 272f0eded2
parent 32df911836
6 changed files with 449 additions and 23 deletions
--- a/7
+++ b/7
@ -1,7 +1,8 @@
+# See docker/README for how to use this.
+
 FROM debian:stretch
 LABEL maintainer="Michael Halstead <mhalstead@linuxfoundation.org>"

-EXPOSE 80
 ENV PYTHONUNBUFFERED=1 \
    LANG=en_US.UTF-8 \
    LC_ALL=en_US.UTF-8 \
@ -30,7 +31,6 @@ RUN apt-get install -y --no-install-recommends \
 	libjpeg-dev \
 	libmariadbclient-dev \
 	locales \
-	rabbitmq-server \
 	netcat-openbsd \
 	curl \
 	git-core \
@ -49,7 +49,8 @@ RUN apt-get purge -y autoconf g++ make python3-dev libjpeg-dev libmariadbclient-
 	&& rm -rf /var/lib/apt/lists/* \
 	&& apt-get clean
 COPY . /opt/layerindex
-COPY settings.py /opt/layerindex/settings.py
+COPY docker/settings.py /opt/layerindex/settings.py
+COPY docker/refreshlayers.sh /opt/refreshlayers.sh
 COPY docker/updatelayers.sh /opt/updatelayers.sh
 COPY docker/migrate.sh /opt/migrate.sh

--- a/Dockerfile.web
+++ b/Dockerfile.web
@ -1,4 +1,3 @@
 FROM nginx:latest
 LABEL maintainer="Michael Halstead <mhalstead@linuxfoundation.org>"
 COPY docker/nginx.conf /etc/nginx/nginx.conf
-COPY layerindex/static /usr/share/nginx/html/static
--- a/docker/README
+++ b/docker/README
@ -1,26 +1,85 @@
-## This is set up to make a cluster of three containers. First we build two from the root of the repo.
+## Layerindex example docker instructions
+
+## This is set up to make a cluster of 5 containers:
+## - layersapp: the application
+## - layersdb: the database
+## - layersweb: NGINX web server (as a proxy and for serving static content)
+## - layerscelery: Celery (for running background jobs)
+## - layersrabbit: RabbitMQ (required by Celery)
+
+## First, find and replace layers.openembedded.org below with your hostname
+## You'll probably also want to replace the database password "testingpw".
+## If you are using a version of Docker older than 17.06 then you'll need to replace --mount src=<src>,dst=<dst> with -v <src>:<dst>
+
+## If you want to change any of the application configuration, edit docker/settings.py as desired.
+## Some settings have been set so that values can be passed in via environment variables.
+## You will definitely need to set SECRET_KEY.
+
+## If you are on a network that requires a proxy to get out to the internet, then you'll need to:
+## - Uncomment several lines in Dockerfile (search for "proxy")
+## - Edit docker/.gitconfig and docker/git-proxy
+
+## Build the main container from the root of the repo.
 docker build -t halstead/layerindex-app .
+
+## Build the static web server container
+## (for SSL, first move docker/nginx-ssl.conf to docker/nginx.conf and edit as needed.)
 docker build -t halstead/layerindex-web -f Dockerfile.web .

-## Start a database server. We use MariaDB in production.
-## In order to configure your settings.py file to use this database server, use:
-##   'ENGINE': 'django.db.backends.mysql',
-##   'NAME': 'layersdb',
-##   'USER': 'root',
-##   'PASSWORD': 'testingpw',
-##   'HOST': 'layersdb',
-##   'PORT': '',
-docker run -d --name layerdb -e MYSQL_ROOT_PASSWORD=testingpw -e MYSQL_DATABASE=layersdb mariadb --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+## Add a network for our containers
+docker network create layerindex

-## If you have a copy of the the production data now is the time to insert it.
-## If not you can skip the next step for a clean install.
-xzcat ./layerdb.sql.xz | docker run -i --link layerdb:layersdb --rm mariadb sh -c 'exec mysql -hlayersdb -uroot -p"testingpw" layersdb'
+## Start a database server - here we use MariaDB, though you can obviously use something else and change docker/settings.py as appropriate
+#  run one of the following.
+## To use an existing dump run the following and wait for startup and import
+docker run --detach --name layersdb --network layerindex --mount src=layers-database-dump.sql,dst=/docker-entrypoint-initdb.d/layerdb.sql -e MYSQL_ROOT_PASSWORD=testingpw mariadb:10.2 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+## Or to start fresh
+docker run --detach --name layersdb --network layerindex -e MYSQL_DATABASE=layersdb -e MYSQL_ROOT_PASSWORD=testingpw mariadb:10.2 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci

-docker run -d --link layerdb:layersdb --name layersapp halstead/layerindex-app
-docker run -d --link layersapp:layersapp --name layersweb -p 49153:80 halstead/layerindex-web
+## Start the RabbitMQ container
+docker run --detach --network layerindex --name layersrabbit rabbitmq:alpine

-## To apply layerindex migration
-docker run --rm --link layerdb:layersdb halstead/layerindex-app /opt/migrate.sh
+## Start the Celery container
+docker run --detach --network layerindex --name layerscelery -e DATABASE_PASSWORD=testingpw -e DATABASE_HOST=layersdb  halstead/layerindex-app /usr/local/bin/celery -A layerindex.tasks worker --loglevel=info --workdir=/opt/layerindex

-## To update the layer info we can run the job in a temporary container.
-docker run --rm --link layerdb:layersdb halstead/layerindex-app /opt/updatelayers.sh
+
+## Apply any pending layerindex migrations / initialize the database
+docker run --rm --network layerindex --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app /opt/migrate.sh
+
+## For a fresh database, create an admin account
+docker run --rm -it --network layerindex --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app /opt/layerindex/manage.py createsuperuser
+
+
+## Start the layerindex application
+docker run --detach --network layerindex --name layersapp --hostname layers.openembedded.org  -e DATABASE_PASSWORD=testingpw -e DATABASE_HOST=layersdb halstead/layerindex-app
+
+## Create a volume for static assets
+docker volume create layersstatic
+
+## Set the volume permissions using debian:stretch since we recently fetched it
+docker run --mount src=layersstatic,dst=/usr/share/nginx/html debian:stretch chown 500 /usr/share/nginx/html
+
+## Generate static assets. Run this command again to regenerate at any time (when static assets in the code are updated)
+docker run --env STATIC_ROOT=/usr/share/nginx/html -ti --rm --network layerindex --hostname layers.openembedded.org --name generatestatic --mount src=layersstatic,dst=/usr/share/nginx/html --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app python3 /opt/layerindex/manage.py collectstatic
+
+## Start the reverse proxy
+##  run one of the following:
+## A) for local/test use forward port 8080:
+docker run --detach --network layerindex -p 8080:80 --mount src=layersstatic,dst=/usr/share/nginx/html --name layersweb --hostname layers.openembedded.org halstead/layerindex-web
+## B) with SSL for production:
+#     Make sure your DNS is setup and then run the following to get the certs
+docker run -it --rm -p 80:80 -p 443:443 --name certbot --mount src=layerscerts,dst=/etc/letsencrypt --mount src=certbotvar,dst=/var/lib/letsencrypt certbot/certbot certonly #renew
+#     then start the proxy with ssl
+docker run --detach --network layerindex -p 80:80 -p 443:443 --mount src=layersstatic,dst=/usr/share/nginx/html --mount src=layerscerts,dst=/etc/letsencrypt --name layersweb --hostname layers.openembedded.org halstead/layerindex-web
+
+## Create a workdir to prevent downloading repos fresh each time
+docker volume create update-workdir
+
+## Set the volume permissions using debian:stretch since we recently fetched it
+docker run --mount src=update-workdir,dst=/opt/workdir debian:stretch chown 500 /opt/workdir
+
+## Run the layer updates
+docker run --rm --network layerindex --hostname updatelayers.openembedded.org --name updatelayers-throwaway --mount src=update-workdir,dst=/opt/workdir --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app python3 /opt/layerindex/layerindex/update.py
+
+## Or do a full refresh
+docker run --rm --network layerindex --hostname updatelayers.openembedded.org --name updatelayers-throwaway --mount src=update-workdir,dst=/opt/workdir --env DATABASE_HOST=layersdb --env DATABASE_PASSWORD=testingpw halstead/layerindex-app python3 /opt/layerindex/layerindex/update.py -r
--- a/docker/nginx-ssl.conf
+++ b/docker/nginx-ssl.conf
@ -0,0 +1,116 @@
+#daemon off; ##Included in CMD
+error_log /dev/stdout info;
+worker_processes 1;
+
+# user nobody nogroup;
+pid /tmp/nginx.pid;
+
+events {
+    worker_connections 1024;
+    accept_mutex off;
+}
+
+http {
+    include mime.types;
+    default_type application/octet-stream;
+    access_log /dev/stdout combined;
+    sendfile on;
+
+    upstream app_server {
+        # For a TCP configuration:
+        server layersapp:5000 fail_timeout=0;
+    }
+
+    server {
+        listen 80 default;
+        client_max_body_size 4G;
+        server_name _;
+
+        keepalive_timeout 5;
+
+        # path for static files
+        root /usr/share/nginx/html;
+
+	return 301 https://layers.openembedded.org$request_uri;
+    }
+
+    server {
+        listen 80;
+        client_max_body_size 4G;
+        server_name layers.openembedded.org;
+
+        keepalive_timeout 5;
+
+        # path for static files
+        root /usr/share/nginx/html;
+
+	location /favicon.ico {
+	    return 301 http://layers.openembedded.org/static/img/favicon.ico;
+	}
+
+	location /admin {
+	    return 301 https://layers.openembedded.org$request_uri;
+	}
+
+        location /accounts/login {
+	    return 301 https://layers.openembedded.org$request_uri;
+        }
+
+        location / {
+	   try_files $uri @proxy_to_app;
+        }
+
+        location @proxy_to_app {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $http_host;
+            proxy_redirect off;
+
+            proxy_pass   http://app_server;
+        }
+
+    }
+
+    server {
+        listen              443 ssl default;
+        server_name _;
+        ssl_certificate     /etc/letsencrypt/live/layers.openembedded.org/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/layers.openembedded.org/privkey.pem;
+        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         HIGH:!aNULL:!MD5;
+
+        keepalive_timeout 5;
+
+        # path for static files
+        root /usr/share/nginx/html;
+
+	return 301 https://layers.openembedded.org$request_uri;
+    }
+
+    server {
+        listen              443 ssl;
+        server_name         layers.openembedded.org;
+        ssl_certificate     /etc/letsencrypt/live/layers.openembedded.org/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/layers.openembedded.org/privkey.pem;
+        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         HIGH:!aNULL:!MD5;
+
+        # path for static files
+        root /usr/share/nginx/html;
+
+	location /favicon.ico {
+	    return 301 https://layers.openembedded.org/static/img/favicon.ico;
+	}
+
+        location / {
+           try_files $uri @proxy_to_app;
+        }
+
+        location @proxy_to_app {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $http_host;
+            proxy_redirect off;
+
+            proxy_pass   http://app_server;
+        }
+    }
+}
--- a/docker/refreshlayers.sh
+++ b/docker/refreshlayers.sh
@ -0,0 +1,3 @@
+#!/bin/bash
+update=/opt/layerindex/layerindex/update.py
+$update -q -r
--- a/docker/settings.py
+++ b/docker/settings.py
@ -0,0 +1,248 @@
+# Django settings for layerindex project.
+#
+# Based on settings.py from the Django project template
+# Copyright (c) Django Software Foundation and individual contributors.
+
+import os
+
+DEBUG = os.getenv('DEBUG', False)
+
+ADMINS = (
+    ('Paul Eggleton', 'paul.eggleton@linux.intel.com'),
+    ('Michael Halstead', 'mhalstead@linuxfoundation.org'),
+)
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'layersdb',
+        'USER': 'root',
+        'PASSWORD': os.getenv('DATABASE_PASSWORD', 'testingpw'),
+        'HOST': os.getenv('DATABASE_HOST', 'layersdb'),
+        'PORT': '',
+    }
+}
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# On Unix systems, a value of None will cause Django to use the same
+# timezone as the operating system.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'Etc/UTC'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# If you set this to False, Django will not format dates, numbers and
+# calendars according to the current locale
+USE_L10N = True
+
+# Avoid specific paths
+BASE_DIR = os.path.dirname(__file__)
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/home/media/media.lawrence.com/media/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash.
+# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
+MEDIA_URL = ''
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/home/media/media.lawrence.com/static/"
+STATIC_ROOT = '/usr/share/nginx/html/static/'
+
+# URL prefix for static files.
+# Example: "http://media.lawrence.com/static/"
+STATIC_URL = '/static/'
+
+# URL prefix for admin static files -- CSS, JavaScript and images.
+# Make sure to use a trailing slash.
+# Examples: "http://foo.com/static/admin/", "/static/admin/".
+ADMIN_MEDIA_PREFIX = '/static/admin/'
+
+# Additional locations of static files
+STATICFILES_DIRS = (
+    # Put strings here, like "/home/html/static" or "C:/www/django/static".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
+)
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = os.getenv('SECRET_KEY', '')
+
+MIDDLEWARE_CLASSES = (
+    'corsheaders.middleware.CorsMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'reversion.middleware.RevisionMiddleware',
+)
+
+# We allow CORS calls from everybody
+CORS_ORIGIN_ALLOW_ALL = True
+# for the API pages
+CORS_URLS_REGEX = r'.*/api/.*';
+
+
+# Clickjacking protection
+X_FRAME_OPTIONS = 'DENY'
+
+ROOT_URLCONF = 'urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [
+            BASE_DIR + "/templates",
+        ],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.contrib.auth.context_processors.auth',
+                'django.template.context_processors.debug',
+                'django.template.context_processors.i18n',
+                'django.template.context_processors.media',
+                'django.template.context_processors.static',
+                'django.template.context_processors.tz',
+                'django.contrib.messages.context_processors.messages',
+                'django.template.context_processors.request',
+                'layerindex.context_processors.layerindex_context',
+            ],
+        },
+    },
+]
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+    'layerindex',
+    'registration',
+    'reversion',
+    'reversion_compare',
+    'captcha',
+    'rest_framework',
+    'corsheaders',
+    'django_nvd3'
+)
+
+REST_FRAMEWORK = {
+    'DEFAULT_PERMISSION_CLASSES': (
+        'layerindex.restperm.ReadOnlyPermission',
+    ),
+    'DATETIME_FORMAT': '%Y-%m-%dT%H:%m:%S+0000',
+}
+
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'class': 'django.utils.log.AdminEmailHandler'
+        }
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['mail_admins'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+    }
+}
+
+# Set bootstrap alert CSS styles for each message level
+from django.contrib.messages import constants as messages
+MESSAGE_TAGS = {
+    messages.SUCCESS: 'alert-success',
+    messages.INFO: 'alert-info',
+    messages.WARNING: '',
+    messages.ERROR: 'alert-error',
+}
+
+# Registration settings
+ACCOUNT_ACTIVATION_DAYS = 2
+EMAIL_HOST = os.getenv('EMAIL_HOST', 'layers.test')
+EMAIL_PORT = os.getenv('EMAIL_PORT', '25')
+DEFAULT_FROM_EMAIL = 'noreply@' + os.getenv('HOSTNAME', 'layers.test')
+LOGIN_REDIRECT_URL = '/layerindex'
+
+# Full path to directory where layers should be fetched into by the update script
+LAYER_FETCH_DIR = "/opt/workdir"
+
+# Base temporary directory in which to create a directory in which to run BitBake
+TEMP_BASE_DIR = "/tmp"
+
+# Fetch URL of the BitBake repository for the update script
+BITBAKE_REPO_URL = "git://git.openembedded.org/bitbake"
+
+# Core layer to be used by the update script for basic BitBake configuration
+CORE_LAYER_NAME = "openembedded-core"
+
+# Update records older than this number of days will be deleted every update
+UPDATE_PURGE_DAYS = 30
+
+# Remove layer dependencies that are not specified in conf/layer.conf
+REMOVE_LAYER_DEPENDENCIES = False
+
+# Always use https:// for review URLs in emails (since it may be redirected to
+# the login page)
+FORCE_REVIEW_HTTPS = True
+
+# Settings for layer submission feature
+SUBMIT_EMAIL_FROM = 'noreply@' + os.getenv('HOSTNAME', 'layers.test')
+SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'
+
+# Send email to maintainer(s) when their layer is published
+SEND_PUBLISH_EMAIL = True
+
+# RabbitMQ settings
+RABBIT_BROKER = 'amqp://guest:guest@layersrabbit:5672/'
+RABBIT_BACKEND = 'rpc://layersrabbit/'
+
+# Used for fetching repo
+PARALLEL_JOBS = "4"
+
+# Full path to directory where rrs tools stores logs
+TOOLS_LOG_DIR = ""
+
+USE_X_FORWARDED_HOST = True
+ALLOWED_HOSTS = [os.getenv('HOSTNAME', 'layers.test')]
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')