bitbake: toaster: Monitoring - implement Django logging system

(Bitbake rev: 2efb146480ee46c0463d9edb71bf1c03ce15bcf2)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/toaster/bldcollector/views.py

@@ -14,8 +14,11 @@ import subprocess
 import toastermain
 from django.views.decorators.csrf import csrf_exempt
 
+from toastermain.logs import log_view_mixin
+
 
 @csrf_exempt
+@log_view_mixin
 def eventfile(request):
     """ Receives a file by POST, and runs toaster-eventreply on this file """
     if request.method != "POST":

bitbake/lib/toaster/logs/.gitignore

@@ -0,0 +1 @@
+*.log*

bitbake/lib/toaster/toastergui/views.py

@@ -34,6 +34,8 @@ import mimetypes
 
 import logging
 
+from toastermain.logs import log_view_mixin
+
 logger = logging.getLogger("toaster")
 
 # Project creation and managed build enable
@@ -56,6 +58,7 @@ class MimeTypeFinder(object):
         return guessed_type
 
 # single point to add global values into the context before rendering
+@log_view_mixin
 def toaster_render(request, page, context):
     context['project_enable'] = project_enable
     context['project_specific'] = is_project_specific
@@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id):
     return response
 
 from django.http import HttpResponse
+@log_view_mixin
 def xhr_dirinfo(request, build_id, target_id):
     top = request.GET.get('start', '/')
     return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json")
@@ -1612,6 +1616,7 @@ if True:
 
     from django.views.decorators.csrf import csrf_exempt
     @csrf_exempt
+    @log_view_mixin
     def xhr_testreleasechange(request, pid):
         def response(data):
             return HttpResponse(jsonfilter(data),
@@ -1648,6 +1653,7 @@ if True:
         except Exception as e:
             return response({"error": str(e) })
 
+    @log_view_mixin
     def xhr_configvaredit(request, pid):
         try:
             prj = Project.objects.get(id = pid)
@@ -1726,6 +1732,7 @@ if True:
             return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json")
 
 
+    @log_view_mixin
     def customrecipe_download(request, pid, recipe_id):
         recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id)
 

bitbake/lib/toaster/toastergui/widgets.py

@@ -32,6 +32,7 @@ import re
 import os
 
 from toastergui.tablefilter import TableFilterMap
+from toastermain.logs import log_view_mixin
 
 try:
     from urllib import unquote_plus
@@ -84,6 +85,7 @@ class ToasterTable(TemplateView):
 
         return context
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         if request.GET.get('format', None) == 'json':
 
@@ -415,6 +417,7 @@ class ToasterTypeAhead(View):
     def __init__(self, *args, **kwargs):
         super(ToasterTypeAhead, self).__init__()
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         def response(data):
             return HttpResponse(json.dumps(data,
@@ -470,6 +473,7 @@ class MostRecentBuildsView(View):
 
         return False
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         """
         Returns a list of builds in JSON format.

bitbake/lib/toaster/toastermain/logs.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import logging
+import json
+from pathlib import Path
+from django.http import HttpRequest
+
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
+
+
+def log_api_request(request, response, view, logger_name='api'):
+    """Helper function for LogAPIMixin"""
+
+    repjson = {
+        'view': view,
+        'path': request.path,
+        'method': request.method,
+        'status': response.status_code
+    }
+
+    logger = logging.getLogger(logger_name)
+    logger.info(
+        json.dumps(repjson, indent=4, separators=(", ", " : "))
+    )
+
+
+def log_view_mixin(view):
+    def log_view_request(*args, **kwargs):
+        # get request from args else kwargs
+        request = None
+        if len(args) > 0:
+            for req in args:
+                if isinstance(req, HttpRequest):
+                    request = req
+                    break 
+        elif request is None:
+            request = kwargs.get('request')
+
+        response = view(*args, **kwargs)
+        log_api_request(
+            request, response, request.resolver_match.view_name, 'toaster')
+        return response
+    return log_view_request
+
+
+
+class LogAPIMixin:
+    """Logs API requests
+
+    tested with:
+        - APIView
+        - ModelViewSet
+        - ReadOnlyModelViewSet
+        - GenericAPIView
+
+    Note: you can set `view_name` attribute in View to override get_view_name()
+    """
+
+    def get_view_name(self):
+        if hasattr(self, 'view_name'):
+            return self.view_name
+        return super().get_view_name()
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        log_api_request(request, response, self.get_view_name())
+        return super().finalize_response(request, response, *args, **kwargs)
+
+
+LOGGING_SETTINGS = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'formatters': {
+        'datetime': {
+            'format': '%(asctime)s %(levelname)s %(message)s'
+        },
+        'verbose': {
+            'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}',
+            'datefmt': "%d/%b/%Y %H:%M:%S",
+            'style': '{',
+        },
+        'api': {
+            'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}',
+            'style': '{'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'datetime',
+        },
+        'file_django': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BASE_DIR / 'logs/django.log',
+            'when': 'D',  # interval type
+            'interval': 1,  # defaults to 1
+            'backupCount': 10,  # how many files to keep
+            'formatter': 'verbose',
+        },
+        'file_api': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BASE_DIR / 'logs/api.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+        'file_toaster': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BASE_DIR / 'logs/toaster.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARN',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARNING',
+            'propogate': True,
+        },
+        'toaster': {
+            'handlers': ['file_toaster'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+        'api': {
+            'handlers': ['file_api'],
+            'level': 'INFO',
+            'propagate': False,
+        }
+    }
+}

bitbake/lib/toaster/toastermain/settings.py

@@ -9,6 +9,8 @@
 # Django settings for Toaster project.
 
 import os
+from pathlib import Path
+from toastermain.logs import LOGGING_SETTINGS
 
 DEBUG = True
 
@@ -186,7 +188,13 @@ TEMPLATES = [
                 'django.template.loaders.app_directories.Loader',
                 #'django.template.loaders.eggs.Loader',
             ],
-            'string_if_invalid': InvalidString("%s"),
+            # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled
+            # Generally, string_if_invalid should only be enabled in order to debug
+            # a specific template problem, then cleared once debugging is complete.
+            # If you assign a value other than '' to string_if_invalid,
+            # you will experience rendering problems with these templates and sites.
+            #  'string_if_invalid': InvalidString("%s"),
+            'string_if_invalid': "",
             'debug': DEBUG,
         },
     },
@@ -242,6 +250,9 @@ INSTALLED_APPS = (
     'django.contrib.humanize',
     'bldcollector',
     'toastermain',
+
+    # 3rd-lib
+    "log_viewer",
 )
 
 
@@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)):
 # the site admins on every HTTP 500 error when DEBUG=False.
 # See http://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
-LOGGING = {
+LOGGING = LOGGING_SETTINGS
-    'version': 1,
+
-    'disable_existing_loggers': False,
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
-    'filters': {
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
-        'require_debug_false': {
+
-            '()': 'django.utils.log.RequireDebugFalse'
+# LOG VIEWER
-        }
+# https://pypi.org/project/django-log-viewer/
-    },
+LOG_VIEWER_FILES_PATTERN = '*.log*'
-    'formatters': {
+LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs')
-        'datetime': {
+LOG_VIEWER_PAGE_LENGTH = 25      # total log lines per-page
-            'format': '%(asctime)s %(levelname)s %(message)s'
+LOG_VIEWER_MAX_READ_LINES = 100000  # total log lines will be read
-        }
+LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
-    },
+
-    'handlers': {
+# Optionally you can set the next variables in order to customize the admin:
-        'mail_admins': {
+LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
-            'level': 'ERROR',
+
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
-        },
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'datetime',
-        }
-    },
-    'loggers': {
-        'toaster' : {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-        },
-        'django.request': {
-            'handlers': ['console'],
-            'level': 'WARN',
-            'propagate': True,
-        },
-    }
-}
 
 if DEBUG and SQL_DEBUG:
     LOGGING['loggers']['django.db.backends'] = {

bitbake/lib/toaster/toastermain/urls.py

@@ -28,6 +28,8 @@ urlpatterns = [
     # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
 
+    url(r'^logs/', include('log_viewer.urls')),
+
     # This is here to maintain backward compatibility and will be deprecated
     # in the future.
     url(r'^orm/eventfile$', bldcollector.views.eventfile),