backend/src/appenlight/views/logs.py

# -*- coding: utf-8 -*-

# Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
from datetime import datetime, timedelta

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPUnprocessableEntity
from appenlight.models import Datastores, Log
from appenlight.models.services.log import LogService
from appenlight.lib.utils import (
    build_filter_settings_from_query_dict,
    es_index_name_limiter,
)
from appenlight.lib.helpers import gen_pagination_headers
from appenlight.celery.tasks import logs_cleanup

log = logging.getLogger(__name__)

section_filters_key = "appenlight:logs:filter:%s"


@view_config(route_name="logs_no_id", renderer="json", permission="authenticated")
def fetch_logs(request):
    """
    Returns list of log entries from Elasticsearch
    """

    filter_settings = build_filter_settings_from_query_dict(
        request, request.GET.mixed()
    )
    logs_paginator = LogService.get_paginator_by_app_ids(
        app_ids=filter_settings["resource"],
        page=filter_settings["page"],
        filter_settings=filter_settings,
    )
    headers = gen_pagination_headers(request, logs_paginator)
    request.response.headers.update(headers)

    return [l.get_dict() for l in logs_paginator.sa_items]


@view_config(
    route_name="section_view",
    match_param=["section=logs_section", "view=fetch_series"],
    renderer="json",
    permission="authenticated",
)
def logs_fetch_series(request):
    """
    Handles metric dashboard graphs
    Returns information for time/tier breakdown
    """
    filter_settings = build_filter_settings_from_query_dict(
        request, request.GET.mixed()
    )
    paginator = LogService.get_paginator_by_app_ids(
        app_ids=filter_settings["resource"],
        page=1,
        filter_settings=filter_settings,
        items_per_page=1,
    )
    now = datetime.utcnow().replace(microsecond=0, second=0)
    delta = timedelta(days=7)
    if paginator.sa_items:
        start_date = paginator.sa_items[-1].timestamp.replace(microsecond=0, second=0)
        filter_settings["start_date"] = start_date - delta
    else:
        filter_settings["start_date"] = now - delta
    filter_settings["end_date"] = filter_settings["start_date"] + timedelta(days=7)

    @request.registry.cache_regions.redis_sec_30.cache_on_arguments("logs_graphs")
    def cached(apps, search_params, delta, now):
        data = LogService.get_time_series_aggregate(
            filter_settings["resource"], filter_settings
        )
        if not data:
            return []
        buckets = data["aggregations"]["events_over_time"]["buckets"]
        return [
            {
                "x": datetime.utcfromtimestamp(item["key"] / 1000),
                "logs": item["doc_count"],
            }
            for item in buckets
        ]

    return cached(filter_settings, request.GET.mixed(), delta, now)


@view_config(
    route_name="logs_no_id",
    renderer="json",
    request_method="DELETE",
    permission="authenticated",
)
def logs_mass_delete(request):
    params = request.GET.mixed()
    if "resource" not in params:
        raise HTTPUnprocessableEntity()
    # this might be '' and then colander will not validate the schema
    if not params.get("namespace"):
        params.pop("namespace", None)
    filter_settings = build_filter_settings_from_query_dict(
        request, params, resource_permissions=["update_reports"]
    )

    resource_id = list(filter_settings["resource"])[0]
    # filter settings returns list of all of users applications
    # if app is not matching - normally we would not care as its used for search
    # but here user playing with params would possibly wipe out their whole data
    if int(resource_id) != int(params["resource"]):
        raise HTTPUnprocessableEntity()

    logs_cleanup.delay(resource_id, filter_settings)
    msg = (
        "Log cleanup process started - it may take a while for "
        "everything to get removed"
    )
    request.session.flash(msg)
    return {}


@view_config(
    route_name="section_view",
    match_param=("view=common_tags", "section=logs_section"),
    renderer="json",
    permission="authenticated",
)
def common_tags(request):
    config = request.GET.mixed()
    filter_settings = build_filter_settings_from_query_dict(request, config)

    resources = list(filter_settings["resource"])
    query = {
        "query": {"bool": {"filter": [{"terms": {"resource_id": list(resources)}}]}}
    }
    start_date = filter_settings.get("start_date")
    end_date = filter_settings.get("end_date")
    filter_part = query["query"]["bool"]["filter"]

    date_range = {"range": {"timestamp": {}}}
    if start_date:
        date_range["range"]["timestamp"]["gte"] = start_date
    if end_date:
        date_range["range"]["timestamp"]["lte"] = end_date
    if start_date or end_date:
        filter_part.append(date_range)

    levels = filter_settings.get("level")
    if levels:
        filter_part.append({"terms": {"log_level": levels}})
    namespaces = filter_settings.get("namespace")
    if namespaces:
        filter_part.append({"terms": {"namespace": namespaces}})

    query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list.keyword", "size": 50}}}
    # tags
    index_names = es_index_name_limiter(ixtypes=[config.get("datasource", "logs")])
    result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
    tag_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
    # namespaces
    query["aggs"] = {"sub_agg": {"terms": {"field": "namespace.keyword", "size": 50}}}
    result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
    namespaces_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
    return {
        "tags": [item["key"] for item in tag_buckets],
        "namespaces": [item["key"] for item in namespaces_buckets],
    }


@view_config(
    route_name="section_view",
    match_param=("view=common_values", "section=logs_section"),
    renderer="json",
    permission="authenticated",
)
def common_values(request):
    config = request.GET.mixed()
    datasource = config.pop("datasource", "logs")
    filter_settings = build_filter_settings_from_query_dict(request, config)
    resources = list(filter_settings["resource"])
    tag_name = filter_settings["tags"][0]["value"][0]

    and_part = [{"terms": {"resource_id": list(resources)}}]
    if filter_settings["namespace"]:
        and_part.append({"terms": {"namespace": filter_settings["namespace"]}})
    query = {"query": {"bool": {"filter": and_part}}}
    query["aggs"] = {
        "sub_agg": {"terms": {"field": "tags.{}.values".format(tag_name), "size": 50}}
    }
    index_names = es_index_name_limiter(ixtypes=[datasource])
    result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
    values_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
    return {"values": [item["key"] for item in values_buckets]}