logs: fix for common values

backend/src/appenlight/views/logs.py

@@ -1,219 +1,222 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 from datetime import datetime, timedelta
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 from appenlight.models import Datastores, Log
 from appenlight.models.services.log import LogService
 from appenlight.lib.utils import (
     build_filter_settings_from_query_dict,
     es_index_name_limiter,
 )
 from appenlight.lib.helpers import gen_pagination_headers
 from appenlight.celery.tasks import logs_cleanup
 
 log = logging.getLogger(__name__)
 
 section_filters_key = "appenlight:logs:filter:%s"
 
 
 @view_config(route_name="logs_no_id", renderer="json", permission="authenticated")
 def fetch_logs(request):
     """
     Returns list of log entries from Elasticsearch
     """
 
     filter_settings = build_filter_settings_from_query_dict(
         request, request.GET.mixed()
     )
     logs_paginator = LogService.get_paginator_by_app_ids(
         app_ids=filter_settings["resource"],
         page=filter_settings["page"],
         filter_settings=filter_settings,
     )
     headers = gen_pagination_headers(request, logs_paginator)
     request.response.headers.update(headers)
 
     return [l.get_dict() for l in logs_paginator.sa_items]
 
 
 @view_config(
     route_name="section_view",
     match_param=["section=logs_section", "view=fetch_series"],
     renderer="json",
     permission="authenticated",
 )
 def logs_fetch_series(request):
     """
     Handles metric dashboard graphs
     Returns information for time/tier breakdown
     """
     filter_settings = build_filter_settings_from_query_dict(
         request, request.GET.mixed()
     )
     paginator = LogService.get_paginator_by_app_ids(
         app_ids=filter_settings["resource"],
         page=1,
         filter_settings=filter_settings,
         items_per_page=1,
     )
     now = datetime.utcnow().replace(microsecond=0, second=0)
     delta = timedelta(days=7)
     if paginator.sa_items:
         start_date = paginator.sa_items[-1].timestamp.replace(microsecond=0, second=0)
         filter_settings["start_date"] = start_date - delta
     else:
         filter_settings["start_date"] = now - delta
     filter_settings["end_date"] = filter_settings["start_date"] + timedelta(days=7)
 
     @request.registry.cache_regions.redis_sec_30.cache_on_arguments("logs_graphs")
     def cached(apps, search_params, delta, now):
         data = LogService.get_time_series_aggregate(
             filter_settings["resource"], filter_settings
         )
         if not data:
             return []
         buckets = data["aggregations"]["events_over_time"]["buckets"]
         return [
             {
                 "x": datetime.utcfromtimestamp(item["key"] / 1000),
                 "logs": item["doc_count"],
             }
             for item in buckets
         ]
 
     return cached(filter_settings, request.GET.mixed(), delta, now)
 
 
 @view_config(
     route_name="logs_no_id",
     renderer="json",
     request_method="DELETE",
     permission="authenticated",
 )
 def logs_mass_delete(request):
     params = request.GET.mixed()
     if "resource" not in params:
         raise HTTPUnprocessableEntity()
     # this might be '' and then colander will not validate the schema
     if not params.get("namespace"):
         params.pop("namespace", None)
     filter_settings = build_filter_settings_from_query_dict(
         request, params, resource_permissions=["update_reports"]
     )
 
     resource_id = list(filter_settings["resource"])[0]
     # filter settings returns list of all of users applications
     # if app is not matching - normally we would not care as its used for search
     # but here user playing with params would possibly wipe out their whole data
     if int(resource_id) != int(params["resource"]):
         raise HTTPUnprocessableEntity()
 
     logs_cleanup.delay(resource_id, filter_settings)
     msg = (
         "Log cleanup process started - it may take a while for "
         "everything to get removed"
     )
     request.session.flash(msg)
     return {}
 
 
 @view_config(
     route_name="section_view",
     match_param=("view=common_tags", "section=logs_section"),
     renderer="json",
     permission="authenticated",
 )
 def common_tags(request):
     config = request.GET.mixed()
     filter_settings = build_filter_settings_from_query_dict(request, config)
 
     resources = list(filter_settings["resource"])
     query = {
         "query": {
             "filtered": {
                 "filter": {"and": [{"terms": {"resource_id": list(resources)}}]}
             }
         }
     }
     start_date = filter_settings.get("start_date")
     end_date = filter_settings.get("end_date")
     filter_part = query["query"]["filtered"]["filter"]["and"]
 
     date_range = {"range": {"timestamp": {}}}
     if start_date:
         date_range["range"]["timestamp"]["gte"] = start_date
     if end_date:
         date_range["range"]["timestamp"]["lte"] = end_date
     if start_date or end_date:
         filter_part.append(date_range)
 
     levels = filter_settings.get("level")
     if levels:
         filter_part.append({"terms": {"log_level": levels}})
     namespaces = filter_settings.get("namespace")
     if namespaces:
         filter_part.append({"terms": {"namespace": namespaces}})
 
     query["aggs"] = {"sub_agg": {"terms": {"field": "tag_list", "size": 50}}}
     # tags
     index_names = es_index_name_limiter(ixtypes=[config.get("datasource", "logs")])
     result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
     tag_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
     # namespaces
     query["aggs"] = {"sub_agg": {"terms": {"field": "namespace", "size": 50}}}
     result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
     namespaces_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
     return {
         "tags": [item["key"] for item in tag_buckets],
         "namespaces": [item["key"] for item in namespaces_buckets],
     }
 
 
 @view_config(
     route_name="section_view",
     match_param=("view=common_values", "section=logs_section"),
     renderer="json",
     permission="authenticated",
 )
 def common_values(request):
     config = request.GET.mixed()
     datasource = config.pop("datasource", "logs")
     filter_settings = build_filter_settings_from_query_dict(request, config)
     resources = list(filter_settings["resource"])
     tag_name = filter_settings["tags"][0]["value"][0]
+
+    and_part = [
+        {"terms": {"resource_id": list(resources)}},
+    ]
+    if filter_settings["namespace"]:
+        and_part.append({"terms": {"namespace": filter_settings["namespace"]}})
     query = {
         "query": {
             "filtered": {
                 "filter": {
-                    "and": [
-                        {"terms": {"resource_id": list(resources)}},
-                        {"terms": {"namespace": filter_settings["namespace"]}},
-                    ]
+                    "and": and_part
                 }
             }
         }
     }
     query["aggs"] = {
         "sub_agg": {"terms": {"field": "tags.{}.values".format(tag_name), "size": 50}}
     }
     index_names = es_index_name_limiter(ixtypes=[datasource])
     result = Datastores.es.search(body=query, index=index_names, doc_type="log", size=0)
     values_buckets = result["aggregations"]["sub_agg"].get("buckets", [])
     return {"values": [item["key"] for item in values_buckets]}