partitions: add some logging

backend/src/appenlight/views/admin/partitions.py

@@ -1,151 +1,153 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # AppEnlight Enterprise Edition, including its added features, Support
 # services, and proprietary license terms, please see
 # https://rhodecode.com/licenses/
 
 from pyramid.view import view_config
 from appenlight.models import DBSession, Datastores
 from appenlight.forms import get_partition_deletion_form
 
 import logging
 
 from zope.sqlalchemy import mark_changed
 from datetime import datetime
 import sqlalchemy as sa
 
 log = logging.getLogger(__name__)
 
 
 def get_partition_stats():
     table_query = """
         SELECT table_name
         FROM information_schema.tables
         GROUP BY table_name
         ORDER BY table_name
     """
 
     permanent_partitions = {}
     daily_partitions = {}
 
     def is_int(data):
         try:
             int(data)
             return True
         except Exception:
             pass
         return False
 
     def add_key(key, holder):
         if not ix_time in holder:
             holder[ix_time] = {'pg': [], 'elasticsearch': []}
 
     for partition in list(Datastores.es.aliases().keys()):
         if not partition.startswith('rcae'):
             continue
         split_data = partition.split('_')
         permanent = False
         # if we dont have a day then treat it as permanent partion
         if False in list(map(is_int, split_data[-3:])):
             ix_time = datetime(year=int(split_data[-2]),
                                month=int(split_data[-1]),
                                day=1).date()
             permanent = True
         else:
             ix_time = datetime(year=int(split_data[-3]),
                                month=int(split_data[-2]),
                                day=int(split_data[-1])).date()
 
         ix_time = str(ix_time)
         if permanent:
             add_key(ix_time, permanent_partitions)
             if ix_time not in permanent_partitions:
                 permanent_partitions[ix_time]['elasticsearch'] = []
             permanent_partitions[ix_time]['elasticsearch'].append(partition)
         else:
             add_key(ix_time, daily_partitions)
             if ix_time not in daily_partitions:
                 daily_partitions[ix_time]['elasticsearch'] = []
             daily_partitions[ix_time]['elasticsearch'].append(partition)
 
     for row in DBSession.execute(table_query):
         splitted = row['table_name'].split('_')
         if 'p' in splitted:
             # dealing with partition
             split_data = [int(x) for x in splitted[splitted.index('p') + 1:]]
             if len(split_data) == 3:
                 ix_time = datetime(split_data[0], split_data[1],
                                    split_data[2]).date()
                 ix_time = str(ix_time)
                 add_key(ix_time, daily_partitions)
                 daily_partitions[ix_time]['pg'].append(row['table_name'])
             else:
                 ix_time = datetime(split_data[0], split_data[1], 1).date()
                 ix_time = str(ix_time)
                 add_key(ix_time, permanent_partitions)
                 permanent_partitions[ix_time]['pg'].append(row['table_name'])
 
     return permanent_partitions, daily_partitions
 
 
 @view_config(route_name='section_view', permission='root_administration',
              match_param=['section=admin_section', 'view=partitions'],
              renderer='json', request_method='GET')
 def index(request):
     permanent_partitions, daily_partitions = get_partition_stats()
 
     return {"permanent_partitions": sorted(list(permanent_partitions.items()),
                                            key=lambda x: x[0], reverse=True),
             "daily_partitions": sorted(list(daily_partitions.items()),
                                        key=lambda x: x[0], reverse=True)}
 
 
 @view_config(route_name='section_view', request_method='POST',
              match_param=['section=admin_section', 'view=partitions_remove'],
              renderer='json', permission='root_administration')
 def partitions_remove(request):
     permanent_partitions, daily_partitions = get_partition_stats()
     pg_partitions = []
     es_partitions = []
     for item in list(permanent_partitions.values()) + list(daily_partitions.values()):
         es_partitions.extend(item['elasticsearch'])
         pg_partitions.extend(item['pg'])
     FormCls = get_partition_deletion_form(es_partitions, pg_partitions)
     form = FormCls(es_index=request.unsafe_json_body['es_indices'],
                    pg_index=request.unsafe_json_body['pg_indices'],
                    confirm=request.unsafe_json_body['confirm'],
                    csrf_context=request)
     if form.validate():
         for ix in form.data['es_index']:
+            log.warning('deleting ES partition: {}'.format(ix))
             Datastores.es.delete_index(ix)
         for ix in form.data['pg_index']:
+            log.warning('deleting PG partition: {}'.format(ix))
             stmt = sa.text('DROP TABLE %s CASCADE' % sa.text(ix))
             session = DBSession()
             session.connection().execute(stmt)
             mark_changed(session)
 
     for field, error in form.errors.items():
         msg = '%s: %s' % (field, error[0])
         request.session.flash(msg, 'error')
 
     permanent_partitions, daily_partitions = get_partition_stats()
     return {
         "permanent_partitions": sorted(
             list(permanent_partitions.items()), key=lambda x: x[0], reverse=True),
         "daily_partitions": sorted(
             list(daily_partitions.items()), key=lambda x: x[0], reverse=True)}