requirements: bump ziggurat_foundations to 0.8.3

.gitignore

@@ -1,220 +1,220 b''
 # Created by .ignore support plugin (hsz.mobi)
 ### Node template
 # Logs
 logs
 *.log
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
 # Runtime data
 pids
 *.pid
 *.seed
 *.pid.lock
 
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 
 # Coverage directory used by tools like istanbul
 coverage
 
 # nyc test coverage
 .nyc_output
 
 # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
 .grunt
 
 # Bower dependency directory (https://bower.io/)
 bower_components
 
 # node-waf configuration
 .lock-wscript
 
 # Compiled binary addons (https://nodejs.org/api/addons.html)
 build/Release
 
 # Dependency directories
 node_modules/
 jspm_packages/
 
 # Typescript v1 declaration files
 typings/
 
 # Optional npm cache directory
 .npm
 
 # Optional eslint cache
 .eslintcache
 
 # Optional REPL history
 .node_repl_history
 
 # Output of 'npm pack'
 *.tgz
 
 # Yarn Integrity file
 .yarn-integrity
 
 # dotenv environment variables file
 .env
 
 ### JetBrains template
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
 # User-specific stuff:
 .idea/**/workspace.xml
 .idea/**/tasks.xml
 .idea/dictionaries
 
 # Sensitive or high-churn files:
 .idea/**/dataSources/
 .idea/**/dataSources.ids
 .idea/**/dataSources.xml
 .idea/**/dataSources.local.xml
 .idea/**/sqlDataSources.xml
 .idea/**/dynamic.xml
 .idea/**/uiDesigner.xml
 
 # Gradle:
 .idea/**/gradle.xml
 .idea/**/libraries
 
 # CMake
 cmake-build-debug/
 
 # Mongo Explorer plugin:
 .idea/**/mongoSettings.xml
 
 ## File-based project format:
 *.iws
 
 ## Plugin-specific files:
 
 # IntelliJ
 out/
 
 # mpeltonen/sbt-idea plugin
 .idea_modules/
 
 # JIRA plugin
 atlassian-ide-plugin.xml
 
 # Cursive Clojure plugin
 .idea/replstate.xml
 
 # Crashlytics plugin (for Android Studio and IntelliJ)
 com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
 fabric.properties
 ### Python template
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 
 # C extensions
 *.so
 
 # Distribution / packaging
 .Python
 build/
 develop-eggs/
 dist/
 downloads/
 eggs/
 .eggs/
-lib/
-lib64/
+/lib/
+/lib64/
 parts/
 sdist/
 var/
 wheels/
 *.egg-info/
 .installed.cfg
 *.egg
 MANIFEST
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
 
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
 
 # Unit test / coverage reports
 htmlcov/
 .tox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
 
 # Translations
 *.mo
 *.pot
 
 # Django stuff:
 local_settings.py
 
 # Flask stuff:
 instance/
 .webassets-cache
 
 # Scrapy stuff:
 .scrapy
 
 # Sphinx documentation
 docs/_build/
 
 # PyBuilder
 target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
 
 # pyenv
 .python-version
 
 # celery beat schedule file
 celerybeat-schedule
 
 # SageMath parsed files
 *.sage.py
 
 # Environments
 .venv
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
 
 # Spyder project settings
 .spyderproject
 .spyproject
 
 # Rope project settings
 .ropeproject
 
 # mkdocs documentation
 /site
 
 # mypy
 .mypy_cache/
 ### Example user template template
 ### Example user template
 
 # IntelliJ project files
 .idea
 *.iml
 out
 gen

backend/requirements.txt

@@ -1,49 +1,49 b''
 repoze.sendmail==4.1
 pyramid==1.7.3
 pyramid_tm==0.12
 pyramid_debugtoolbar
 pyramid_authstack==1.0.1
 SQLAlchemy==1.0.12
 alembic==1.0.8
 webhelpers2==2.0
 transaction==1.4.3
 zope.sqlalchemy==0.7.6
 pyramid_mailer==0.14.1
 redis==2.10.5
 redlock-py==1.0.8
 pyramid_jinja2==2.6.2
 psycopg2==2.7.7
 wtforms==2.1
 celery==3.1.23
 formencode==1.3.0
 psutil==2.1.2
-ziggurat_foundations==0.6.8
+ziggurat_foundations==0.8.3
 bcrypt==3.1.6
 appenlight_client
 markdown==2.5
 colander==1.7
 defusedxml==0.5.0
 dogpile.cache==0.5.7
 pyramid_redis_sessions==1.0.1
 simplejson==3.8.2
 waitress==1.0
 gunicorn==19.9.0
 requests==2.20.0
 requests_oauthlib==0.6.1
 gevent==1.1.1
 gevent-websocket==0.9.5
 pygments==2.1.3
 lxml==4.3.2
 paginate==0.5.4
 paginate-sqlalchemy==0.2.0
 pyelasticsearch==1.4
 six==1.9.0
 mock==1.0.1
 itsdangerous==1.1.0
 camplight==0.9.6
 jira==1.0.7
 python-dateutil==2.5.3
 authomatic==0.1.0.post1
 cryptography==2.6.1
 webassets==0.11.1
 

backend/src/appenlight/celery/tasks.py

@@ -1,660 +1,662 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import bisect
 import collections
 import math
 from datetime import datetime, timedelta
 
 import sqlalchemy as sa
 import pyelasticsearch
 
 from celery.utils.log import get_task_logger
 from zope.sqlalchemy import mark_changed
 from pyramid.threadlocal import get_current_request, get_current_registry
+from ziggurat_foundations.models.services.resource import ResourceService
+
 from appenlight.celery import celery
 from appenlight.models.report_group import ReportGroup
 from appenlight.models import DBSession, Datastores
 from appenlight.models.report import Report
 from appenlight.models.log import Log
 from appenlight.models.metric import Metric
 from appenlight.models.event import Event
 
 from appenlight.models.services.application import ApplicationService
 from appenlight.models.services.event import EventService
 from appenlight.models.services.log import LogService
 from appenlight.models.services.report import ReportService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.user import UserService
 from appenlight.models.tag import Tag
 from appenlight.lib import print_traceback
 from appenlight.lib.utils import parse_proto, in_batches
 from appenlight.lib.ext_json import json
 from appenlight.lib.redis_keys import REDIS_KEYS
 from appenlight.lib.enums import ReportType
 
 log = get_task_logger(__name__)
 
 sample_boundries = list(range(100, 1000, 100)) + \
                    list(range(1000, 10000, 1000)) + \
                    list(range(10000, 100000, 5000))
 
 
 def pick_sample(total_occurences, report_type=None):
     every = 1.0
     position = bisect.bisect_left(sample_boundries, total_occurences)
     if position > 0:
         if report_type == ReportType.not_found:
             divide = 10.0
         else:
             divide = 100.0
         every = sample_boundries[position - 1] / divide
     return total_occurences % every == 0
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_exception_task():
     log.error('test celery log', extra={'location': 'celery'})
     log.warning('test celery log', extra={'location': 'celery'})
     raise Exception('Celery exception test')
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def test_retry_exception_task():
     try:
         import time
 
         time.sleep(1.3)
         log.error('test retry celery log', extra={'location': 'celery'})
         log.warning('test retry celery log', extra={'location': 'celery'})
         raise Exception('Celery exception test')
     except Exception as exc:
         test_retry_exception_task.retry(exc=exc)
 
 
 @celery.task(queue="reports", default_retry_delay=600, max_retries=144)
 def add_reports(resource_id, request_params, dataset, **kwargs):
     proto_version = parse_proto(request_params.get('protocol_version', ''))
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         # we will store solr docs here for single insert
         es_report_docs = {}
         es_report_group_docs = {}
         resource = ApplicationService.by_id(resource_id)
 
         tags = []
         es_slow_calls_docs = {}
         es_reports_stats_rows = {}
         for report_data in dataset:
             # build report details for later
             added_details = 0
             report = Report()
             report.set_data(report_data, resource, proto_version)
             report._skip_ft_index = True
 
             # find latest group in this months partition
             report_group = ReportGroupService.by_hash_and_resource(
                 report.resource_id,
                 report.grouping_hash,
                 since_when=datetime.utcnow().date().replace(day=1)
             )
             occurences = report_data.get('occurences', 1)
             if not report_group:
                 # total reports will be +1 moment later
                 report_group = ReportGroup(grouping_hash=report.grouping_hash,
                                            occurences=0, total_reports=0,
                                            last_report=0,
                                            priority=report.priority,
                                            error=report.error,
                                            first_timestamp=report.start_time)
                 report_group._skip_ft_index = True
                 report_group.report_type = report.report_type
             report.report_group_time = report_group.first_timestamp
             add_sample = pick_sample(report_group.occurences,
                                      report_type=report_group.report_type)
             if add_sample:
                 resource.report_groups.append(report_group)
                 report_group.reports.append(report)
                 added_details += 1
                 DBSession.flush()
                 if report.partition_id not in es_report_docs:
                     es_report_docs[report.partition_id] = []
                 es_report_docs[report.partition_id].append(report.es_doc())
                 tags.extend(list(report.tags.items()))
                 slow_calls = report.add_slow_calls(report_data, report_group)
                 DBSession.flush()
                 for s_call in slow_calls:
                     if s_call.partition_id not in es_slow_calls_docs:
                         es_slow_calls_docs[s_call.partition_id] = []
                     es_slow_calls_docs[s_call.partition_id].append(
                         s_call.es_doc())
                     # try generating new stat rows if needed
             else:
                 # required for postprocessing to not fail later
                 report.report_group = report_group
 
             stat_row = ReportService.generate_stat_rows(
                 report, resource, report_group)
             if stat_row.partition_id not in es_reports_stats_rows:
                 es_reports_stats_rows[stat_row.partition_id] = []
             es_reports_stats_rows[stat_row.partition_id].append(
                 stat_row.es_doc())
 
             # see if we should mark 10th occurence of report
             last_occurences_10 = int(math.floor(report_group.occurences / 10))
             curr_occurences_10 = int(math.floor(
                 (report_group.occurences + report.occurences) / 10))
             last_occurences_100 = int(
                 math.floor(report_group.occurences / 100))
             curr_occurences_100 = int(math.floor(
                 (report_group.occurences + report.occurences) / 100))
             notify_occurences_10 = last_occurences_10 != curr_occurences_10
             notify_occurences_100 = last_occurences_100 != curr_occurences_100
             report_group.occurences = ReportGroup.occurences + occurences
             report_group.last_timestamp = report.start_time
             report_group.summed_duration = ReportGroup.summed_duration + report.duration
             summed_duration = ReportGroup.summed_duration + report.duration
             summed_occurences = ReportGroup.occurences + occurences
             report_group.average_duration = summed_duration / summed_occurences
             report_group.run_postprocessing(report)
             if added_details:
                 report_group.total_reports = ReportGroup.total_reports + 1
                 report_group.last_report = report.id
             report_group.set_notification_info(notify_10=notify_occurences_10,
                                                notify_100=notify_occurences_100)
             DBSession.flush()
             report_group.get_report().notify_channel(report_group)
             if report_group.partition_id not in es_report_group_docs:
                 es_report_group_docs[report_group.partition_id] = []
             es_report_group_docs[report_group.partition_id].append(
                 report_group.es_doc())
 
             action = 'REPORT'
             log_msg = '%s: %s %s, client: %s, proto: %s' % (
                 action,
                 report_data.get('http_status', 'unknown'),
                 str(resource),
                 report_data.get('client'),
                 proto_version)
             log.info(log_msg)
         total_reports = len(dataset)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
         key = REDIS_KEYS['counters']['reports_per_minute'].format(current_time)
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600 * 24)
         key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
             resource.owner_user_id, current_time)
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600)
         key = REDIS_KEYS['counters']['reports_per_hour_per_app'].format(
             resource_id, current_time.replace(minute=0))
         redis_pipeline.incr(key, total_reports)
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
             REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
                 current_time.replace(minute=0)), resource_id)
         redis_pipeline.execute()
 
         add_reports_es(es_report_group_docs, es_report_docs)
         add_reports_slow_calls_es(es_slow_calls_docs)
         add_reports_stats_rows_es(es_reports_stats_rows)
         return True
     except Exception as exc:
         print_traceback(log)
         add_reports.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_es(report_group_docs, report_docs):
     for k, v in report_group_docs.items():
         Datastores.es.bulk_index(k, 'report_group', v, id_field="_id")
     for k, v in report_docs.items():
         Datastores.es.bulk_index(k, 'report', v, id_field="_id",
                                  parent_field='_parent')
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_slow_calls_es(es_docs):
     for k, v in es_docs.items():
         Datastores.es.bulk_index(k, 'log', v)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_reports_stats_rows_es(es_docs):
     for k, v in es_docs.items():
         Datastores.es.bulk_index(k, 'log', v)
 
 
 @celery.task(queue="logs", default_retry_delay=600, max_retries=144)
 def add_logs(resource_id, request_params, dataset, **kwargs):
     proto_version = request_params.get('protocol_version')
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
 
     try:
         es_docs = collections.defaultdict(list)
         resource = ApplicationService.by_id_cached()(resource_id)
         resource = DBSession.merge(resource, load=False)
         ns_pairs = []
         for entry in dataset:
             # gather pk and ns so we can remove older versions of row later
             if entry['primary_key'] is not None:
                 ns_pairs.append({"pk": entry['primary_key'],
                                  "ns": entry['namespace']})
             log_entry = Log()
             log_entry.set_data(entry, resource=resource)
             log_entry._skip_ft_index = True
             resource.logs.append(log_entry)
             DBSession.flush()
             # insert non pk rows first
             if entry['primary_key'] is None:
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
         # 2nd pass to delete all log entries from db foe same pk/ns pair
         if ns_pairs:
             ids_to_delete = []
             es_docs = collections.defaultdict(list)
             es_docs_to_delete = collections.defaultdict(list)
             found_pkey_logs = LogService.query_by_primary_key_and_namespace(
                 list_of_pairs=ns_pairs)
             log_dict = {}
             for log_entry in found_pkey_logs:
                 log_key = (log_entry.primary_key, log_entry.namespace)
                 if log_key not in log_dict:
                     log_dict[log_key] = []
                 log_dict[log_key].append(log_entry)
 
             for ns, entry_list in log_dict.items():
                 entry_list = sorted(entry_list, key=lambda x: x.timestamp)
                 # newest row needs to be indexed in es
                 log_entry = entry_list[-1]
                 # delete everything from pg and ES, leave the last row in pg
                 for e in entry_list[:-1]:
                     ids_to_delete.append(e.log_id)
                     es_docs_to_delete[e.partition_id].append(e.delete_hash)
 
                 es_docs_to_delete[log_entry.partition_id].append(
                     log_entry.delete_hash)
 
                 es_docs[log_entry.partition_id].append(log_entry.es_doc())
 
             if ids_to_delete:
                 query = DBSession.query(Log).filter(
                     Log.log_id.in_(ids_to_delete))
                 query.delete(synchronize_session=False)
             if es_docs_to_delete:
                 # batch this to avoid problems with default ES bulk limits
                 for es_index in es_docs_to_delete.keys():
                     for batch in in_batches(es_docs_to_delete[es_index], 20):
                         query = {'terms': {'delete_hash': batch}}
 
                         try:
                             Datastores.es.delete_by_query(
                                 es_index, 'log', query)
                         except pyelasticsearch.ElasticHttpNotFoundError as exc:
                             msg = 'skipping index {}'.format(es_index)
                             log.info(msg)
 
         total_logs = len(dataset)
 
         log_msg = 'LOG_NEW: %s, entries: %s, proto:%s' % (
             str(resource),
             total_logs,
             proto_version)
         log.info(log_msg)
         # mark_changed(session)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
         key = REDIS_KEYS['counters']['logs_per_minute'].format(current_time)
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600 * 24)
         key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
             resource.owner_user_id, current_time)
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600)
         key = REDIS_KEYS['counters']['logs_per_hour_per_app'].format(
             resource_id, current_time.replace(minute=0))
         redis_pipeline.incr(key, total_logs)
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
             REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
                 current_time.replace(minute=0)), resource_id)
         redis_pipeline.execute()
         add_logs_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         add_logs.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_logs_es(es_docs):
     for k, v in es_docs.items():
         Datastores.es.bulk_index(k, 'log', v)
 
 
 @celery.task(queue="metrics", default_retry_delay=600, max_retries=144)
 def add_metrics(resource_id, request_params, dataset, proto_version):
     current_time = datetime.utcnow().replace(second=0, microsecond=0)
     try:
         resource = ApplicationService.by_id_cached()(resource_id)
         resource = DBSession.merge(resource, load=False)
         es_docs = []
         rows = []
         for metric in dataset:
             tags = dict(metric['tags'])
             server_n = tags.get('server_name', metric['server_name']).lower()
             tags['server_name'] = server_n or 'unknown'
             new_metric = Metric(
                 timestamp=metric['timestamp'],
                 resource_id=resource.resource_id,
                 namespace=metric['namespace'],
                 tags=tags)
             rows.append(new_metric)
             es_docs.append(new_metric.es_doc())
         session = DBSession()
         session.bulk_save_objects(rows)
         session.flush()
 
         action = 'METRICS'
         metrics_msg = '%s: %s, metrics: %s, proto:%s' % (
             action,
             str(resource),
             len(dataset),
             proto_version
         )
         log.info(metrics_msg)
 
         mark_changed(session)
         redis_pipeline = Datastores.redis.pipeline(transaction=False)
         key = REDIS_KEYS['counters']['metrics_per_minute'].format(current_time)
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600 * 24)
         key = REDIS_KEYS['counters']['events_per_minute_per_user'].format(
             resource.owner_user_id, current_time)
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600)
         key = REDIS_KEYS['counters']['metrics_per_hour_per_app'].format(
             resource_id, current_time.replace(minute=0))
         redis_pipeline.incr(key, len(rows))
         redis_pipeline.expire(key, 3600 * 24 * 7)
         redis_pipeline.sadd(
             REDIS_KEYS['apps_that_got_new_data_per_hour'].format(
                 current_time.replace(minute=0)), resource_id)
         redis_pipeline.execute()
         add_metrics_es(es_docs)
         return True
     except Exception as exc:
         print_traceback(log)
         add_metrics.retry(exc=exc)
 
 
 @celery.task(queue="es", default_retry_delay=600, max_retries=144)
 def add_metrics_es(es_docs):
     for doc in es_docs:
         partition = 'rcae_m_%s' % doc['timestamp'].strftime('%Y_%m_%d')
         Datastores.es.index(partition, 'log', doc)
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_user_report_notifications(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
         error_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
             ReportType.error, resource_id)
         slow_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
             ReportType.slow, resource_id)
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
             key = REDIS_KEYS['counters']['report_group_occurences'].format(
                 g_id)
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
             application, 'alert', report_groups=report_groups,
             occurence_dict=occurence_dict)
-        users = set([p.user for p in application.users_for_perm('view')])
+        users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
         report_groups = report_groups.all()
         for user in users:
             UserService.report_notify(user, request, application,
                                       report_groups=report_groups,
                                       occurence_dict=occurence_dict)
         for group in report_groups:
             # marks report_groups as notified
             if not group.notified:
                 group.notified = True
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=5, max_retries=2)
 def check_alerts(resource_id):
     since_when = datetime.utcnow()
     try:
         request = get_current_request()
         application = ApplicationService.by_id(resource_id)
         if not application:
             return
         error_key = REDIS_KEYS[
             'reports_to_notify_per_type_per_app_alerting'].format(
             ReportType.error, resource_id)
         slow_key = REDIS_KEYS[
             'reports_to_notify_per_type_per_app_alerting'].format(
             ReportType.slow, resource_id)
         error_group_ids = Datastores.redis.smembers(error_key)
         slow_group_ids = Datastores.redis.smembers(slow_key)
         Datastores.redis.delete(error_key)
         Datastores.redis.delete(slow_key)
         err_gids = [int(g_id) for g_id in error_group_ids]
         slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
         group_ids = err_gids + slow_gids
         occurence_dict = {}
         for g_id in group_ids:
             key = REDIS_KEYS['counters'][
                 'report_group_occurences_alerting'].format(
                 g_id)
             val = Datastores.redis.get(key)
             Datastores.redis.delete(key)
             if val:
                 occurence_dict[g_id] = int(val)
             else:
                 occurence_dict[g_id] = 1
         report_groups = ReportGroupService.by_ids(group_ids)
         report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
 
         ApplicationService.check_for_groups_alert(
             application, 'alert', report_groups=report_groups,
             occurence_dict=occurence_dict, since_when=since_when)
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=1, max_retries=2)
 def close_alerts():
     log.warning('Checking alerts')
     since_when = datetime.utcnow()
     try:
         event_types = [Event.types['error_report_alert'],
                        Event.types['slow_report_alert'], ]
         statuses = [Event.statuses['active']]
         # get events older than 5 min
         events = EventService.by_type_and_status(
             event_types,
             statuses,
             older_than=(since_when - timedelta(minutes=5)))
         for event in events:
             # see if we can close them
             event.validate_or_close(
                 since_when=(since_when - timedelta(minutes=1)))
     except Exception as exc:
         print_traceback(log)
         raise
 
 
 @celery.task(queue="default", default_retry_delay=600, max_retries=144)
 def update_tag_counter(tag_name, tag_value, count):
     try:
         query = DBSession.query(Tag).filter(Tag.name == tag_name).filter(
             sa.cast(Tag.value, sa.types.TEXT) == sa.cast(json.dumps(tag_value),
                                                          sa.types.TEXT))
         query.update({'times_seen': Tag.times_seen + count,
                       'last_timestamp': datetime.utcnow()},
                      synchronize_session=False)
         session = DBSession()
         mark_changed(session)
         return True
     except Exception as exc:
         print_traceback(log)
         update_tag_counter.retry(exc=exc)
 
 
 @celery.task(queue="default")
 def update_tag_counters():
     """
     Sets task to update counters for application tags
     """
     tags = Datastores.redis.lrange(REDIS_KEYS['seen_tag_list'], 0, -1)
     Datastores.redis.delete(REDIS_KEYS['seen_tag_list'])
     c = collections.Counter(tags)
     for t_json, count in c.items():
         tag_info = json.loads(t_json)
         update_tag_counter.delay(tag_info[0], tag_info[1], count)
 
 
 @celery.task(queue="default")
 def daily_digest():
     """
     Sends daily digest with top 50 error reports
     """
     request = get_current_request()
     apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
     Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
     since_when = datetime.utcnow() - timedelta(hours=8)
     log.warning('Generating daily digests')
     for resource_id in apps:
         resource_id = resource_id.decode('utf8')
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings = {'resource': [resource_id],
                            'tags': [{'name': 'type',
                                      'value': ['error'], 'op': None}],
                            'type': 'error', 'start_date': since_when,
                            'end_date': end_date}
 
         reports = ReportGroupService.get_trending(
             request, filter_settings=filter_settings, limit=50)
 
         application = ApplicationService.by_id(resource_id)
         if application:
-            users = set([p.user for p in application.users_for_perm('view')])
+            users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
             for user in users:
                 user.send_digest(request, application, reports=reports,
                                  since_when=since_when)
 
 
 @celery.task(queue="default")
 def notifications_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     issue notifications
     """
     apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
     Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
     for app in apps:
         log.warning('Notify for app: %s' % app)
         check_user_report_notifications.delay(app.decode('utf8'))
 
 @celery.task(queue="default")
 def alerting_reports():
     """
     Loop that checks redis for info and then issues new tasks to celery to
     perform the following:
     - which applications should have new alerts opened
     """
 
     apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports_alerting'])
     Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports_alerting'])
     for app in apps:
         log.warning('Notify for app: %s' % app)
         check_alerts.delay(app.decode('utf8'))
 
 
 @celery.task(queue="default", soft_time_limit=3600 * 4,
              hard_time_limit=3600 * 4, max_retries=144)
 def logs_cleanup(resource_id, filter_settings):
     request = get_current_request()
     request.tm.begin()
     es_query = {
         "_source": False,
         "size": 5000,
         "query": {
             "filtered": {
                 "filter": {
                     "and": [{"term": {"resource_id": resource_id}}]
                 }
             }
         }
     }
 
     query = DBSession.query(Log).filter(Log.resource_id == resource_id)
     if filter_settings['namespace']:
         query = query.filter(Log.namespace == filter_settings['namespace'][0])
         es_query['query']['filtered']['filter']['and'].append(
             {"term": {"namespace": filter_settings['namespace'][0]}}
         )
     query.delete(synchronize_session=False)
     request.tm.commit()
     result = request.es_conn.search(es_query, index='rcae_l_*',
                                     doc_type='log', es_scroll='1m',
                                     es_search_type='scan')
     scroll_id = result['_scroll_id']
     while True:
         log.warning('log_cleanup, app:{} ns:{} batch'.format(
             resource_id,
             filter_settings['namespace']
         ))
         es_docs_to_delete = []
         result = request.es_conn.send_request(
             'POST', ['_search', 'scroll'],
             body=scroll_id, query_params={"scroll": '1m'})
         scroll_id = result['_scroll_id']
         if not result['hits']['hits']:
             break
         for doc in result['hits']['hits']:
             es_docs_to_delete.append({"id": doc['_id'],
                                       "index": doc['_index']})
 
         for batch in in_batches(es_docs_to_delete, 10):
             Datastores.es.bulk([Datastores.es.delete_op(doc_type='log',
                                                         **to_del)
                                 for to_del in batch])

backend/src/appenlight/forms.py

@@ -1,896 +1,896 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import wtforms
 import formencode
 import re
 import pyramid.threadlocal
 import datetime
 import appenlight.lib.helpers as h
 
-from appenlight.models.user import User
-from appenlight.models.group import Group
+from ziggurat_foundations.models.services.user import UserService
+from ziggurat_foundations.models.services.group import GroupService
 from appenlight.models import DBSession
 from appenlight.models.alert_channel import AlertChannel
 from appenlight.models.integrations import IntegrationException
 from appenlight.models.integrations.campfire import CampfireIntegration
 from appenlight.models.integrations.bitbucket import BitbucketIntegration
 from appenlight.models.integrations.github import GithubIntegration
 from appenlight.models.integrations.flowdock import FlowdockIntegration
 from appenlight.models.integrations.hipchat import HipchatIntegration
 from appenlight.models.integrations.jira import JiraClient
 from appenlight.models.integrations.slack import SlackIntegration
 from appenlight.lib.ext_json import json
 from wtforms.ext.csrf.form import SecureForm
 from wtforms.compat import iteritems
 from collections import defaultdict
 
 _ = str
 
 strip_filter = lambda x: x.strip() if x else None
 uppercase_filter = lambda x: x.upper() if x else None
 
 FALSE_VALUES = ('false', '', False, None)
 
 
 class CSRFException(Exception):
     pass
 
 
 class ReactorForm(SecureForm):
     def __init__(self, formdata=None, obj=None, prefix='', csrf_context=None,
                  **kwargs):
         super(ReactorForm, self).__init__(formdata=formdata, obj=obj,
                                           prefix=prefix,
                                           csrf_context=csrf_context, **kwargs)
         self._csrf_context = csrf_context
 
     def generate_csrf_token(self, csrf_context):
         return csrf_context.session.get_csrf_token()
 
     def validate_csrf_token(self, field):
         request = self._csrf_context or pyramid.threadlocal.get_current_request()
         is_from_auth_token = 'auth:auth_token' in request.effective_principals
         if is_from_auth_token:
             return True
 
         if field.data != field.current_token:
             # try to save the day by using token from angular
             if request.headers.get('X-XSRF-TOKEN') != field.current_token:
                 raise CSRFException('Invalid CSRF token')
 
     @property
     def errors_dict(self):
         r_dict = defaultdict(list)
         for k, errors in self.errors.items():
             r_dict[k].extend([str(e) for e in errors])
         return r_dict
 
     @property
     def errors_json(self):
         return json.dumps(self.errors_dict)
 
     def populate_obj(self, obj, ignore_none=False):
         """
         Populates the attributes of the passed `obj` with data from the form's
         fields.
 
         :note: This is a destructive operation; Any attribute with the same name
                as a field will be overridden. Use with caution.
         """
         if ignore_none:
             for name, field in iteritems(self._fields):
                 if field.data is not None:
                     field.populate_obj(obj, name)
         else:
             for name, field in iteritems(self._fields):
                 field.populate_obj(obj, name)
 
     css_classes = {}
     ignore_labels = {}
 
 
 class SignInForm(ReactorForm):
     came_from = wtforms.HiddenField()
     sign_in_user_name = wtforms.StringField(_('User Name'))
     sign_in_user_password = wtforms.PasswordField(_('Password'))
 
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
     html_attrs = {'sign_in_user_name': {'placeholder': 'Your login'},
                   'sign_in_user_password': {
                       'placeholder': 'Your password'}}
 
 
 from wtforms.widgets import html_params, HTMLString
 
 
 def select_multi_checkbox(field, ul_class='set', **kwargs):
     """Render a multi-checkbox widget"""
     kwargs.setdefault('type', 'checkbox')
     field_id = kwargs.pop('id', field.id)
     html = ['<ul %s>' % html_params(id=field_id, class_=ul_class)]
     for value, label, checked in field.iter_choices():
         choice_id = '%s-%s' % (field_id, value)
         options = dict(kwargs, name=field.name, value=value, id=choice_id)
         if checked:
             options['checked'] = 'checked'
         html.append('<li><input %s /> ' % html_params(**options))
         html.append('<label for="%s">%s</label></li>' % (choice_id, label))
     html.append('</ul>')
     return HTMLString(''.join(html))
 
 
 def button_widget(field, button_cls='ButtonField btn btn-default', **kwargs):
     """Render a button widget"""
     kwargs.setdefault('type', 'button')
     field_id = kwargs.pop('id', field.id)
     kwargs.setdefault('value', field.label.text)
     html = ['<button %s>%s</button>' % (html_params(id=field_id,
                                                      class_=button_cls),
                                          kwargs['value'],)]
     return HTMLString(''.join(html))
 
 
 def clean_whitespace(value):
     if value:
         return value.strip()
     return value
 
 
 def found_username_validator(form, field):
-    user = User.by_user_name(field.data)
+    user = UserService.by_user_name(field.data)
     # sets user to recover in email validator
     form.field_user = user
     if not user:
         raise wtforms.ValidationError('This username does not exist')
 
 
 def found_username_email_validator(form, field):
-    user = User.by_email(field.data)
+    user = UserService.by_email(field.data)
     if not user:
         raise wtforms.ValidationError('Email is incorrect')
 
 
 def unique_username_validator(form, field):
-    user = User.by_user_name(field.data)
+    user = UserService.by_user_name(field.data)
     if user:
         raise wtforms.ValidationError('This username already exists in system')
 
 
 def unique_groupname_validator(form, field):
-    group = Group.by_group_name(field.data)
+    group = GroupService.by_group_name(field.data)
     mod_group = getattr(form, '_modified_group', None)
     if group and (not mod_group or mod_group.id != group.id):
         raise wtforms.ValidationError(
             'This group name already exists in system')
 
 
 def unique_email_validator(form, field):
-    user = User.by_email(field.data)
+    user = UserService.by_email(field.data)
     if user:
         raise wtforms.ValidationError('This email already exists in system')
 
 
 def email_validator(form, field):
     validator = formencode.validators.Email()
     try:
         validator.to_python(field.data)
     except formencode.Invalid as e:
         raise wtforms.ValidationError(e)
 
 
 def unique_alert_email_validator(form, field):
     q = DBSession.query(AlertChannel)
     q = q.filter(AlertChannel.channel_name == 'email')
     q = q.filter(AlertChannel.channel_value == field.data)
     email = q.first()
     if email:
         raise wtforms.ValidationError(
             'This email already exists in alert system')
 
 
 def blocked_email_validator(form, field):
     blocked_emails = [
         'goood-mail.org',
         'shoeonlineblog.com',
         'louboutinemart.com',
         'guccibagshere.com',
         'nikeshoesoutletforsale.com'
     ]
     data = field.data or ''
     domain = data.split('@')[-1]
     if domain in blocked_emails:
         raise wtforms.ValidationError('Don\'t spam')
 
 
 def old_password_validator(form, field):
-    if not field.user.check_password(field.data or ''):
+    if not UserService.check_password(field.user, field.data or ''):
         raise wtforms.ValidationError('You need to enter correct password')
 
 
 class UserRegisterForm(ReactorForm):
     user_name = wtforms.StringField(
         _('User Name'),
         filters=[strip_filter],
         validators=[
             wtforms.validators.Length(min=2, max=30),
             wtforms.validators.Regexp(
                 re.compile(r'^[\.\w-]+$', re.UNICODE),
                 message="Invalid characters used"),
             unique_username_validator,
             wtforms.validators.DataRequired()
         ])
 
     user_password = wtforms.PasswordField(_('User Password'),
                                           filters=[strip_filter],
                                           validators=[
                                               wtforms.validators.Length(min=4),
                                               wtforms.validators.DataRequired()
                                           ])
 
     email = wtforms.StringField(_('Email Address'),
                                 filters=[strip_filter],
                                 validators=[email_validator,
                                             unique_email_validator,
                                             blocked_email_validator,
                                             wtforms.validators.DataRequired()])
     first_name = wtforms.HiddenField(_('First Name'))
     last_name = wtforms.HiddenField(_('Last Name'))
 
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
     html_attrs = {'user_name': {'placeholder': 'Your login'},
                   'user_password': {'placeholder': 'Your password'},
                   'email': {'placeholder': 'Your email'}}
 
 
 class UserCreateForm(UserRegisterForm):
     status = wtforms.BooleanField('User status',
                                   false_values=FALSE_VALUES)
 
 
 class UserUpdateForm(UserCreateForm):
     user_name = None
     user_password = wtforms.PasswordField(_('User Password'),
                                           filters=[strip_filter],
                                           validators=[
                                               wtforms.validators.Length(min=4),
                                               wtforms.validators.Optional()
                                           ])
     email = wtforms.StringField(_('Email Address'),
                                 filters=[strip_filter],
                                 validators=[email_validator,
                                             wtforms.validators.DataRequired()])
 
 
 class LostPasswordForm(ReactorForm):
     email = wtforms.StringField(_('Email Address'),
                                 filters=[strip_filter],
                                 validators=[email_validator,
                                             found_username_email_validator,
                                             wtforms.validators.DataRequired()])
 
     submit = wtforms.SubmitField(_('Reset password'))
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 class ChangePasswordForm(ReactorForm):
     old_password = wtforms.PasswordField(
         'Old Password',
         filters=[strip_filter],
         validators=[old_password_validator,
                     wtforms.validators.DataRequired()])
 
     new_password = wtforms.PasswordField(
         'New Password',
         filters=[strip_filter],
         validators=[wtforms.validators.Length(min=4),
                     wtforms.validators.DataRequired()])
     new_password_confirm = wtforms.PasswordField(
         'Confirm Password',
         filters=[strip_filter],
         validators=[wtforms.validators.EqualTo('new_password'),
                     wtforms.validators.DataRequired()])
     submit = wtforms.SubmitField('Change Password')
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 class CheckPasswordForm(ReactorForm):
     password = wtforms.PasswordField(
         'Password',
         filters=[strip_filter],
         validators=[old_password_validator,
                     wtforms.validators.DataRequired()])
 
 
 class NewPasswordForm(ReactorForm):
     new_password = wtforms.PasswordField(
         'New Password',
         filters=[strip_filter],
         validators=[wtforms.validators.Length(min=4),
                     wtforms.validators.DataRequired()])
     new_password_confirm = wtforms.PasswordField(
         'Confirm Password',
         filters=[strip_filter],
         validators=[wtforms.validators.EqualTo('new_password'),
                     wtforms.validators.DataRequired()])
     submit = wtforms.SubmitField('Set Password')
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 class CORSTextAreaField(wtforms.StringField):
     """
     This field represents an HTML ``<textarea>`` and can be used to take
     multi-line input.
     """
     widget = wtforms.widgets.TextArea()
 
     def process_formdata(self, valuelist):
         self.data = []
         if valuelist:
             data = [x.strip() for x in valuelist[0].split('\n')]
             for d in data:
                 if not d:
                     continue
                 if d.startswith('www.'):
                     d = d[4:]
                 if data:
                     self.data.append(d)
         else:
             self.data = []
         self.data = '\n'.join(self.data)
 
 
 class ApplicationCreateForm(ReactorForm):
     resource_name = wtforms.StringField(
         _('Application name'),
         filters=[strip_filter],
         validators=[wtforms.validators.Length(min=1),
                     wtforms.validators.DataRequired()])
 
     domains = CORSTextAreaField(
         _('Domain names for CORS headers '),
         validators=[wtforms.validators.Length(min=1),
                     wtforms.validators.Optional()],
         description='Required for Javascript error '
                     'tracking (one line one domain, skip http:// part)')
 
     submit = wtforms.SubmitField(_('Create Application'))
 
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
     html_attrs = {'resource_name': {'placeholder': 'Application Name'},
                   'uptime_url': {'placeholder': 'http://somedomain.com'}}
 
 
 class ApplicationUpdateForm(ApplicationCreateForm):
     default_grouping = wtforms.SelectField(
         _('Default grouping for errors'),
         choices=[('url_type', 'Error Type + location',),
                  ('url_traceback', 'Traceback + location',),
                  ('traceback_server', 'Traceback + Server',)],
         default='url_traceback')
 
     error_report_threshold = wtforms.IntegerField(
         _('Alert on error reports'),
         validators=[
             wtforms.validators.NumberRange(min=1),
             wtforms.validators.DataRequired()
         ],
         description='Application requires to send at least this amount of '
                     'error reports per minute to open alert'
     )
 
     slow_report_threshold = wtforms.IntegerField(
         _('Alert on slow reports'),
         validators=[wtforms.validators.NumberRange(min=1),
                     wtforms.validators.DataRequired()],
         description='Application requires to send at least this amount of '
                     'slow reports per minute to open alert')
 
     allow_permanent_storage = wtforms.BooleanField(
         _('Permanent logs'),
         false_values=FALSE_VALUES,
         description=_(
             'Allow permanent storage of logs in separate DB partitions'))
 
     submit = wtforms.SubmitField(_('Create Application'))
 
 
 class UserSearchSchemaForm(ReactorForm):
     user_name = wtforms.StringField('User Name',
                                     filters=[strip_filter], )
 
     submit = wtforms.SubmitField(_('Search User'))
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
     '<li class="user_exists"><span></span></li>'
 
 
 class YesNoForm(ReactorForm):
     no = wtforms.SubmitField('No', default='')
     yes = wtforms.SubmitField('Yes', default='')
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 status_codes = [('', 'All',), ('500', '500',), ('404', '404',)]
 
 priorities = [('', 'All',)]
 for i in range(1, 11):
     priorities.append((str(i), str(i),))
 
 report_status_choices = [('', 'All',),
                          ('never_reviewed', 'Never revieved',),
                          ('reviewed', 'Revieved',),
                          ('public', 'Public',),
                          ('fixed', 'Fixed',), ]
 
 
 class ReportBrowserForm(ReactorForm):
     applications = wtforms.SelectMultipleField('Applications',
                                                widget=select_multi_checkbox)
     http_status = wtforms.SelectField('HTTP Status', choices=status_codes)
     priority = wtforms.SelectField('Priority', choices=priorities, default='')
     start_date = wtforms.DateField('Start Date')
     end_date = wtforms.DateField('End Date')
     error = wtforms.StringField('Error')
     url_path = wtforms.StringField('URL Path')
     url_domain = wtforms.StringField('URL Domain')
     report_status = wtforms.SelectField('Report status',
                                         choices=report_status_choices,
                                         default='')
     submit = wtforms.SubmitField('<span class="glyphicon glyphicon-search">'
                                  '</span> Filter results',
                                  widget=button_widget)
 
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 slow_report_status_choices = [('', 'All',),
                               ('never_reviewed', 'Never revieved',),
                               ('reviewed', 'Revieved',),
                               ('public', 'Public',), ]
 
 
 class BulkOperationForm(ReactorForm):
     applications = wtforms.SelectField('Applications')
     start_date = wtforms.DateField(
         'Start Date',
         default=lambda: datetime.datetime.utcnow() - datetime.timedelta(
             days=90))
     end_date = wtforms.DateField('End Date')
     confirm = wtforms.BooleanField(
         'Confirm operation',
         validators=[wtforms.validators.DataRequired()])
 
 
 class LogBrowserForm(ReactorForm):
     applications = wtforms.SelectMultipleField('Applications',
                                                widget=select_multi_checkbox)
     start_date = wtforms.DateField('Start Date')
     log_level = wtforms.StringField('Log level')
     message = wtforms.StringField('Message')
     namespace = wtforms.StringField('Namespace')
     submit = wtforms.SubmitField(
         '<span class="glyphicon glyphicon-search"></span> Filter results',
         widget=button_widget)
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 class CommentForm(ReactorForm):
     body = wtforms.TextAreaField('Comment', validators=[
         wtforms.validators.Length(min=1),
         wtforms.validators.DataRequired()
     ])
     submit = wtforms.SubmitField('Comment', )
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 class EmailChannelCreateForm(ReactorForm):
     email = wtforms.StringField(_('Email Address'),
                                 filters=[strip_filter],
                                 validators=[email_validator,
                                             unique_alert_email_validator,
                                             wtforms.validators.DataRequired()])
     submit = wtforms.SubmitField('Add email channel', )
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 def gen_user_profile_form():
     class UserProfileForm(ReactorForm):
         email = wtforms.StringField(
             _('Email Address'),
             validators=[email_validator, wtforms.validators.DataRequired()])
         first_name = wtforms.StringField(_('First Name'))
         last_name = wtforms.StringField(_('Last Name'))
         company_name = wtforms.StringField(_('Company Name'))
         company_address = wtforms.TextAreaField(_('Company Address'))
         zip_code = wtforms.StringField(_('ZIP code'))
         city = wtforms.StringField(_('City'))
         notifications = wtforms.BooleanField('Account notifications',
                                              false_values=FALSE_VALUES)
         submit = wtforms.SubmitField(_('Update Account'))
         ignore_labels = ['submit']
         css_classes = {'submit': 'btn btn-primary'}
 
     return UserProfileForm
 
 
 class PurgeAppForm(ReactorForm):
     resource_id = wtforms.HiddenField(
         'App Id',
         validators=[wtforms.validators.DataRequired()])
     days = wtforms.IntegerField(
         'Days',
         validators=[wtforms.validators.DataRequired()])
     password = wtforms.PasswordField(
         'Admin Password',
         validators=[old_password_validator, wtforms.validators.DataRequired()])
     submit = wtforms.SubmitField(_('Purge Data'))
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 class IntegrationRepoForm(ReactorForm):
     host_name = wtforms.StringField("Service Host", default='')
     user_name = wtforms.StringField(
         "User Name",
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired(),
                     wtforms.validators.Length(min=1)])
     repo_name = wtforms.StringField(
         "Repo Name",
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired(),
                     wtforms.validators.Length(min=1)])
 
 
 class IntegrationBitbucketForm(IntegrationRepoForm):
     host_name = wtforms.StringField("Service Host",
                                     default='https://bitbucket.org')
 
     def validate_user_name(self, field):
         try:
             request = pyramid.threadlocal.get_current_request()
             client = BitbucketIntegration.create_client(
                 request,
                 self.user_name.data,
                 self.repo_name.data)
             client.get_assignees()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
 
 class IntegrationGithubForm(IntegrationRepoForm):
     host_name = wtforms.StringField("Service Host",
                                     default='https://github.com')
 
     def validate_user_name(self, field):
         try:
             request = pyramid.threadlocal.get_current_request()
             client = GithubIntegration.create_client(
                 request,
                 self.user_name.data,
                 self.repo_name.data)
             client.get_assignees()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
             raise wtforms.validators.ValidationError(str(e))
 
 
 def filter_rooms(data):
     if data is not None:
         rooms = data.split(',')
         return ','.join([r.strip() for r in rooms])
 
 
 class IntegrationCampfireForm(ReactorForm):
     account = wtforms.StringField(
         'Account',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     api_token = wtforms.StringField(
         'Api Token',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     rooms = wtforms.StringField('Room ID list', filters=[filter_rooms])
 
     def validate_api_token(self, field):
         try:
             client = CampfireIntegration.create_client(self.api_token.data,
                                                        self.account.data)
             client.get_account()
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
     def validate_rooms(self, field):
         if not field.data:
             return
         client = CampfireIntegration.create_client(self.api_token.data,
                                                    self.account.data)
 
         try:
             room_list = [r['id'] for r in client.get_rooms()]
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
         rooms = field.data.split(',')
         if len(rooms) > 3:
             msg = 'You can use up to 3 room ids'
             raise wtforms.validators.ValidationError(msg)
         if rooms:
             for room_id in rooms:
                 if int(room_id) not in room_list:
                     msg = "Room %s doesn't exist"
                     raise wtforms.validators.ValidationError(msg % room_id)
                 if not room_id.strip().isdigit():
                     msg = 'You must use only integers for room ids'
                     raise wtforms.validators.ValidationError(msg)
 
     submit = wtforms.SubmitField(_('Connect to Campfire'))
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 def filter_rooms(data):
     if data is not None:
         rooms = data.split(',')
         return ','.join([r.strip() for r in rooms])
 
 
 class IntegrationHipchatForm(ReactorForm):
     api_token = wtforms.StringField(
         'Api Token',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     rooms = wtforms.StringField(
         'Room ID list',
         filters=[filter_rooms],
         validators=[wtforms.validators.DataRequired()])
 
     def validate_rooms(self, field):
         if not field.data:
             return
         client = HipchatIntegration.create_client(self.api_token.data)
         rooms = field.data.split(',')
         if len(rooms) > 3:
             msg = 'You can use up to 3 room ids'
             raise wtforms.validators.ValidationError(msg)
         if rooms:
             for room_id in rooms:
                 if not room_id.strip().isdigit():
                     msg = 'You must use only integers for room ids'
                     raise wtforms.validators.ValidationError(msg)
                 try:
                     client.send({
                         "message_format": 'text',
                         "message": "testing for room existence",
                         "from": "AppEnlight",
                         "room_id": room_id,
                         "color": "green"
                     })
                 except IntegrationException as exc:
                     msg = 'Room id: %s exception: %s'
                     raise wtforms.validators.ValidationError(msg % (room_id,
                                                                     exc))
 
 
 class IntegrationFlowdockForm(ReactorForm):
     api_token = wtforms.StringField('API Token',
                                     filters=[strip_filter],
                                     validators=[
                                         wtforms.validators.DataRequired()
                                     ], )
 
     def validate_api_token(self, field):
         try:
             client = FlowdockIntegration.create_client(self.api_token.data)
             registry = pyramid.threadlocal.get_current_registry()
             payload = {
                 "source": registry.settings['mailing.from_name'],
                 "from_address": registry.settings['mailing.from_email'],
                 "subject": "Integration test",
                 "content": "If you can see this it was successful",
                 "tags": ["appenlight"],
                 "link": registry.settings['mailing.app_url']
             }
             client.send_to_inbox(payload)
         except IntegrationException as e:
             raise wtforms.validators.ValidationError(str(e))
 
 
 class IntegrationSlackForm(ReactorForm):
     webhook_url = wtforms.StringField(
         'Reports webhook',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
 
     def validate_webhook_url(self, field):
         registry = pyramid.threadlocal.get_current_registry()
         client = SlackIntegration.create_client(field.data)
         link = "<%s|%s>" % (registry.settings['mailing.app_url'],
                             registry.settings['mailing.from_name'])
         test_data = {
             "username": "AppEnlight",
             "icon_emoji": ":fire:",
             "attachments": [
                 {"fallback": "Testing integration channel: %s" % link,
                  "pretext": "Testing integration channel:  %s" % link,
                  "color": "good",
                  "fields": [
                      {
                          "title": "Status",
                          "value": "Integration is working fine",
                          "short": False
                      }
                  ]}
             ]
         }
         try:
             client.make_request(data=test_data)
         except IntegrationException as exc:
             raise wtforms.validators.ValidationError(str(exc))
 
 
 class IntegrationWebhooksForm(ReactorForm):
     reports_webhook = wtforms.StringField(
         'Reports webhook',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     alerts_webhook = wtforms.StringField(
         'Alerts webhook',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     submit = wtforms.SubmitField(_('Setup webhooks'))
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-primary'}
 
 
 class IntegrationJiraForm(ReactorForm):
     host_name = wtforms.StringField(
         'Server URL',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     user_name = wtforms.StringField(
         'Username',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     password = wtforms.PasswordField(
         'Password',
         filters=[strip_filter],
         validators=[wtforms.validators.DataRequired()])
     project = wtforms.StringField(
         'Project key',
         filters=[uppercase_filter, strip_filter],
         validators=[wtforms.validators.DataRequired()])
 
     def validate_project(self, field):
         if not field.data:
             return
         try:
             client = JiraClient(self.user_name.data,
                                 self.password.data,
                                 self.host_name.data,
                                 self.project.data)
         except Exception as exc:
             raise wtforms.validators.ValidationError(str(exc))
 
         room_list = [r.key.upper() for r in client.get_projects()]
         if field.data.upper() not in room_list:
             msg = "Project %s doesn\t exist in your Jira Instance"
             raise wtforms.validators.ValidationError(msg % field.data)
 
 
 def get_deletion_form(resource):
     class F(ReactorForm):
         application_name = wtforms.StringField(
             'Application Name',
             filters=[strip_filter],
             validators=[wtforms.validators.AnyOf([resource.resource_name])])
         resource_id = wtforms.HiddenField(default=resource.resource_id)
         submit = wtforms.SubmitField(_('Delete my application'))
         ignore_labels = ['submit']
         css_classes = {'submit': 'btn btn-danger'}
 
     return F
 
 
 class ChangeApplicationOwnerForm(ReactorForm):
     password = wtforms.PasswordField(
         'Password',
         filters=[strip_filter],
         validators=[old_password_validator,
                     wtforms.validators.DataRequired()])
 
     user_name = wtforms.StringField(
         'New owners username',
         filters=[strip_filter],
         validators=[found_username_validator,
                     wtforms.validators.DataRequired()])
     submit = wtforms.SubmitField(_('Transfer ownership of application'))
     ignore_labels = ['submit']
     css_classes = {'submit': 'btn btn-danger'}
 
 
 def default_filename():
     return 'Invoice %s' % datetime.datetime.utcnow().strftime('%Y/%m')
 
 
 class FileUploadForm(ReactorForm):
     title = wtforms.StringField('File Title',
                                 default=default_filename,
                                 validators=[wtforms.validators.DataRequired()])
     file = wtforms.FileField('File')
 
     def validate_file(self, field):
         if not hasattr(field.data, 'file'):
             raise wtforms.ValidationError('File is missing')
 
     submit = wtforms.SubmitField(_('Upload'))
 
 
 def get_partition_deletion_form(es_indices, pg_indices):
     class F(ReactorForm):
         es_index = wtforms.SelectMultipleField('Elasticsearch',
                                                choices=[(ix, '') for ix in
                                                         es_indices])
         pg_index = wtforms.SelectMultipleField('pg',
                                                choices=[(ix, '') for ix in
                                                         pg_indices])
         confirm = wtforms.TextField('Confirm',
                                     filters=[uppercase_filter, strip_filter],
                                     validators=[
                                         wtforms.validators.AnyOf(['CONFIRM']),
                                         wtforms.validators.DataRequired()])
         ignore_labels = ['submit']
         css_classes = {'submit': 'btn btn-danger'}
 
     return F
 
 
 class GroupCreateForm(ReactorForm):
     group_name = wtforms.StringField(
         _('Group Name'),
         filters=[strip_filter],
         validators=[
             wtforms.validators.Length(min=2, max=50),
             unique_groupname_validator,
             wtforms.validators.DataRequired()
         ])
     description = wtforms.StringField(_('Group description'))
 
 
 time_choices = [(k, v['label'],) for k, v in h.time_deltas.items()]
 
 
 class AuthTokenCreateForm(ReactorForm):
     description = wtforms.StringField(_('Token description'))
     expires = wtforms.SelectField('Expires',
                                   coerce=lambda x: x,
                                   choices=time_choices,
                                   validators=[wtforms.validators.Optional()])

backend/src/appenlight/lib/request.py

@@ -1,135 +1,135 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import json
 
 from pyramid.security import unauthenticated_userid
 
 import appenlight.lib.helpers as helpers
 
 from authomatic.providers import oauth2, oauth1
 from authomatic import Authomatic
-from appenlight.models.user import User
+from ziggurat_foundations.models.services.user import UserService
 
 
 class CSRFException(Exception):
     pass
 
 
 class JSONException(Exception):
     pass
 
 
 def get_csrf_token(request):
     return request.session.get_csrf_token()
 
 
 def safe_json_body(request):
     """
     Returns None if json body is missing or erroneous
     """
     try:
         return request.json_body
     except ValueError:
         return None
 
 
 def unsafe_json_body(request):
     """
     Throws JSONException if json can't deserialize
     """
     try:
         return request.json_body
     except ValueError:
         raise JSONException('Incorrect JSON')
 
 
 def get_user(request):
     if not request.path_info.startswith('/static'):
         user_id = unauthenticated_userid(request)
         try:
             user_id = int(user_id)
         except Exception:
             return None
 
         if user_id:
-            user = User.by_id(user_id)
+            user = UserService.by_id(user_id)
             if user:
                 request.environ['appenlight.username'] = '%d:%s' % (
                     user_id, user.user_name)
             return user
         else:
             return None
 
 
 def es_conn(request):
     return request.registry.es_conn
 
 
 def add_flash_to_headers(request, clear=True):
     """
     Adds pending flash messages to response, if clear is true clears out the
     flash queue
     """
     flash_msgs = helpers.get_type_formatted_flash(request)
     request.response.headers['x-flash-messages'] = json.dumps(flash_msgs)
     helpers.clear_flash(request)
 
 
 def get_authomatic(request):
     settings = request.registry.settings
     # authomatic social auth
     authomatic_conf = {
         # callback http://yourapp.com/social_auth/twitter
         'twitter': {
             'class_': oauth1.Twitter,
             'consumer_key': settings.get('authomatic.pr.twitter.key', ''),
             'consumer_secret': settings.get('authomatic.pr.twitter.secret',
                                             ''),
         },
         # callback http://yourapp.com/social_auth/facebook
         'facebook': {
             'class_': oauth2.Facebook,
             'consumer_key': settings.get('authomatic.pr.facebook.app_id', ''),
             'consumer_secret': settings.get('authomatic.pr.facebook.secret',
                                             ''),
             'scope': ['email'],
         },
         # callback http://yourapp.com/social_auth/google
         'google': {
             'class_': oauth2.Google,
             'consumer_key': settings.get('authomatic.pr.google.key', ''),
             'consumer_secret': settings.get(
                 'authomatic.pr.google.secret', ''),
             'scope': ['profile', 'email'],
         },
         'github': {
             'class_': oauth2.GitHub,
             'consumer_key': settings.get('authomatic.pr.github.key', ''),
             'consumer_secret': settings.get(
                 'authomatic.pr.github.secret', ''),
             'scope': ['repo', 'public_repo', 'user:email'],
             'access_headers': {'User-Agent': 'AppEnlight'},
         },
         'bitbucket': {
             'class_': oauth1.Bitbucket,
             'consumer_key': settings.get('authomatic.pr.bitbucket.key', ''),
             'consumer_secret': settings.get(
                 'authomatic.pr.bitbucket.secret', '')
         }
     }
     return Authomatic(
         config=authomatic_conf, secret=settings['authomatic.secret'])

backend/src/appenlight/lib/utils/__init__.py

@@ -1,490 +1,491 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """
 Utility functions.
 """
 import logging
 import requests
 import hashlib
 import json
 import copy
 import uuid
 import appenlight.lib.helpers as h
 from collections import namedtuple
 from datetime import timedelta, datetime, date
 from dogpile.cache.api import NO_VALUE
 from appenlight.models import Datastores
 from appenlight.validators import (LogSearchSchema,
                                    TagListSchema,
                                    accepted_search_params)
 from itsdangerous import TimestampSigner
 from ziggurat_foundations.permissions import ALL_PERMISSIONS
+from ziggurat_foundations.models.services.user import UserService
 from dateutil.relativedelta import relativedelta
 from dateutil.rrule import rrule, MONTHLY, DAILY
 
 log = logging.getLogger(__name__)
 
 
 Stat = namedtuple('Stat', 'start_interval value')
 
 
 def default_extractor(item):
     """
     :param item - item to extract date from
     """
     if hasattr(item, 'start_interval'):
         return item.start_interval
     return item['start_interval']
 
 
 # fast gap generator
 def gap_gen_default(start, step, itemiterator, end_time=None,
                     iv_extractor=None):
     """ generates a list of time/value items based on step and itemiterator
         if there are entries missing from iterator time/None will be returned
         instead
     :param start - datetime - what time should we start generating our values
     :param step - timedelta - stepsize
     :param itemiterator - iterable - we will check this iterable for values
     corresponding to generated steps
     :param end_time - datetime - when last step is >= end_time stop iterating
     :param iv_extractor - extracts current step from iterable items
     """
 
     if not iv_extractor:
         iv_extractor = default_extractor
 
     next_step = start
     minutes = step.total_seconds() / 60.0
     while next_step.minute % minutes != 0:
         next_step = next_step.replace(minute=next_step.minute - 1)
     for item in itemiterator:
         item_start_interval = iv_extractor(item)
         # do we have a match for current time step in our data?
         # no gen a new tuple with 0 values
         while next_step < item_start_interval:
             yield Stat(next_step, None)
             next_step = next_step + step
         if next_step == item_start_interval:
             yield Stat(item_start_interval, item)
             next_step = next_step + step
     if end_time:
         while next_step < end_time:
             yield Stat(next_step, None)
             next_step = next_step + step
 
 
 class DateTimeEncoder(json.JSONEncoder):
     """ Simple datetime to ISO encoder for json serialization"""
 
     def default(self, obj):
         if isinstance(obj, date):
             return obj.isoformat()
         if isinstance(obj, datetime):
             return obj.isoformat()
         return json.JSONEncoder.default(self, obj)
 
 
 def channelstream_request(secret, endpoint, payload, throw_exceptions=False,
                    servers=None):
     responses = []
     if not servers:
         servers = []
 
     signer = TimestampSigner(secret)
     sig_for_server = signer.sign(endpoint)
     for secret, server in [(s['secret'], s['server']) for s in servers]:
         response = {}
         secret_headers = {'x-channelstream-secret': sig_for_server,
                           'x-channelstream-endpoint': endpoint,
                           'Content-Type': 'application/json'}
         url = '%s%s' % (server, endpoint)
         try:
             response = requests.post(url,
                                      data=json.dumps(payload,
                                                      cls=DateTimeEncoder),
                                      headers=secret_headers,
                                      verify=False,
                                      timeout=2).json()
         except requests.exceptions.RequestException as e:
             if throw_exceptions:
                 raise
         responses.append(response)
     return responses
 
 
 def add_cors_headers(response):
     # allow CORS
     response.headers.add('Access-Control-Allow-Origin', '*')
     response.headers.add('XDomainRequestAllowed', '1')
     response.headers.add('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
     # response.headers.add('Access-Control-Allow-Credentials', 'true')
     response.headers.add('Access-Control-Allow-Headers',
                          'Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Pragma, Origin, Connection, Referer, Cookie')
     response.headers.add('Access-Control-Max-Age', '86400')
 
 
 from sqlalchemy.sql import compiler
 from psycopg2.extensions import adapt as sqlescape
 
 
 # or use the appropiate escape function from your db driver
 
 def compile_query(query):
     dialect = query.session.bind.dialect
     statement = query.statement
     comp = compiler.SQLCompiler(dialect, statement)
     comp.compile()
     enc = dialect.encoding
     params = {}
     for k, v in comp.params.items():
         if isinstance(v, str):
             v = v.encode(enc)
         params[k] = sqlescape(v)
     return (comp.string.encode(enc) % params).decode(enc)
 
 
 def convert_es_type(input_data):
     """
     This might need to convert some text or other types to corresponding ES types
     """
     return str(input_data)
 
 
 ProtoVersion = namedtuple('ProtoVersion', ['major', 'minor', 'patch'])
 
 
 def parse_proto(input_data):
     try:
         parts = [int(x) for x in input_data.split('.')]
         while len(parts) < 3:
             parts.append(0)
         return ProtoVersion(*parts)
     except Exception as e:
         log.info('Unknown protocol version: %s' % e)
     return ProtoVersion(99, 99, 99)
 
 
 def es_index_name_limiter(start_date=None, end_date=None, months_in_past=6,
                           ixtypes=None):
     """
     This function limits the search to 6 months by default so we don't have to
     query 300 elasticsearch indices for 20 years of historical data for example
     """
 
     # should be cached later
     def get_possible_names():
         return list(Datastores.es.aliases().keys())
 
     possible_names = get_possible_names()
     es_index_types = []
     if not ixtypes:
         ixtypes = ['reports', 'metrics', 'logs']
     for t in ixtypes:
         if t == 'reports':
             es_index_types.append('rcae_r_%s')
         elif t == 'logs':
             es_index_types.append('rcae_l_%s')
         elif t == 'metrics':
             es_index_types.append('rcae_m_%s')
         elif t == 'uptime':
             es_index_types.append('rcae_u_%s')
         elif t == 'slow_calls':
             es_index_types.append('rcae_sc_%s')
 
     if start_date:
         start_date = copy.copy(start_date)
     else:
         if not end_date:
             end_date = datetime.utcnow()
         start_date = end_date + relativedelta(months=months_in_past * -1)
 
     if not end_date:
         end_date = start_date + relativedelta(months=months_in_past)
 
     index_dates = list(rrule(MONTHLY,
                              dtstart=start_date.date().replace(day=1),
                              until=end_date.date(),
                              count=36))
     index_names = []
     for ix_type in es_index_types:
         to_extend = [ix_type % d.strftime('%Y_%m') for d in index_dates
                      if ix_type % d.strftime('%Y_%m') in possible_names]
         index_names.extend(to_extend)
         for day in list(rrule(DAILY, dtstart=start_date.date(),
                               until=end_date.date(), count=366)):
             ix_name = ix_type % day.strftime('%Y_%m_%d')
             if ix_name in possible_names:
                 index_names.append(ix_name)
     return index_names
 
 
 def build_filter_settings_from_query_dict(
         request, params=None, override_app_ids=None,
         resource_permissions=None):
     """
     Builds list of normalized search terms for ES from query params
     ensuring application list is restricted to only applications user
     has access to
 
     :param params (dictionary)
     :param override_app_ids - list of application id's to use instead of
     applications user normally has access to
     """
     params = copy.deepcopy(params)
     applications = []
     if not resource_permissions:
         resource_permissions = ['view']
 
     if request.user:
-        applications = request.user.resources_with_perms(
-            resource_permissions, resource_types=['application'])
+        applications = UserService.resources_with_perms(
+            request.user, resource_permissions, resource_types=['application'])
 
     # CRITICAL - this ensures our resultset is limited to only the ones
     # user has view permissions
     all_possible_app_ids = set([app.resource_id for app in applications])
 
     # if override is preset we force permission for app to be present
     # this allows users to see dashboards and applications they would
     # normally not be able to
 
     if override_app_ids:
         all_possible_app_ids = set(override_app_ids)
 
     schema = LogSearchSchema().bind(resources=all_possible_app_ids)
     tag_schema = TagListSchema()
     filter_settings = schema.deserialize(params)
     tag_list = []
     for k, v in list(filter_settings.items()):
         if k in accepted_search_params:
             continue
         tag_list.append({"name": k, "value": v, "op": 'eq'})
         # remove the key from filter_settings
         filter_settings.pop(k, None)
     tags = tag_schema.deserialize(tag_list)
     filter_settings['tags'] = tags
     return filter_settings
 
 
 def gen_uuid():
     return str(uuid.uuid4())
 
 
 def gen_uuid4_sha_hex():
     return hashlib.sha1(uuid.uuid4().bytes).hexdigest()
 
 
 def permission_tuple_to_dict(data):
     out = {
         "user_name": None,
         "perm_name": data.perm_name,
         "owner": data.owner,
         "type": data.type,
         "resource_name": None,
         "resource_type": None,
         "resource_id": None,
         "group_name": None,
         "group_id": None
     }
     if data.user:
         out["user_name"] = data.user.user_name
     if data.perm_name == ALL_PERMISSIONS:
         out['perm_name'] = '__all_permissions__'
     if data.resource:
         out['resource_name'] = data.resource.resource_name
         out['resource_type'] = data.resource.resource_type
         out['resource_id'] = data.resource.resource_id
     if data.group:
         out['group_name'] = data.group.group_name
         out['group_id'] = data.group.id
     return out
 
 
 def get_cached_buckets(request, stats_since, end_time, fn, cache_key,
                        gap_gen=None, db_session=None, step_interval=None,
                        iv_extractor=None,
                        rerange=False, *args, **kwargs):
     """ Takes "fn" that should return some data and tries to load the data
     dividing it into daily buckets - if the stats_since and end time give a
     delta bigger than 24hours, then only "todays" data is computed on the fly
 
     :param request: (request) request object
     :param stats_since: (datetime) start date of buckets range
     :param end_time: (datetime) end date of buckets range - utcnow() if None
     :param fn: (callable) callable to use to populate buckets should have
     following signature:
         def get_data(request, since_when, until, *args, **kwargs):
 
     :param cache_key: (string) cache key that will be used to build bucket
     caches
     :param gap_gen: (callable) gap generator - should return step intervals
     to use with out `fn` callable
     :param db_session: (Session) sqlalchemy session
     :param step_interval: (timedelta) optional step interval if we want to
     override the default determined from total start/end time delta
     :param iv_extractor: (callable) used to get step intervals from data
     returned by `fn` callable
     :param rerange: (bool) handy if we want to change ranges from hours to
     days when cached data is missing - will shorten execution time if `fn`
     callable supports that and we are working with multiple rows - like metrics
     :param args:
     :param kwargs:
 
     :return: iterable
     """
     if not end_time:
         end_time = datetime.utcnow().replace(second=0, microsecond=0)
     delta = end_time - stats_since
     # if smaller than 3 days we want to group by 5min else by 1h,
     # for 60 min group by min
     if not gap_gen:
         gap_gen = gap_gen_default
     if not iv_extractor:
         iv_extractor = default_extractor
 
     # do not use custom interval if total time range with new iv would exceed
     # end time
     if not step_interval or stats_since + step_interval >= end_time:
         if delta < h.time_deltas.get('12h')['delta']:
             step_interval = timedelta(seconds=60)
         elif delta < h.time_deltas.get('3d')['delta']:
             step_interval = timedelta(seconds=60 * 5)
         elif delta > h.time_deltas.get('2w')['delta']:
             step_interval = timedelta(days=1)
         else:
             step_interval = timedelta(minutes=60)
 
     if step_interval >= timedelta(minutes=60):
         log.info('cached_buckets:{}: adjusting start time '
                  'for hourly or daily intervals'.format(cache_key))
         stats_since = stats_since.replace(hour=0, minute=0)
 
     ranges = [i.start_interval for i in list(gap_gen(stats_since,
                                                      step_interval, [],
                                                      end_time=end_time))]
     buckets = {}
     storage_key = 'buckets:' + cache_key + '{}|{}'
     # this means we basicly cache per hour in 3-14 day intervals but i think
     # its fine at this point - will be faster than db access anyways
 
     if len(ranges) >= 1:
         last_ranges = [ranges[-1]]
     else:
         last_ranges = []
     if step_interval >= timedelta(minutes=60):
         for r in ranges:
             k = storage_key.format(step_interval.total_seconds(), r)
             value = request.registry.cache_regions.redis_day_30.get(k)
             # last buckets are never loaded from cache
             is_last_result = (
                 r >= end_time - timedelta(hours=6) or r in last_ranges)
             if value is not NO_VALUE and not is_last_result:
                 log.info("cached_buckets:{}: "
                          "loading range {} from cache".format(cache_key, r))
                 buckets[r] = value
             else:
                 log.info("cached_buckets:{}: "
                          "loading range {} from storage".format(cache_key, r))
                 range_size = step_interval
                 if (step_interval == timedelta(minutes=60) and
                         not is_last_result and rerange):
                     range_size = timedelta(days=1)
                     r = r.replace(hour=0, minute=0)
                     log.info("cached_buckets:{}: "
                              "loading collapsed "
                              "range {} {}".format(cache_key, r,
                                                   r + range_size))
                 bucket_data = fn(
                     request, r, r + range_size, step_interval,
                     gap_gen, bucket_count=len(ranges), *args, **kwargs)
                 for b in bucket_data:
                     b_iv = iv_extractor(b)
                     buckets[b_iv] = b
                     k2 = storage_key.format(
                         step_interval.total_seconds(), b_iv)
                     request.registry.cache_regions.redis_day_30.set(k2, b)
         log.info("cached_buckets:{}: saving cache".format(cache_key))
     else:
         # bucket count is 1 for short time ranges <= 24h from now
         bucket_data = fn(request, stats_since, end_time, step_interval,
                          gap_gen, bucket_count=1, *args, **kwargs)
         for b in bucket_data:
             buckets[iv_extractor(b)] = b
     return buckets
 
 
 def get_cached_split_data(request, stats_since, end_time, fn, cache_key,
                           db_session=None, *args, **kwargs):
     """ Takes "fn" that should return some data and tries to load the data
     dividing it into 2 buckets - cached "since_from" bucket and "today"
     bucket - then the data can be reduced into single value
 
     Data is cached if the stats_since and end time give a delta bigger
     than 24hours - then only 24h is computed on the fly
     """
     if not end_time:
         end_time = datetime.utcnow().replace(second=0, microsecond=0)
     delta = end_time - stats_since
 
     if delta >= timedelta(minutes=60):
         log.info('cached_split_data:{}: adjusting start time '
                  'for hourly or daily intervals'.format(cache_key))
         stats_since = stats_since.replace(hour=0, minute=0)
 
     storage_key = 'buckets_split_data:' + cache_key + ':{}|{}'
     old_end_time = end_time.replace(hour=0, minute=0)
 
     final_storage_key = storage_key.format(delta.total_seconds(),
                                            old_end_time)
     older_data = None
 
     cdata = request.registry.cache_regions.redis_day_7.get(
         final_storage_key)
 
     if cdata:
         log.info("cached_split_data:{}: found old "
                  "bucket data".format(cache_key))
         older_data = cdata
 
     if (stats_since < end_time - h.time_deltas.get('24h')['delta'] and
             not cdata):
         log.info("cached_split_data:{}: didn't find the "
                  "start bucket in cache so load older data".format(cache_key))
         recent_stats_since = old_end_time
         older_data = fn(request, stats_since, recent_stats_since,
                         db_session=db_session, *args, **kwargs)
         request.registry.cache_regions.redis_day_7.set(final_storage_key,
                                                        older_data)
     elif stats_since < end_time - h.time_deltas.get('24h')['delta']:
         recent_stats_since = old_end_time
     else:
         recent_stats_since = stats_since
 
     log.info("cached_split_data:{}: loading fresh "
              "data bucksts from last 24h ".format(cache_key))
     todays_data = fn(request, recent_stats_since, end_time,
                      db_session=db_session, *args, **kwargs)
     return older_data, todays_data
 
 
 def in_batches(seq, size):
     """
     Splits am iterable into batches of specified size
     :param seq (iterable)
     :param size integer
     """
     return (seq[pos:pos + size] for pos in range(0, len(seq), size))

backend/src/appenlight/models/alert_channel_action.py

@@ -1,79 +1,79 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 
-from appenlight.models.resource import Resource
+from ziggurat_foundations.models.services.resource import ResourceService
 from appenlight.models import Base, get_db_session
 from sqlalchemy.orm import validates
 from ziggurat_foundations.models.base import BaseModel
 
 
 class AlertChannelAction(Base, BaseModel):
     """
     Stores notifications conditions for user's alert channels
     This is later used for rule parsing like "alert if http_status == 500"
     """
     __tablename__ = 'alert_channels_actions'
 
     types = ['report', 'chart']
 
     owner_id = sa.Column(sa.Integer,
                          sa.ForeignKey('users.id', onupdate='CASCADE',
                                        ondelete='CASCADE'))
     resource_id = sa.Column(sa.Integer())
     action = sa.Column(sa.Unicode(10), nullable=False, default='always')
     type = sa.Column(sa.Unicode(10), nullable=False)
     other_id = sa.Column(sa.Unicode(40))
     pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
     rule = sa.Column(sa.dialects.postgresql.JSON,
                      nullable=False, default={'field': 'http_status',
                                               "op": "ge", "value": "500"})
     config = sa.Column(sa.dialects.postgresql.JSON)
     name = sa.Column(sa.Unicode(255))
 
     @validates('notify_type')
     def validate_email(self, key, notify_type):
         assert notify_type in ['always', 'only_first']
         return notify_type
 
     def resource_name(self, db_session=None):
         db_session = get_db_session(db_session)
         if self.resource_id:
-            return Resource.by_resource_id(self.resource_id,
-                                           db_session=db_session).resource_name
+            return ResourceService.by_resource_id(
+                self.resource_id, db_session=db_session).resource_name
         else:
             return 'any resource'
 
     def get_dict(self, exclude_keys=None, include_keys=None,
                  extended_info=False):
         """
         Returns dictionary with required information that will be consumed by
         angular
         """
         instance_dict = super(AlertChannelAction, self).get_dict()
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         if extended_info:
             instance_dict['channels'] = [
                 c.get_dict(extended_info=False) for c in self.channels]
 
         d = {}
         for k in instance_dict.keys():
             if (k not in exclude_keys_list and
                     (k in include_keys_list or not include_keys)):
                 d[k] = instance_dict[k]
         return d

backend/src/appenlight/models/event.py

@@ -1,165 +1,165 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 import logging
 
 from datetime import datetime
 from appenlight.models import Base, get_db_session
 from appenlight.models.services.report_stat import ReportStatService
-from appenlight.models.resource import Resource
 from appenlight.models.integrations import IntegrationException
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.dialects.postgresql import JSON
 from ziggurat_foundations.models.base import BaseModel
+from ziggurat_foundations.models.services.resource import ResourceService
 
 log = logging.getLogger(__name__)
 
 
 class Event(Base, BaseModel):
     __tablename__ = 'events'
 
     types = {'error_report_alert': 1,
              'slow_report_alert': 3,
              'comment': 5,
              'assignment': 6,
              'uptime_alert': 7,
              'chart_alert': 9}
 
     statuses = {'active': 1,
                 'closed': 0}
 
     id = sa.Column(sa.Integer, primary_key=True)
     start_date = sa.Column(sa.DateTime, default=datetime.utcnow)
     end_date = sa.Column(sa.DateTime)
     status = sa.Column(sa.Integer, default=1)
     event_type = sa.Column(sa.Integer, default=1)
     origin_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
                                nullable=True)
     target_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
                                nullable=True)
     resource_id = sa.Column(sa.Integer(),
                             sa.ForeignKey('resources.resource_id'),
                             nullable=True)
     target_id = sa.Column(sa.Integer)
     target_uuid = sa.Column(sa.Unicode(40))
     text = sa.Column(sa.UnicodeText())
     values = sa.Column(JSON(), nullable=False, default=None)
 
     def __repr__(self):
         return '<Event %s, app:%s, %s>' % (self.unified_alert_name(),
                                             self.resource_id,
                                             self.unified_alert_action())
 
     @property
     def reverse_types(self):
         return dict([(v, k) for k, v in self.types.items()])
 
     def unified_alert_name(self):
         return self.reverse_types[self.event_type]
 
     def unified_alert_action(self):
         event_name = self.reverse_types[self.event_type]
         if self.status == Event.statuses['closed']:
             return "CLOSE"
         if self.status != Event.statuses['closed']:
             return "OPEN"
         return event_name
 
     def send_alerts(self, request=None, resource=None, db_session=None):
         """" Sends alerts to applicable channels """
         db_session = get_db_session(db_session)
         db_session.flush()
         if not resource:
-            resource = Resource.by_resource_id(self.resource_id)
+            resource = ResourceService.by_resource_id(self.resource_id)
         if not request:
             request = get_current_request()
         if not resource:
             return
-        users = set([p.user for p in resource.users_for_perm('view')])
+        users = set([p.user for p in ResourceService.users_for_perm(resource, 'view')])
         for user in users:
             for channel in user.alert_channels:
                 matches_resource = not channel.resources or resource in [r.resource_id for r in channel.resources]
                 if (
                     not channel.channel_validated or
                     not channel.send_alerts or
                     not matches_resource
                 ):
                     continue
                 else:
                     try:
                         channel.notify_alert(resource=resource,
                                              event=self,
                                              user=user,
                                              request=request)
                     except IntegrationException as e:
                         log.warning('%s' % e)
 
     def validate_or_close(self, since_when, db_session=None):
         """ Checks if alerts should stay open or it's time to close them.
             Generates close alert event if alerts get closed """
         event_types = [Event.types['error_report_alert'],
                        Event.types['slow_report_alert']]
-        app = Resource.by_resource_id(self.resource_id)
+        app = ResourceService.by_resource_id(self.resource_id)
         # if app was deleted close instantly
         if not app:
             self.close()
             return
 
         if self.event_type in event_types:
             total = ReportStatService.count_by_type(
                 self.event_type, self.resource_id, since_when)
             if Event.types['error_report_alert'] == self.event_type:
                 threshold = app.error_report_threshold
             if Event.types['slow_report_alert'] == self.event_type:
                 threshold = app.slow_report_threshold
 
             if total < threshold:
                 self.close()
 
     def close(self, db_session=None):
         """
         Closes an event and sends notification to affected users
         """
         self.end_date = datetime.utcnow()
         self.status = Event.statuses['closed']
         log.warning('ALERT: CLOSE: %s' % self)
         self.send_alerts()
 
     def text_representation(self):
         alert_type = self.unified_alert_name()
         text = ''
         if 'slow_report' in alert_type:
             text += 'Slow report alert'
         if 'error_report' in alert_type:
             text += 'Exception report alert'
         if 'uptime_alert' in alert_type:
             text += 'Uptime alert'
         if 'chart_alert' in alert_type:
             text += 'Metrics value alert'
 
         alert_action = self.unified_alert_action()
         if alert_action == 'OPEN':
             text += ' got opened.'
         if alert_action == 'CLOSE':
             text += ' got closed.'
         return text
 
     def get_dict(self, request=None):
         dict_data = super(Event, self).get_dict()
         dict_data['text'] = self.text_representation()
         dict_data['resource_name'] = self.resource.resource_name
         return dict_data

backend/src/appenlight/models/resource.py

@@ -1,83 +1,84 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from appenlight.models import Base
 from appenlight.lib.utils import permission_tuple_to_dict
 from pyramid.security import Allow, ALL_PERMISSIONS
 from ziggurat_foundations.models.resource import ResourceMixin
+from ziggurat_foundations.models.services.resource import ResourceService
 
 
 class Resource(ResourceMixin, Base):
     events = sa.orm.relationship('Event',
                                  lazy='dynamic',
                                  backref='resource',
                                  passive_deletes=True,
                                  passive_updates=True)
 
     @property
     def owner_user_name(self):
         if self.owner:
             return self.owner.user_name
 
     @property
     def owner_group_name(self):
         if self.owner_group:
             return self.owner_group.group_name
 
     def get_dict(self, exclude_keys=None, include_keys=None,
                  include_perms=False, include_processing_rules=False):
         result = super(Resource, self).get_dict(exclude_keys, include_keys)
         result['possible_permissions'] = self.__possible_permissions__
         if include_perms:
             result['current_permissions'] = self.user_permissions_list
         else:
             result['current_permissions'] = []
         if include_processing_rules:
             result["postprocessing_rules"] = [rule.get_dict() for rule
                                               in self.postprocess_conf]
         else:
             result["postprocessing_rules"] = []
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         d = {}
         for k in result.keys():
             if (k not in exclude_keys_list and
                     (k in include_keys_list or not include_keys)):
                 d[k] = result[k]
         for k in ['owner_user_name', 'owner_group_name']:
             if (k not in exclude_keys_list and
                     (k in include_keys_list or not include_keys)):
                 d[k] = getattr(self, k)
         return d
 
     @property
     def user_permissions_list(self):
         return [permission_tuple_to_dict(perm) for perm in
-                self.users_for_perm('__any_permission__',
-                                    limit_group_permissions=True)]
+                ResourceService.users_for_perm(
+                    self, '__any_permission__', limit_group_permissions=True)]
 
     @property
     def __acl__(self):
         acls = []
 
         if self.owner_user_id:
             acls.extend([(Allow, self.owner_user_id, ALL_PERMISSIONS,), ])
 
         if self.owner_group_id:
             acls.extend([(Allow, "group:%s" % self.owner_group_id,
                           ALL_PERMISSIONS,), ])
         return acls

backend/src/appenlight/models/services/event.py

@@ -1,109 +1,109 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import sqlalchemy as sa
 from pyramid.threadlocal import get_current_registry
 from paginate_sqlalchemy import SqlalchemyOrmPage
+from ziggurat_foundations.models.services.user import UserService
+
 from appenlight.models import get_db_session
 from appenlight.models.event import Event
 from appenlight.models.services.base import BaseService
 
 
 class EventService(BaseService):
     @classmethod
     def for_resource(cls, resource_ids, event_type=None, status=None,
                      since_when=None, limit=20, event_id=None,
                      target_uuid=None, order_by=None, or_target_user_id=None,
                      db_session=None):
         """
         Fetches events including based on passed params OR if target_user_id
         is present include events that just target this user
         """
         db_session = get_db_session(db_session)
         query = db_session.query(Event)
         query = query.options(sa.orm.joinedload(Event.resource))
         and_cond = [Event.resource_id.in_(resource_ids)]
         if not resource_ids:
             and_cond = [Event.resource_id == -999]
 
         if event_type:
             and_cond.append(Event.event_type == event_type)
         if status:
             and_cond.append(Event.status == status)
         if since_when:
             and_cond.append(Event.start_date >= since_when)
         if event_id:
             and_cond.append(Event.id == event_id)
         if target_uuid:
             and_cond.append(Event.target_uuid == target_uuid)
 
         or_cond = []
 
         if or_target_user_id:
             or_cond.append(sa.or_(Event.target_user_id == or_target_user_id))
 
         query = query.filter(sa.or_(sa.and_(*and_cond),
                                     *or_cond))
         if not order_by:
             query = query.order_by(sa.desc(Event.start_date))
         if limit:
             query = query.limit(limit)
 
         return query
 
     @classmethod
     def by_type_and_status(cls, event_types, status_types, since_when=None,
                            older_than=None, db_session=None, app_ids=None):
         db_session = get_db_session(db_session)
         query = db_session.query(Event)
         query = query.filter(Event.event_type.in_(event_types))
         query = query.filter(Event.status.in_(status_types))
         if since_when:
             query = query.filter(Event.start_date >= since_when)
         if older_than:
             query = query.filter(Event.start_date <= older_than)
         if app_ids:
             query = query.filter(Event.resource_id.in_(app_ids))
         return query
 
     @classmethod
     def latest_for_user(cls, user, db_session=None):
         registry = get_current_registry()
-        resources = user.resources_with_perms(
-            ['view'], resource_types=registry.resource_types)
+        resources = UserService.resources_with_perms(user, ['view'], resource_types=registry.resource_types)
         resource_ids = [r.resource_id for r in resources]
         db_session = get_db_session(db_session)
         return EventService.for_resource(
             resource_ids, or_target_user_id=user.id, limit=10,
             db_session=db_session)
 
     @classmethod
     def get_paginator(cls, user, page=1, item_count=None, items_per_page=50,
                       order_by=None, filter_settings=None, db_session=None):
         if not filter_settings:
             filter_settings = {}
         registry = get_current_registry()
-        resources = user.resources_with_perms(
-            ['view'], resource_types=registry.resource_types)
+        resources = UserService.resources_with_perms(user, ['view'], resource_types=registry.resource_types)
         resource_ids = [r.resource_id for r in resources]
         query = EventService.for_resource(
             resource_ids, or_target_user_id=user.id, limit=100,
             db_session=db_session)
 
         paginator = SqlalchemyOrmPage(query, page=page,
                                       items_per_page=items_per_page,
                                       **filter_settings)
         return paginator

backend/src/appenlight/models/services/group.py

@@ -1,27 +1,27 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models import get_db_session
 from appenlight.models.group import Group
-from appenlight.models.services.base import BaseService
+from ziggurat_foundations.models.services.group import GroupService
 
 
-class GroupService(BaseService):
+class GroupService(GroupService):
     @classmethod
     def by_id(cls, group_id, db_session=None):
         db_session = get_db_session(db_session)
         query = db_session.query(Group).filter(Group.id == group_id)
         return query.first()

backend/src/appenlight/models/services/group_resource_permission.py

@@ -1,33 +1,33 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.models.group_resource_permission import GroupResourcePermission
 from appenlight.models import get_db_session
-from appenlight.models.services.base import BaseService
+from ziggurat_foundations.models.services.group_resource_permission import GroupResourcePermissionService
 
 
-class GroupResourcePermissionService(BaseService):
+class GroupResourcePermissionService(GroupResourcePermissionService):
     @classmethod
     def by_resource_group_and_perm(cls, group_id, perm_name, resource_id,
                                    db_session=None):
         """ return all instances by user name, perm name and resource id """
         db_session = get_db_session(db_session)
         query = db_session.query(GroupResourcePermission)
         query = query.filter(GroupResourcePermission.group_id == group_id)
         query = query.filter(
             GroupResourcePermission.resource_id == resource_id)
         query = query.filter(GroupResourcePermission.perm_name == perm_name)
         return query.first()

backend/src/appenlight/models/services/user.py

@@ -1,151 +1,152 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import pyramid_mailer
 import pyramid.renderers
 import sqlalchemy as sa
 
 from collections import namedtuple
 from datetime import datetime
 
+from ziggurat_foundations.models.services.user import UserService
+
 from appenlight.lib.rule import Rule
 from appenlight.models import get_db_session
 from appenlight.models.integrations import IntegrationException
 from appenlight.models.report import REPORT_TYPE_MATRIX
 from appenlight.models.user import User
-from appenlight.models.services.base import BaseService
 from paginate_sqlalchemy import SqlalchemyOrmPage
 from pyramid.threadlocal import get_current_registry
 
 log = logging.getLogger(__name__)
 
 GroupOccurence = namedtuple('GroupOccurence', ['occurences', 'group'])
 
 
-class UserService(BaseService):
+class UserService(UserService):
     @classmethod
     def all(cls, db_session=None):
         return get_db_session(db_session).query(User).order_by(User.user_name)
 
     @classmethod
     def send_email(cls, request, recipients, variables, template,
                    immediately=False, silent=False):
         html = pyramid.renderers.render(template, variables, request)
         title = variables.get('email_title',
                               variables.get('title', "No Title"))
         title = title.replace('\r', '').replace('\n', '')
         sender = "{} <{}>".format(
             request.registry.settings['mailing.from_name'],
             request.registry.settings['mailing.from_email'])
         message = pyramid_mailer.message.Message(
             subject=title, sender=sender, recipients=recipients, html=html)
         if immediately:
             try:
                 request.registry.mailer.send_immediately(message)
             except Exception as e:
                 log.warning('Exception %s' % e)
                 if not silent:
                     raise
         else:
             request.registry.mailer.send(message)
 
     @classmethod
     def get_paginator(cls, page=1, item_count=None, items_per_page=50,
                       order_by=None, filter_settings=None,
                       exclude_columns=None, db_session=None):
         registry = get_current_registry()
         if not exclude_columns:
             exclude_columns = []
         if not filter_settings:
             filter_settings = {}
         db_session = get_db_session(db_session)
         q = db_session.query(User)
         if filter_settings.get('order_col'):
             order_col = filter_settings.get('order_col')
             if filter_settings.get('order_dir') == 'dsc':
                 sort_on = 'desc'
             else:
                 sort_on = 'asc'
             q = q.order_by(getattr(sa, sort_on)(getattr(User, order_col)))
         else:
             q = q.order_by(sa.desc(User.registered_date))
             # remove urlgen or it never caches count
         cache_params = dict(filter_settings)
         cache_params.pop('url', None)
         cache_params.pop('url_maker', None)
 
         @registry.cache_regions.redis_min_5.cache_on_arguments()
         def estimate_users(cache_key):
             o_q = q.order_by(False)
             return o_q.count()
 
         item_count = estimate_users(cache_params)
         # if the number of pages is low we may want to invalidate the count to
         # provide 'real time' update - use case -
         # errors just started to flow in
         if item_count < 1000:
             item_count = estimate_users.refresh(cache_params)
         paginator = SqlalchemyOrmPage(q, page=page,
                                       item_count=item_count,
                                       items_per_page=items_per_page,
                                       **filter_settings)
         return paginator
 
     @classmethod
     def get_valid_channels(cls, user):
         return [channel for channel in user.alert_channels
                 if channel.channel_validated]
 
     @classmethod
     def report_notify(cls, user, request, application, report_groups,
                       occurence_dict, db_session=None):
         db_session = get_db_session(db_session)
         if not report_groups:
             return True
         since_when = datetime.utcnow()
         for channel in cls.get_valid_channels(user):
             confirmed_groups = []
 
             for group in report_groups:
                 occurences = occurence_dict.get(group.id, 1)
                 for action in channel.channel_actions:
                     not_matched = (
                         action.resource_id and action.resource_id !=
                         application.resource_id)
                     if action.type != 'report' or not_matched:
                         continue
                     should_notify = (action.action == 'always' or
                                      not group.notified)
                     rule_obj = Rule(action.rule, REPORT_TYPE_MATRIX)
                     report_dict = group.get_report().get_dict(request)
                     if rule_obj.match(report_dict) and should_notify:
                         item = GroupOccurence(occurences, group)
                         if item not in confirmed_groups:
                             confirmed_groups.append(item)
 
             # send individual reports
             total_confirmed = len(confirmed_groups)
             if not total_confirmed:
                 continue
             try:
                 channel.notify_reports(resource=application,
                                        user=user,
                                        request=request,
                                        since_when=since_when,
                                        reports=confirmed_groups)
             except IntegrationException as e:
                 log.warning('%s' % e)

backend/src/appenlight/models/user.py

@@ -1,133 +1,133 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 import sqlalchemy as sa
 from datetime import datetime
 from appenlight.models import Base, get_db_session
 from appenlight.models.services.event import EventService
 from appenlight.models.integrations import IntegrationException
 from pyramid.threadlocal import get_current_request
 from ziggurat_foundations.models.user import UserMixin
+from ziggurat_foundations.models.services.user import UserService
 
 log = logging.getLogger(__name__)
 
 
 class User(UserMixin, Base):
     __possible_permissions__ = []
 
     first_name = sa.Column(sa.Unicode(25))
     last_name = sa.Column(sa.Unicode(25))
     company_name = sa.Column(sa.Unicode(255), default='')
     company_address = sa.Column(sa.Unicode(255), default='')
     zip_code = sa.Column(sa.Unicode(25), default='')
     city = sa.Column(sa.Unicode(50), default='')
     default_report_sort = sa.Column(sa.Unicode(25), default='newest')
     notes = sa.Column(sa.UnicodeText, default='')
     notifications = sa.Column(sa.Boolean(), default=True)
     registration_ip = sa.Column(sa.UnicodeText(), default='')
     alert_channels = sa.orm.relationship('AlertChannel',
                                          cascade="all,delete-orphan",
                                          passive_deletes=True,
                                          passive_updates=True,
                                          backref='owner',
                                          order_by='AlertChannel.channel_name, '
                                                   'AlertChannel.channel_value')
 
     alert_actions = sa.orm.relationship('AlertChannelAction',
                                         cascade="all,delete-orphan",
                                         passive_deletes=True,
                                         passive_updates=True,
                                         backref='owner',
                                         order_by='AlertChannelAction.pkey')
 
     auth_tokens = sa.orm.relationship('AuthToken',
                                       cascade="all,delete-orphan",
                                       passive_deletes=True,
                                       passive_updates=True,
                                       backref='owner',
                                       order_by='AuthToken.creation_date')
 
     def get_dict(self, exclude_keys=None, include_keys=None,
                  extended_info=False):
         result = super(User, self).get_dict(exclude_keys, include_keys)
         if extended_info:
             result['groups'] = [g.group_name for g in self.groups]
-            result['permissions'] = [p.perm_name for p in self.permissions]
+            result['permissions'] = [p.perm_name for p in UserService.permissions(self)]
             request = get_current_request()
-            apps = self.resources_with_perms(
+            apps = UserService.resources_with_perms(self,
                 ['view'], resource_types=['application'])
             result['applications'] = sorted(
                 [{'resource_id': a.resource_id,
                   'resource_name': a.resource_name}
                  for a in apps.all()],
                 key=lambda x: x['resource_name'].lower())
             result['assigned_reports'] = [r.get_dict(request) for r
                                           in self.assigned_report_groups]
             result['latest_events'] = [ev.get_dict(request) for ev
                                        in self.latest_events()]
 
         exclude_keys_list = exclude_keys or []
         include_keys_list = include_keys or []
         d = {}
         for k in result.keys():
             if (k not in exclude_keys_list and
                     (k in include_keys_list or not include_keys)):
                 d[k] = result[k]
         return d
 
     def __repr__(self):
         return '<User: %s, id: %s>' % (self.user_name, self.id)
 
     @property
     def assigned_report_groups(self):
         from appenlight.models.report_group import ReportGroup
 
-        resources = self.resources_with_perms(
-            ['view'], resource_types=['application'])
+        resources = UserService.resources_with_perms(self, ['view'], resource_types=['application'])
         query = self.assigned_reports_relation
         rid_list = [r.resource_id for r in resources]
         query = query.filter(ReportGroup.resource_id.in_(rid_list))
         query = query.limit(50)
         return query
 
     def feed_report(self, report):
         """ """
         if not hasattr(self, 'current_reports'):
             self.current_reports = []
         self.current_reports.append(report)
 
     def send_digest(self, request, application, reports, since_when=None,
                     db_session=None):
         db_session = get_db_session(db_session)
         if not reports:
             return True
         if not since_when:
             since_when = datetime.utcnow()
         for channel in self.alert_channels:
             if not channel.channel_validated or not channel.daily_digest:
                 continue
             try:
                 channel.send_digest(resource=application,
                                     user=self,
                                     request=request,
                                     since_when=since_when,
                                     reports=reports)
             except IntegrationException as e:
                 log.warning('%s' % e)
 
     def latest_events(self):
         return EventService.latest_for_user(self)

backend/src/appenlight/scripts/initialize_db.py

@@ -1,153 +1,155 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import argparse
 import getpass
 import logging
 
 from pyramid.paster import setup_logging, bootstrap
 from pyramid.threadlocal import get_current_request
+from ziggurat_foundations.models.services.user import UserService
+
 
 from appenlight.forms import UserRegisterForm
 from appenlight.lib.ext_json import json
 from appenlight.models import (
     DBSession,
     Group,
     GroupPermission,
     User,
     AuthToken
 )
 from appenlight.models.services.group import GroupService
 
 log = logging.getLogger(__name__)
 
 _ = str
 
 
 def is_yes(input_data):
     return input_data in ['y', 'yes']
 
 
 def is_no(input_data):
     return input_data in ['n', 'no']
 
 
 def main():
     parser = argparse.ArgumentParser(
         description='Populate AppEnlight database',
         add_help=False)
     parser.add_argument('-c', '--config', required=True,
                         help='Configuration ini file of application')
     parser.add_argument('--username', default=None,
                         help='User  to create')
     parser.add_argument('--password', default=None,
                         help='Password for created user')
     parser.add_argument('--email', default=None,
                         help='Email for created user')
     parser.add_argument('--auth-token', default=None,
                         help='Auth token for created user')
     args = parser.parse_args()
     config_uri = args.config
 
     setup_logging(config_uri)
     env = bootstrap(config_uri)
     request = env['request']
     with get_current_request().tm:
         group = GroupService.by_id(1)
         if not group:
             group = Group(id=1, group_name='Administrators',
                            description="Top level permission owners")
             DBSession.add(group)
             permission = GroupPermission(perm_name='root_administration')
             group.permissions.append(permission)
 
     create_user = True if args.username else None
     while create_user is None:
         response = input(
             'Do you want to create a new admin? (n)\n').lower()
 
         if is_yes(response or 'n'):
             create_user = True
         elif is_no(response or 'n'):
             create_user = False
 
     if create_user:
         csrf_token = request.session.get_csrf_token()
         user_name = args.username
         print('*********************************************************')
         while user_name is None:
             response = input('What is the username of new admin?\n')
             form = UserRegisterForm(
                 user_name=response, csrf_token=csrf_token,
                 csrf_context=request)
             form.validate()
             if form.user_name.errors:
                 print(form.user_name.errors[0])
             else:
                 user_name = response
                 print('The admin username is "{}"\n'.format(user_name))
         print('*********************************************************')
         email = args.email
         while email is None:
             response = input('What is the email of admin account?\n')
             form = UserRegisterForm(
                 email=response, csrf_token=csrf_token,
                 csrf_context=request)
             form.validate()
             if form.email.errors:
                 print(form.email.errors[0])
             else:
                 email = response
                 print('The admin email is "{}"\n'.format(email))
         print('*********************************************************')
         user_password = args.password
         confirmed_password = args.password
         while user_password is None or confirmed_password is None:
             response = getpass.getpass(
                 'What is the password for admin account?\n')
             form = UserRegisterForm(
                 user_password=response, csrf_token=csrf_token,
                 csrf_context=request)
             form.validate()
             if form.user_password.errors:
                 print(form.user_password.errors[0])
             else:
                 user_password = response
 
             response = getpass.getpass('Please confirm the password.\n')
             if user_password == response:
                 confirmed_password = response
             else:
                 print('Passwords do not match. Please try again')
         print('*********************************************************')
 
     with get_current_request().tm:
         if create_user:
             group = GroupService.by_id(1)
             user = User(user_name=user_name, email=email, status=1)
-            user.regenerate_security_code()
-            user.set_password(user_password)
+            UserService.regenerate_security_code(user)
+            UserService.set_password(user, user_password)
             DBSession.add(user)
             token = AuthToken(description="Uptime monitoring token")
             if args.auth_token:
                 token.token = args.auth_token
             user.auth_tokens.append(token)
             group.users.append(user)
             print('USER CREATED')
             print(json.dumps(user.get_dict()))
             print('*********************************************************')
             print('AUTH TOKEN')
             print(json.dumps(user.auth_tokens[0].get_dict()))

backend/src/appenlight/security.py

@@ -1,346 +1,348 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from pyramid.security import Allow, Everyone, Authenticated, ALL_PERMISSIONS
 from pyramid.authentication import CallbackAuthenticationPolicy
 import appenlight.models.resource
 from appenlight.models.services.auth_token import AuthTokenService
 from appenlight.models.services.application import ApplicationService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.plugin_config import PluginConfigService
 from appenlight.lib import to_integer_safe
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest
 from ziggurat_foundations.permissions import permission_to_04_acls
+from ziggurat_foundations.models.services.user import UserService
+from ziggurat_foundations.models.services.resource import ResourceService
 import defusedxml.ElementTree as ElementTree
 import urllib.request, urllib.error, urllib.parse
 import logging
 import re
 from xml.sax.saxutils import quoteattr
 
 log = logging.getLogger(__name__)
 
 
 def groupfinder(userid, request):
     if userid and hasattr(request, 'user') and request.user:
         groups = ['group:%s' % g.id for g in request.user.groups]
         return groups
     return []
 
 
 class AuthTokenAuthenticationPolicy(CallbackAuthenticationPolicy):
     def __init__(self, callback=None):
         self.callback = callback
 
     def remember(self, request, principal, **kw):
         return []
 
     def forget(self, request):
         return []
 
     def unauthenticated_userid(self, request):
         token = request.headers.get('x-appenlight-auth-token')
         if token:
             auth_token = AuthTokenService.by_token(token)
             if auth_token and not auth_token.is_expired:
                 log.info('%s is valid' % auth_token)
                 return auth_token.owner_id
             elif auth_token:
                 log.warning('%s is expired' % auth_token)
             else:
                 log.warning('token: %s is not found' % token)
 
     def authenticated_userid(self, request):
         return self.unauthenticated_userid(request)
 
 
 def rewrite_root_perm(perm_user, perm_name):
     """
     Translates root_administration into ALL_PERMISSIONS object
     """
     if perm_name == 'root_administration':
         return (Allow, perm_user, ALL_PERMISSIONS,)
     else:
         return (Allow, perm_user, perm_name,)
 
 
 def add_root_superperm(request, context):
     """
     Adds ALL_PERMISSIONS to every resource if user somehow has 'root_permission'
     non-resource permission
     """
     if hasattr(request, 'user') and request.user:
-        acls = permission_to_04_acls(request.user.permissions)
+        acls = permission_to_04_acls(UserService.permissions(request.user))
         for perm_user, perm_name in acls:
             if perm_name == 'root_administration':
                 context.__acl__.append(rewrite_root_perm(perm_user, perm_name))
 
 
 class RootFactory(object):
     """
     General factory for non-resource/report specific pages
     """
 
     def __init__(self, request):
         self.__acl__ = [(Allow, Authenticated, 'authenticated'),
                         (Allow, Authenticated, 'create_resources')]
         # general page factory - append custom non resource permissions
         if hasattr(request, 'user') and request.user:
-            acls = permission_to_04_acls(request.user.permissions)
+            acls = permission_to_04_acls(UserService.permissions(request.user))
             for perm_user, perm_name in acls:
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
 
 class ResourceFactory(object):
     """
     Checks permissions to specific resource based on user permissions or
     API key headers
     """
 
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
 
         self.__acl__ = []
         resource_id = request.matchdict.get("resource_id",
                                             request.GET.get("resource_id"))
         resource_id = to_integer_safe(resource_id)
-        self.resource = Resource.by_resource_id(resource_id) \
+        self.resource = ResourceService.by_resource_id(resource_id) \
             if resource_id else None
         if self.resource and request.user:
             self.__acl__ = self.resource.__acl__
-            permissions = self.resource.perms_for_user(request.user)
+            permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         add_root_superperm(request, self)
 
 
 class ResourceReportFactory(object):
     """
     Checks permissions to specific resource based on user permissions or
     API key headers
     Resource is fetched based on report group information
     """
 
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
 
         self.__acl__ = []
         group_id = request.matchdict.get("group_id",
                                          request.params.get("group_id"))
         group_id = to_integer_safe(group_id)
         self.report_group = ReportGroupService.by_id(
             group_id) if group_id else None
         if not self.report_group:
             raise HTTPNotFound()
 
         self.public = self.report_group.public
-        self.resource = Resource.by_resource_id(self.report_group.resource_id) \
+        self.resource = ResourceService.by_resource_id(self.report_group.resource_id) \
             if self.report_group else None
 
         if self.resource:
             self.__acl__ = self.resource.__acl__
         if request.user:
-            permissions = self.resource.perms_for_user(request.user)
+            permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         if self.public:
             self.__acl__.append((Allow, Everyone, 'view',))
         if not request.user:
             # unauthed users need to visit using both group and report pair
             report_id = request.params.get('reportId',
                                            request.params.get('report_id', -1))
             report = self.report_group.get_report(report_id, public=True)
             if not report:
                 raise HTTPNotFound()
         add_root_superperm(request, self)
 
 
 class APIFactory(object):
     """
     Checks permissions to perform client API actions based on keys
     """
 
     def __init__(self, request):
         self.__acl__ = []
         self.possibly_public = False
         private_api_key = request.headers.get(
             'x-appenlight-api-key',
             request.params.get('api_key')
         )
         log.debug("private key: %s" % private_api_key)
         if private_api_key:
             self.resource = ApplicationService.by_api_key_cached()(
                 private_api_key)
         # then try public key
         else:
             public_api_key = request.headers.get(
                 'x-appenlight-public-api-key',
                 request.GET.get('public_api_key'))
             log.debug("public key: %s" % public_api_key)
             self.resource = ApplicationService.by_public_api_key(
                 public_api_key, from_cache=True, request=request)
             self.possibly_public = True
         if self.resource:
             self.__acl__.append((Allow, Everyone, 'create',))
 
 
 class AirbrakeV2APIFactory(object):
     """
     Check permission based on Airbrake XML report
     """
 
     def __init__(self, request):
         self.__acl__ = []
         self.possibly_public = False
         fixed_xml_data = ''
         try:
             data = request.GET.get('data')
             if data:
                 self.possibly_public = True
         except (UnicodeDecodeError, UnicodeEncodeError) as exc:
             log.warning(
                 'Problem parsing Airbrake data: %s, failed decoding' % exc)
             raise HTTPBadRequest()
         try:
             if not data:
                 data = request.body
                 # fix shitty airbrake js client not escaping line method attr
 
             def repl(input):
                 return 'line method=%s file' % quoteattr(input.group(1))
 
             fixed_xml_data = re.sub('line method="(.*?)" file', repl, data)
             root = ElementTree.fromstring(fixed_xml_data)
         except Exception as exc:
             log.info(
                 'Problem parsing Airbrake '
                 'data: %s, trying unquoting' % exc)
             self.possibly_public = True
             try:
                 root = ElementTree.fromstring(urllib.parse.unquote(fixed_xml_data))
             except Exception as exc:
                 log.warning('Problem parsing Airbrake '
                             'data: %s, failed completly' % exc)
                 raise HTTPBadRequest()
         self.airbrake_xml_etree = root
         api_key = root.findtext('api-key', '')
 
         self.resource = ApplicationService.by_api_key_cached()(api_key)
         if not self.resource:
             self.resource = ApplicationService.by_public_api_key(api_key,
                                                                  from_cache=True,
                                                                  request=request)
             if self.resource:
                 self.possibly_public = True
 
         if self.resource:
             self.__acl__.append((Allow, Everyone, 'create',))
 
 
 def parse_sentry_header(header):
     parsed = header.split(' ', 1)[1].split(',') or []
     return dict([x.strip().split('=') for x in parsed])
 
 
 class SentryAPIFactory(object):
     """
     Check permission based on Sentry payload
     """
 
     def __init__(self, request):
         self.__acl__ = []
         self.possibly_public = False
         if request.headers.get('X-Sentry-Auth', '').startswith('Sentry'):
             header_string = request.headers['X-Sentry-Auth']
             result = parse_sentry_header(header_string)
         elif request.headers.get('Authorization', '').startswith('Sentry'):
             header_string = request.headers['Authorization']
             result = parse_sentry_header(header_string)
         else:
             result = dict((k, v) for k, v in list(request.GET.items())
                           if k.startswith('sentry_'))
         key = result.get('sentry_key')
         log.info('sentry request {}'.format(result))
 
         self.resource = ApplicationService.by_api_key_cached()(key)
         if not self.resource or \
             result.get('sentry_client', '').startswith('raven-js'):
             self.resource = ApplicationService.by_public_api_key(
                 key, from_cache=True, request=request)
         if self.resource:
             self.__acl__.append((Allow, Everyone, 'create',))
 
 
 class ResourcePluginConfigFactory(object):
 
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
         self.__acl__ = []
         self.resource = None
         plugin_id = to_integer_safe(request.matchdict.get('id'))
         self.plugin = PluginConfigService.by_id(plugin_id)
         if not self.plugin:
             raise HTTPNotFound()
         if self.plugin.resource_id:
-            self.resource = Resource.by_resource_id(self.plugin.resource_id)
+            self.resource = ResourceService.by_resource_id(self.plugin.resource_id)
         if self.resource:
             self.__acl__ = self.resource.__acl__
         if request.user and self.resource:
-            permissions = self.resource.perms_for_user(request.user)
+            permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
 
         add_root_superperm(request, self)
 
 
 class ResourceJSONBodyFactory(object):
     """
     Checks permissions to specific resource based on user permissions or
     API key headers from json body
     """
 
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
 
         self.__acl__ = []
         resource_id = request.unsafe_json_body().get('resource_id')
         resource_id = to_integer_safe(resource_id)
-        self.resource = Resource.by_resource_id(resource_id)
+        self.resource = ResourceService.by_resource_id(resource_id)
         if self.resource and request.user:
             self.__acl__ = self.resource.__acl__
-            permissions = self.resource.perms_for_user(request.user)
+            permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         add_root_superperm(request, self)
 
 
 class ResourcePluginMixedFactory(object):
     def __init__(self, request):
         Resource = appenlight.models.resource.Resource
         self.__acl__ = []
         json_body = request.safe_json_body
         self.resource = None
         if json_body:
             resource_id = json_body.get('resource_id')
         else:
             resource_id = request.GET.get('resource_id')
         if resource_id:
             resource_id = to_integer_safe(resource_id)
-            self.resource = Resource.by_resource_id(resource_id)
+            self.resource = ResourceService.by_resource_id(resource_id)
         if self.resource and request.user:
             self.__acl__ = self.resource.__acl__
-            permissions = self.resource.perms_for_user(request.user)
+            permissions = ResourceService.perms_for_user(self.resource, request.user)
             for perm_user, perm_name in permission_to_04_acls(permissions):
                 self.__acl__.append(rewrite_root_perm(perm_user, perm_name))
         add_root_superperm(request, self)

backend/src/appenlight/tests/test_unit.py

@@ -1,1592 +1,1592 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import logging
 import mock
 import pyramid
 import pytest
 import sqlalchemy as sa
 import webob
 
 from datetime import datetime
 from pyramid import testing
 
 
 from appenlight.models import DBSession
 from appenlight.lib.ext_json import json
 
 
 log = logging.getLogger(__name__)
 
 
 class DummyContext(object):
     pass
 
 
 @pytest.mark.usefixtures('base_app')
 class BasicTest(object):
     pass
 
 
 @pytest.mark.usefixtures('base_app')
 class TestMigration(object):
     def test_migration(self):
         assert 1 == 1
 
 
 class TestSentryProto_7(object):
     def test_log_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.enums import ParsedSentryEventType
         from appenlight.lib.utils.sentry import parse_sentry_event
         event_dict, event_type = parse_sentry_event(
             payload_examples.SENTRY_LOG_PAYLOAD_7)
         assert ParsedSentryEventType.LOG == event_type
         assert event_dict['log_level'] == 'CRITICAL'
         assert event_dict['message'] == 'TEST from django logging'
         assert event_dict['namespace'] == 'testlogger'
         assert event_dict['request_id'] == '9a6172f2e6d2444582f83a6c333d9cfb'
         assert event_dict['server'] == 'ergo-virtual-machine'
         assert event_dict['date'] == datetime.utcnow().date().strftime(
             '%Y-%m-%dT%H:%M:%SZ')
         tags = [('site', 'example.com'),
                 ('sys.argv', ["'manage.py'", "'runserver'"]),
                 ('price', 6),
                 ('tag', "'extra'"),
                 ('dupa', True),
                 ('project', 'sentry'),
                 ('sentry_culprit', 'testlogger in index'),
                 ('sentry_language', 'python'),
                 ('sentry_release', 'test')]
         assert sorted(event_dict['tags']) == sorted(tags)
 
     def test_report_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.enums import ParsedSentryEventType
         from appenlight.lib.utils.sentry import parse_sentry_event
         utcnow = datetime.utcnow().date().strftime('%Y-%m-%dT%H:%M:%SZ')
         event_dict, event_type = parse_sentry_event(
             payload_examples.SENTRY_PYTHON_PAYLOAD_7)
         assert ParsedSentryEventType.ERROR_REPORT == event_type
         assert event_dict['client'] == 'sentry'
         assert event_dict[
                    'error'] == 'Exception: test 500 ' \
                                '\u0142\xf3\u201c\u0107\u201c\u0107\u017c\u0105'
         assert event_dict['language'] == 'python'
         assert event_dict['ip'] == '127.0.0.1'
         assert event_dict['request_id'] == '9fae652c8c1c4d6a8eee09260f613a98'
         assert event_dict['server'] == 'ergo-virtual-machine'
         assert event_dict['start_time'] == utcnow
         assert event_dict['url'] == 'http://127.0.0.1:8000/error'
         assert event_dict['user_agent'] == 'Mozilla/5.0 (X11; Linux x86_64) ' \
                                            'AppleWebKit/537.36 (KHTML, ' \
                                            'like Gecko) Chrome/47.0.2526.106 ' \
                                            'Safari/537.36'
         assert event_dict['view_name'] == 'djangoapp.views in error'
         tags = [('site', 'example.com'), ('sentry_release', 'test')]
         assert sorted(event_dict['tags']) == sorted(tags)
         extra = [('sys.argv', ["'manage.py'", "'runserver'"]),
                  ('project', 'sentry')]
         assert sorted(event_dict['extra']) == sorted(extra)
         request = event_dict['request']
         assert request['url'] == 'http://127.0.0.1:8000/error'
         assert request['cookies'] == {'appenlight': 'X'}
         assert request['data'] is None
         assert request['method'] == 'GET'
         assert request['query_string'] == ''
         assert request['env'] == {'REMOTE_ADDR': '127.0.0.1',
                                   'SERVER_NAME': 'localhost',
                                   'SERVER_PORT': '8000'}
         assert request['headers'] == {
             'Accept': 'text/html,application/xhtml+xml,'
                        'application/xml;q=0.9,image/webp,*/*;q=0.8',
             'Accept-Encoding': 'gzip, deflate, sdch',
             'Accept-Language': 'en-US,en;q=0.8,pl;q=0.6',
             'Connection': 'keep-alive',
             'Content-Length': '',
             'Content-Type': 'text/plain',
             'Cookie': 'appenlight=X',
             'Dnt': '1',
             'Host': '127.0.0.1:8000',
             'Upgrade-Insecure-Requests': '1',
             'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) '
                            'AppleWebKit/537.36 (KHTML, like Gecko) '
                            'Chrome/47.0.2526.106 Safari/537.36'}
         traceback = event_dict['traceback']
         assert traceback[0]['cline'] == 'response = wrapped_callback(request, ' \
                                         '*callback_args, **callback_kwargs)'
         assert traceback[0]['file'] == 'django/core/handlers/base.py'
         assert traceback[0]['fn'] == 'get_response'
         assert traceback[0]['line'] == 111
         assert traceback[0]['module'] == 'django.core.handlers.base'
 
         assert traceback[1]['cline'] == "raise Exception(u'test 500 " \
                                         "\u0142\xf3\u201c\u0107\u201c\u0107" \
                                         "\u017c\u0105')"
         assert traceback[1]['file'] == 'djangoapp/views.py'
         assert traceback[1]['fn'] == 'error'
         assert traceback[1]['line'] == 84
         assert traceback[1]['module'] == 'djangoapp.views'
         assert sorted(traceback[1]['vars']) == sorted([
             ('c',
              '<sqlite3.Cursor object at 0x7fe7c82af8f0>'),
             ('request',
              '<WSGIRequest at 0x140633490316304>'),
             ('conn',
              '<sqlite3.Connection object at 0x7fe7c8b23bf8>')])
 
 
 class TestAPIReports_0_5_Validation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json):
         import colander
         from appenlight.validators import ReportListSchema_0_5
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self):
         dummy_json = [{}]
         import colander
         from appenlight.validators import ReportListSchema_0_5
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         with pytest.raises(colander.Invalid):
             schema.deserialize(dummy_json)
 
     def test_minimal_payload(self):
         dummy_json = [{'report_details': [{}]}]
         from appenlight.validators import ReportListSchema_0_5
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
 
         deserialized = schema.deserialize(dummy_json)
 
         expected_deserialization = [
             {'language': 'unknown',
              'server': 'unknown',
              'occurences': 1,
              'priority': 5,
              'view_name': '',
              'client': 'unknown',
              'http_status': 200,
              'error': '',
              'tags': None,
              'username': '',
              'traceback': None,
              'extra': None,
              'url': '',
              'ip': None,
              'start_time': utcnow,
              'group_string': None,
              'request': {},
              'request_stats': None,
              'end_time': None,
              'request_id': '',
              'message': '',
              'slow_calls': [],
              'user_agent': ''
              }
         ]
         assert deserialized == expected_deserialization
 
     def test_full_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.validators import ReportListSchema_0_5
         PYTHON_PAYLOAD = copy.deepcopy(payload_examples.PYTHON_PAYLOAD_0_5)
         utcnow = datetime.utcnow()
         schema = ReportListSchema_0_5().bind(utcnow=utcnow)
         PYTHON_PAYLOAD["tags"] = [("foo", 1), ("action", "test"), ("baz", 1.1),
                                   ("date",
                                    utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
         dummy_json = [PYTHON_PAYLOAD]
         deserialized = schema.deserialize(dummy_json)[0]
         assert deserialized['error'] == PYTHON_PAYLOAD['error']
         assert deserialized['language'] == PYTHON_PAYLOAD['language']
         assert deserialized['server'] == PYTHON_PAYLOAD['server']
         assert deserialized['priority'] == PYTHON_PAYLOAD['priority']
         assert deserialized['view_name'] == PYTHON_PAYLOAD['view_name']
         assert deserialized['client'] == PYTHON_PAYLOAD['client']
         assert deserialized['http_status'] == PYTHON_PAYLOAD['http_status']
         assert deserialized['error'] == PYTHON_PAYLOAD['error']
         assert deserialized['occurences'] == PYTHON_PAYLOAD['occurences']
         assert deserialized['username'] == PYTHON_PAYLOAD['username']
         assert deserialized['traceback'] == PYTHON_PAYLOAD['traceback']
         assert deserialized['url'] == PYTHON_PAYLOAD['url']
         assert deserialized['ip'] == PYTHON_PAYLOAD['ip']
         assert deserialized['start_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                PYTHON_PAYLOAD['start_time']
         assert deserialized['ip'] == PYTHON_PAYLOAD['ip']
         assert deserialized['group_string'] is None
         assert deserialized['request_stats'] == PYTHON_PAYLOAD['request_stats']
         assert deserialized['end_time'].strftime('%Y-%m-%dT%H:%M:%S.0') == \
                PYTHON_PAYLOAD['end_time']
         assert deserialized['request_id'] == PYTHON_PAYLOAD['request_id']
         assert deserialized['message'] == PYTHON_PAYLOAD['message']
         assert deserialized['user_agent'] == PYTHON_PAYLOAD['user_agent']
         assert deserialized['slow_calls'][0]['start'].strftime(
             '%Y-%m-%dT%H:%M:%S.0') == PYTHON_PAYLOAD['slow_calls'][0][
                    'start']
         assert deserialized['slow_calls'][0]['end'].strftime(
             '%Y-%m-%dT%H:%M:%S.0') == PYTHON_PAYLOAD['slow_calls'][0][
                    'end']
         assert deserialized['slow_calls'][0]['statement'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['statement']
         assert deserialized['slow_calls'][0]['parameters'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['parameters']
         assert deserialized['slow_calls'][0]['type'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['type']
         assert deserialized['slow_calls'][0]['subtype'] == \
                PYTHON_PAYLOAD['slow_calls'][0]['subtype']
         assert deserialized['slow_calls'][0]['location'] == ''
         assert deserialized['tags'] == [
             ('foo', 1), ('action', 'test'),
             ('baz', 1.1), ('date', utcnow.strftime('%Y-%m-%dT%H:%M:%S.0'))]
 
 
 @pytest.mark.usefixtures('log_schema')
 class TestAPILogsValidation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json, log_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             log_schema.deserialize(dummy_json)
 
     def test_minimal_payload(self, log_schema):
         dummy_json = [{}]
         deserialized = log_schema.deserialize(dummy_json)[0]
         expected = {'log_level': 'UNKNOWN',
                     'namespace': '',
                     'server': 'unknown',
                     'request_id': '',
                     'primary_key': None,
                     'date': datetime.utcnow(),
                     'message': '',
                     'tags': None}
         assert deserialized['log_level'] == expected['log_level']
         assert deserialized['message'] == expected['message']
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['request_id'] == expected['request_id']
         assert deserialized['server'] == expected['server']
         assert deserialized['tags'] == expected['tags']
         assert deserialized['primary_key'] == expected['primary_key']
 
     def test_normal_payload(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         deserialized = log_schema.deserialize(payload_examples.LOG_EXAMPLES)[0]
         expected = payload_examples.LOG_EXAMPLES[0]
         assert deserialized['log_level'] == expected['log_level']
         assert deserialized['message'] == expected['message']
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['request_id'] == expected['request_id']
         assert deserialized['server'] == expected['server']
         assert deserialized['date'].strftime('%Y-%m-%dT%H:%M:%S.%f') == \
                expected['date']
         assert deserialized['tags'][0][0] == "tag_name"
         assert deserialized['tags'][0][1] == "tag_value"
         assert deserialized['tags'][1][0] == "tag_name2"
         assert deserialized['tags'][1][1] == 2
 
     def test_normal_payload_date_without_microseconds(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0]['date'] = datetime.utcnow().strftime(
             '%Y-%m-%dT%H:%M:%S')
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M:%S') == \
                LOG_EXAMPLE[0]['date']
 
     def test_normal_payload_date_without_seconds(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0]['date'] = datetime.utcnow().date().strftime(
             '%Y-%m-%dT%H:%M')
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') == \
                LOG_EXAMPLE[0]['date']
 
     def test_payload_empty_date(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0]['date'] = None
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') is not None
 
     def test_payload_no_date(self, log_schema):
         import appenlight.tests.payload_examples as payload_examples
         LOG_EXAMPLE = copy.deepcopy(payload_examples.LOG_EXAMPLES)
         LOG_EXAMPLE[0].pop('date', None)
         deserialized = log_schema.deserialize(LOG_EXAMPLE)
         assert deserialized[0]['date'].strftime('%Y-%m-%dT%H:%M') is not None
 
 
 @pytest.mark.usefixtures('general_metrics_schema')
 class TestAPIGeneralMetricsValidation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json, general_metrics_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             general_metrics_schema.deserialize(dummy_json)
 
     def test_minimal_payload(self, general_metrics_schema):
         dummy_json = [{'tags': [['counter_a', 15.5], ['counter_b', 63]]}]
         deserialized = general_metrics_schema.deserialize(dummy_json)[0]
         expected = {'namespace': '',
                     'server_name': 'unknown',
                     'tags': [('counter_a', 15.5), ('counter_b', 63)],
                     'timestamp': datetime.utcnow()}
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['server_name'] == expected['server_name']
         assert deserialized['tags'] == expected['tags']
 
     def test_normal_payload(self, general_metrics_schema):
         import appenlight.tests.payload_examples as payload_examples
         dummy_json = [payload_examples.METRICS_PAYLOAD]
         deserialized = general_metrics_schema.deserialize(dummy_json)[0]
         expected = {'namespace': 'some.monitor',
                     'server_name': 'server.name',
                     'tags': [('usage_foo', 15.5), ('usage_bar', 63)],
                     'timestamp': datetime.utcnow()}
         assert deserialized['namespace'] == expected['namespace']
         assert deserialized['server_name'] == expected['server_name']
         assert deserialized['tags'] == expected['tags']
 
 
 @pytest.mark.usefixtures('request_metrics_schema')
 class TestAPIRequestMetricsValidation(object):
     @pytest.mark.parametrize('dummy_json', ['', {}, [], None])
     def test_no_payload(self, dummy_json, request_metrics_schema):
         import colander
 
         with pytest.raises(colander.Invalid):
             print(request_metrics_schema.deserialize(dummy_json))
 
     def test_normal_payload(self, request_metrics_schema):
         import appenlight.tests.payload_examples as payload_examples
         dummy_json = payload_examples.REQUEST_METRICS_EXAMPLES
         deserialized = request_metrics_schema.deserialize(dummy_json)[0]
         expected = {'metrics': [('dir/module:func',
                                  {'custom': 0.0,
                                   'custom_calls': 0.0,
                                   'main': 0.01664,
                                   'nosql': 0.00061,
                                   'nosql_calls': 23.0,
                                   'remote': 0.0,
                                   'remote_calls': 0.0,
                                   'requests': 1,
                                   'sql': 0.00105,
                                   'sql_calls': 2.0,
                                   'tmpl': 0.0,
                                   'tmpl_calls': 0.0}),
                                 ('SomeView.function',
                                  {'custom': 0.0,
                                   'custom_calls': 0.0,
                                   'main': 0.647261,
                                   'nosql': 0.306554,
                                   'nosql_calls': 140.0,
                                   'remote': 0.0,
                                   'remote_calls': 0.0,
                                   'requests': 28,
                                   'sql': 0.0,
                                   'sql_calls': 0.0,
                                   'tmpl': 0.0,
                                   'tmpl_calls': 0.0})],
                     'server': 'some.server.hostname',
                     'timestamp': datetime.utcnow()}
         assert deserialized['server'] == expected['server']
         metric = deserialized['metrics'][0]
         expected_metric = expected['metrics'][0]
         assert metric[0] == expected_metric[0]
         assert sorted(metric[1].items()) == sorted(expected_metric[1].items())
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPIReportsView(object):
     def test_no_json_payload(self, default_application):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import reports_create
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.unsafe_json_body = ''
         request.context = context
         route = mock.Mock()
         route.name = 'api_reports'
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             response = reports_create(request)
 
     def test_single_proper_json_0_5_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_reports'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD)]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_grouping_0_5(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_reports'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD),
                                     copy.deepcopy(PYTHON_PAYLOAD)]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 2
 
     def test_grouping_different_reports_0_5(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.views.api import reports_create
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_reports'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         PYTHON_PAYLOAD = payload_examples.PYTHON_PAYLOAD_0_5
         PARSED_REPORT_404 = payload_examples.PARSED_REPORT_404
         request.unsafe_json_body = [copy.deepcopy(PYTHON_PAYLOAD),
                                     copy.deepcopy(PARSED_REPORT_404)]
         reports_create(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 2
         assert report.total_reports == 1
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAirbrakeXMLView(object):
 
     def test_normal_payload_parsing(self):
         import datetime
         import defusedxml.ElementTree as ElementTree
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.lib.utils.airbrake import parse_airbrake_xml
         from appenlight.validators import ReportListSchema_0_5
 
         context = DummyContext()
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/xml'})
         request.context = context
         request.context.possibly_public = False
         root = ElementTree.fromstring(payload_examples.AIRBRAKE_RUBY_EXAMPLE)
         request.context.airbrake_xml_etree = root
         error_dict = parse_airbrake_xml(request)
         schema = ReportListSchema_0_5().bind(utcnow=datetime.datetime.utcnow())
         deserialized_report = schema.deserialize([error_dict])[0]
         assert deserialized_report['client'] == 'Airbrake Notifier'
         assert deserialized_report['error'] == 'NameError: undefined local variable or method `sdfdfdf\' for #<#<Class:0x000000039a8b90>:0x00000002c53df0>'
         assert deserialized_report['http_status'] == 500
         assert deserialized_report['language'] == 'unknown'
         assert deserialized_report['message'] == ''
         assert deserialized_report['occurences'] == 1
         assert deserialized_report['priority'] == 5
         d_request = deserialized_report['request']
         assert d_request['GET'] == {'test': '1234'}
         assert d_request['action_dispatch.request.parameters'] == {
             'action': 'index',
             'controller': 'welcome',
             'test': '1234'}
         assert deserialized_report['request_id'] == 'c11b2267f3ad8b00a1768cae35559fa1'
         assert deserialized_report['server'] == 'ergo-desktop'
         assert deserialized_report['traceback'][0] == {
             'cline': 'block in start_thread',
             'file': '/home/ergo/.rbenv/versions/1.9.3-p327/lib/ruby/1.9.1/webrick/server.rb',
             'fn': 'block in start_thread',
             'line': '191',
             'module': '',
             'vars': {}}
         assert deserialized_report['traceback'][-1] == {
             'cline': '_app_views_welcome_index_html_erb___2570061166873166679_31748940',
             'file': '[PROJECT_ROOT]/app/views/welcome/index.html.erb',
             'fn': '_app_views_welcome_index_html_erb___2570061166873166679_31748940',
             'line': '3',
             'module': '',
             'vars': {}}
         assert deserialized_report['url'] == 'http://0.0.0.0:3000/welcome/index?test=1234'
         assert deserialized_report['view_name'] == 'welcome:index'
 
     def test_normal_payload_view(self):
         import defusedxml.ElementTree as ElementTree
         import appenlight.tests.payload_examples as payload_examples
 
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import airbrake_xml_compat
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/xml'})
         request.context = context
         request.context.possibly_public = False
         root = ElementTree.fromstring(payload_examples.AIRBRAKE_RUBY_EXAMPLE)
         request.context.airbrake_xml_etree = root
         route = mock.Mock()
         route.name = 'api_airbrake'
         request.matched_route = route
         result = airbrake_xml_compat(request)
         assert '<notice><id>' in result
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPILogView(object):
     def test_no_json_payload(self, base_app):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import logs_create
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.context = context
         request.registry = base_app.registry
         request.unsafe_json_body = ''
         route = mock.Mock()
         route.name = 'api_logs'
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             response = logs_create(request)
 
     def test_single_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_logs'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         request.unsafe_json_body = [copy.deepcopy(
             payload_examples.LOG_EXAMPLES[0])]
         logs_create(request)
         query = DBSession.query(Log)
         log = query.first()
         assert query.count() == 1
         assert log.message == "OMG ValueError happened"
 
     def test_multiple_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_logs'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
         LOG_PAYLOAD = payload_examples.LOG_EXAMPLES[0]
         LOG_PAYLOAD2 = payload_examples.LOG_EXAMPLES[1]
         request.unsafe_json_body = copy.deepcopy([LOG_PAYLOAD, LOG_PAYLOAD2])
         logs_create(request)
         query = DBSession.query(Log).order_by(sa.asc(Log.log_id))
         assert query.count() == 2
         assert query[0].message == "OMG ValueError happened"
         assert query[1].message == "OMG ValueError happened2"
 
     def test_public_key_rewriting(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.log import Log
         from appenlight.views.api import logs_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_logs'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.matched_route = route
 
         LOG_PAYLOAD = copy.deepcopy(payload_examples.LOG_EXAMPLES[0])
         LOG_PAYLOAD2 = copy.deepcopy(payload_examples.LOG_EXAMPLES[1])
         LOG_PAYLOAD['primary_key'] = 'X2'
         LOG_PAYLOAD2['primary_key'] = 'X2'
         request.unsafe_json_body = [LOG_PAYLOAD, LOG_PAYLOAD2]
         logs_create(request)
 
         query = DBSession.query(Log).order_by(sa.asc(Log.log_id))
         assert query.count() == 1
         assert query[0].message == "OMG ValueError happened2"
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPIGeneralMetricsView(object):
     def test_no_json_payload(self, base_app):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import general_metrics_create
         route = mock.Mock()
         route.name = 'api_general_metrics'
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.context = context
         request.registry = base_app.registry
         request.unsafe_json_body = ''
         request.matched_route = route
         with pytest.raises(colander.Invalid):
             general_metrics_create(request)
 
     def test_single_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.metric import Metric
         from appenlight.views.api import general_metrics_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_general_metric'
         request = pyramid.threadlocal.get_current_request()
         request.matched_route = route
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.unsafe_json_body = payload_examples.METRICS_PAYLOAD
         general_metrics_create(request)
         query = DBSession.query(Metric)
         metric = query.first()
         assert query.count() == 1
         assert metric.namespace == 'some.monitor'
 
     def test_multiple_json_payload(self):
         import appenlight.tests.payload_examples as payload_examples
         from appenlight.models.metric import Metric
         from appenlight.views.api import general_metrics_create
         from appenlight.models.services.application import ApplicationService
         route = mock.Mock()
         route.name = 'api_general_metrics'
         request = pyramid.threadlocal.get_current_request()
         request.matched_route = route
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request.context = context
         request.unsafe_json_body = [
             copy.deepcopy(payload_examples.METRICS_PAYLOAD),
             copy.deepcopy(payload_examples.METRICS_PAYLOAD),
         ]
         general_metrics_create(request)
         query = DBSession.query(Metric)
         metric = query.first()
         assert query.count() == 2
         assert metric.namespace == 'some.monitor'
 
 
 class TestGroupingMessageReplacements(object):
     def replace_default_repr_python(self):
         test_str = '''
         ConnectionError: ConnectionError((<urllib3.connection.HTTPConnection object at 0x7f87a0ba9fd0>, 'Connection to domain.gr timed out. (connect timeout=10)')) caused by: ConnectTimeoutError((<urllib3.connection.HTTPConnection object at 0x7f87a0ba9fd0>, 'Connection to domain.gr timed out. (connect timeout=10)'))
         '''
         regex = r'<(.*?) object at (.*?)>'
 
 
 class TestRulesKeyGetter(object):
     def test_default_dict_getter_top_key(self):
         from appenlight.lib.rule import Rule
         struct = {
             "a": {
                 "b": 'b',
                 "c": {
                     "d": 'd',
                     "g": {
                         "h": 'h'
                     }
                 },
                 "e": 'e'
             },
             "f": 'f'
         }
         result = Rule.default_dict_struct_getter(struct, "a")
         assert result == struct['a']
 
     def test_default_dict_getter_sub_key(self):
         from appenlight.lib.rule import Rule
         struct = {
             "a": {
                 "b": 'b',
                 "c": {
                     "d": 'd',
                     "g": {
                         "h": 'h'
                     }
                 },
                 "e": 'e'
             },
             "f": 'f'
         }
         result = Rule.default_dict_struct_getter(struct, 'a:b')
         assert result == struct['a']['b']
         result = Rule.default_dict_struct_getter(struct, 'a:c:d')
         assert result == struct['a']['c']['d']
 
     def test_default_obj_getter_top_key(self):
         from appenlight.lib.rule import Rule
         class TestStruct(object):
             def __init__(self, a, b):
                 self.a = a
                 self.b = b
 
         struct = TestStruct(a='a',
                             b=TestStruct(a='x', b='y'))
         result = Rule.default_obj_struct_getter(struct, "a")
         assert result == struct.a
 
     def test_default_obj_getter_sub_key(self):
         from appenlight.lib.rule import Rule
         class TestStruct(object):
             def __init__(self, name, a, b):
                 self.name = name
                 self.a = a
                 self.b = b
 
             def __repr__(self):
                 return '<obj {}>'.format(self.name)
 
         c = TestStruct('c', a=5, b='z')
         b = TestStruct('b', a=c, b='y')
         struct = TestStruct('a', a='a', b=b)
         result = Rule.default_obj_struct_getter(struct, 'b:b')
         assert result == struct.b.b
         result = Rule.default_obj_struct_getter(struct, 'b:a:b')
         assert result == struct.b.a.b
 
 
 @pytest.mark.usefixtures('report_type_matrix')
 class TestRulesParsing():
     @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
         ('eq', 500, 500, True),
         ('eq', 600, 500, False),
         ('eq', 300, 500, False),
         ('eq', "300", 500, False),
         ('eq', "600", 500, False),
         ('eq', "500", 500, True),
         ('ne', 500, 500, False),
         ('ne', 600, 500, True),
         ('ne', 300, 500, True),
         ('ne', "300", 500, True),
         ('ne', "600", 500, True),
         ('ne', "500", 500, False),
         ('ge', 500, 500, True),
         ('ge', 600, 500, True),
         ('ge', 499, 500, False),
         ('gt', 499, 500, False),
         ('gt', 500, 500, False),
         ('gt', 501, 500, True),
         ('le', 499, 500, True),
         ('le', 500, 500, True),
         ('le', 501, 500, False),
         ('lt', 499, 500, True),
         ('lt', 500, 500, False),
         ('lt', 501, 500, False),
     ])
     def test_single_op_int(self, op, struct_value, test_value, match_result,
                            report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "op": op,
             "field": "http_status",
             "value": test_value
         }
         rule = Rule(rule_config, report_type_matrix)
 
         data = {
             "http_status": struct_value
         }
         assert rule.match(data) is match_result
 
     @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
         ('ge', "500.01", 500, True),
         ('ge', "500.01", 500.02, False),
         ('le', "500.01", 500.02, True)
     ])
     def test_single_op_float(self, op, struct_value, test_value, match_result,
                              report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "op": op,
             "field": "duration",
             "value": test_value
         }
         rule = Rule(rule_config, report_type_matrix)
 
         data = {
             "duration": struct_value
         }
         assert rule.match(data) is match_result
 
     @pytest.mark.parametrize("op, struct_value, test_value, match_result", [
         ('contains', 'foo bar baz', 'foo', True),
         ('contains', 'foo bar baz', 'bar', True),
         ('contains', 'foo bar baz', 'dupa', False),
         ('startswith', 'foo bar baz', 'foo', True),
         ('startswith', 'foo bar baz', 'bar', False),
         ('endswith', 'foo bar baz', 'baz', True),
         ('endswith', 'foo bar baz', 'bar', False),
     ])
     def test_single_op_string(self, op, struct_value, test_value,
                               match_result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "op": op,
             "field": "error",
             "value": test_value
         }
         rule = Rule(rule_config, report_type_matrix)
 
         data = {
             "error": struct_value
         }
         assert rule.match(data) is match_result
 
     @pytest.mark.parametrize("field, value, s_type", [
         ('field_unicode', 500, str),
         ('field_unicode', 500.0, str),
         ('field_unicode', "500", str),
         ('field_int', "500", int),
         ('field_int', 500, int),
         ('field_int', 500.0, int),
         ('field_float', "500", float),
         ('field_float', 500, float),
         ('field_float', 500.0, float),
     ])
     def test_type_normalization(self, field, value, s_type):
         from appenlight.lib.rule import Rule
         type_matrix = {
             'field_unicode': {"type": 'unicode'},
             'field_float': {"type": 'float'},
             'field_int': {"type": 'int'},
         }
 
         rule = Rule({}, type_matrix)
         n_value = rule.normalized_type(field, value)
         assert isinstance(n_value, s_type) is True
 
 
 @pytest.mark.usefixtures('report_type_matrix')
 class TestNestedRuleParsing():
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
          False),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          False),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 11}},
          False),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 5}},
          True),
     ])
     def test_NOT_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__NOT__",
             "rules": [
                 {
                     "op": "ge",
                     "field": "group:occurences",
                     "value": "10"
                 },
                 {
                     "op": "ge",
                     "field": "group:priority",
                     "value": "4"
                 }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
          True),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 11}},
          False),
     ])
     def test_nested_OR_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "group:occurences",
                             "value": "10"
                         },
                         {
                             "op": "ge",
                             "field": "group:priority",
                             "value": "4"
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "http_status",
                     "value": "500"
                 }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 501, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
          True),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 1}},
          False),
     ])
     def test_nested_OR_OR_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {"field": "__OR__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"op": "ge",
                       "field": "group:priority",
                       "value": "4"
                       }
                  ]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11}},
          True),
         ({"http_status": 101, "group": {"priority": 7, "occurences": 11}},
          False),
         ({"http_status": 500, "group": {"priority": 1, "occurences": 1}},
          False),
         ({"http_status": 101, "group": {"priority": 3, "occurences": 1}},
          False),
     ])
     def test_nested_AND_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {"field": "__AND__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"op": "ge",
                       "field": "group:priority",
                       "value": "4"
                       }]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": '/test/register', "error": "foo test bar"}, True),
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": '/test/register', "error": "foo INVALID bar"}, False),
     ])
     def test_nested_AND_AND_AND_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {"field": "__AND__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"field": "__AND__",
                       "rules": [
                           {"op": "endswith",
                            "field": "url_path",
                            "value": "register"},
                           {"op": "contains",
                            "field": "error",
                            "value": "test"}]}]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("data, result", [
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": 6, "error": 3}, False),
         ({"http_status": 500, "group": {"priority": 7, "occurences": 11},
           "url_path": '/test/register', "error": "foo INVALID bar"}, True),
     ])
     def test_nested_AND_AND_OR_rule(self, data, result, report_type_matrix):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {"field": "__AND__",
                  "rules": [
                      {"op": "ge",
                       "field": "group:occurences",
                       "value": "10"
                       },
                      {"field": "__OR__",
                       "rules": [
                           {"op": "endswith",
                            "field": "url_path",
                            "value": "register"
                            },
                           {"op": "contains",
                            "field": "error",
                            "value": "test"
                            }]}]
                  },
                 {"op": "eq",
                  "field": "http_status",
                  "value": "500"
                  }
             ]
         }
 
         rule = Rule(rule_config, report_type_matrix)
         assert rule.match(data) is result
 
     @pytest.mark.parametrize("op, field, value, should_fail", [
         ('eq', 'http_status', "1", False),
         ('ne', 'http_status', "1", False),
         ('ne', 'http_status', "foo", True),
         ('startswith', 'http_status', "1", True),
         ('eq', 'group:priority', "1", False),
         ('ne', 'group:priority', "1", False),
         ('ge', 'group:priority', "1", False),
         ('le', 'group:priority', "1", False),
         ('startswith', 'group:priority', "1", True),
         ('eq', 'url_domain', "1", False),
         ('ne', 'url_domain', "1", False),
         ('startswith', 'url_domain', "1", False),
         ('endswith', 'url_domain', "1", False),
         ('contains', 'url_domain', "1", False),
         ('ge', 'url_domain', "1", True),
         ('eq', 'url_path', "1", False),
         ('ne', 'url_path', "1", False),
         ('startswith', 'url_path', "1", False),
         ('endswith', 'url_path', "1", False),
         ('contains', 'url_path', "1", False),
         ('ge', 'url_path', "1", True),
         ('eq', 'error', "1", False),
         ('ne', 'error', "1", False),
         ('startswith', 'error', "1", False),
         ('endswith', 'error', "1", False),
         ('contains', 'error', "1", False),
         ('ge', 'error', "1", True),
         ('ge', 'url_path', "1", True),
         ('eq', 'tags:server_name', "1", False),
         ('ne', 'tags:server_name', "1", False),
         ('startswith', 'tags:server_name', "1", False),
         ('endswith', 'tags:server_name', "1", False),
         ('contains', 'tags:server_name', "1", False),
         ('ge', 'tags:server_name', "1", True),
         ('contains', 'traceback', "1", False),
         ('ge', 'traceback', "1", True),
         ('eq', 'group:occurences', "1", False),
         ('ne', 'group:occurences', "1", False),
         ('ge', 'group:occurences', "1", False),
         ('le', 'group:occurences', "1", False),
         ('contains', 'group:occurences', "1", True),
     ])
     def test_rule_validation(self, op, field, value, should_fail,
                              report_type_matrix):
         import colander
         from appenlight.validators import build_rule_schema
         rule_config = {
             "op": op,
             "field": field,
             "value": value
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         if should_fail:
             with pytest.raises(colander.Invalid):
                 schema.deserialize(rule_config)
         else:
             schema.deserialize(rule_config)
 
     def test_nested_proper_rule_validation(self, report_type_matrix):
         from appenlight.validators import build_rule_schema
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "group:occurences",
                             "value": "10"
                         },
                         {
                             "field": "__OR__",
                             "rules": [
                                 {
                                     "op": "endswith",
                                     "field": "url_path",
                                     "value": "register"
                                 },
                                 {
                                     "op": "contains",
                                     "field": "error",
                                     "value": "test"
                                 }
                             ]
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "http_status",
                     "value": "500"
                 }
             ]
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         deserialized = schema.deserialize(rule_config)
 
     def test_nested_bad_rule_validation(self, report_type_matrix):
         import colander
         from appenlight.validators import build_rule_schema
         rule_config = {
             "field": "__AND__",
             "rules": [
                 {
                     "field": "__AND__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "group:occurences",
                             "value": "10"
                         },
                         {
                             "field": "__OR__",
                             "rules": [
                                 {
                                     "op": "gt",
                                     "field": "url_path",
                                     "value": "register"
                                 },
                                 {
                                     "op": "contains",
                                     "field": "error",
                                     "value": "test"
                                 }
                             ]
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "http_status",
                     "value": "500"
                 }
             ]
         }
 
         schema = build_rule_schema(rule_config, report_type_matrix)
         with pytest.raises(colander.Invalid):
             deserialized = schema.deserialize(rule_config)
 
     def test_config_manipulator(self):
         from appenlight.lib.rule import Rule
         type_matrix = {
             'a': {"type": 'int',
                   "ops": ('eq', 'ne', 'ge', 'le',)},
             'b': {"type": 'int',
                   "ops": ('eq', 'ne', 'ge', 'le',)},
         }
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__OR__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "a",
                             "value": "10"
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "b",
                     "value": "500"
                 }
             ]
         }
 
         def rule_manipulator(rule):
             if 'value' in rule.config:
                 rule.config['value'] = "1"
 
         rule = Rule(rule_config, type_matrix,
                     config_manipulator=rule_manipulator)
         rule.match({"a": 1,
                     "b": "2"})
         assert rule.config['rules'][0]['rules'][0]['value'] == "1"
         assert rule.config['rules'][1]['value'] == "1"
         assert rule.type_matrix["b"]['type'] == "int"
 
     def test_dynamic_config_manipulator(self):
         from appenlight.lib.rule import Rule
         rule_config = {
             "field": "__OR__",
             "rules": [
                 {
                     "field": "__OR__",
                     "rules": [
                         {
                             "op": "ge",
                             "field": "a",
                             "value": "10"
                         }
                     ]
                 },
                 {
                     "op": "eq",
                     "field": "b",
                     "value": "500"
                 }
             ]
         }
 
         def rule_manipulator(rule):
             rule.type_matrix = {
                 'a': {"type": 'int',
                       "ops": ('eq', 'ne', 'ge', 'le',)},
                 'b': {"type": 'unicode',
                       "ops": ('eq', 'ne', 'ge', 'le',)},
             }
 
             if 'value' in rule.config:
                 if rule.config['field'] == 'a':
                     rule.config['value'] = "1"
                 elif rule.config['field'] == 'b':
                     rule.config['value'] = "2"
 
         rule = Rule(rule_config, {},
                     config_manipulator=rule_manipulator)
         rule.match({"a": 11,
                     "b": "55"})
         assert rule.config['rules'][0]['rules'][0]['value'] == "1"
         assert rule.config['rules'][1]['value'] == "2"
         assert rule.type_matrix["b"]['type'] == "unicode"
 
 
 @pytest.mark.usefixtures('base_app', 'with_migrations')
 class TestViewsWithForms(object):
     def test_bad_csrf(self):
         from appenlight.forms import CSRFException
         from appenlight.views.index import register
         post_data = {'dupa': 'dupa'}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         with pytest.raises(CSRFException):
             register(request)
 
     def test_proper_csrf(self):
         from appenlight.views.index import register
         request = pyramid.threadlocal.get_current_request()
         post_data = {'dupa': 'dupa',
                      'csrf_token': request.session.get_csrf_token()}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         result = register(request)
         assert result['form'].errors['email'][0] == 'This field is required.'
 
 
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'default_data')
 class TestRegistration(object):
     def test_invalid_form(self):
         from appenlight.views.index import register
         request = pyramid.threadlocal.get_current_request()
         post_data = {'user_name': '',
                      'user_password': '',
                      'email': '',
                      'csrf_token': request.session.get_csrf_token()}
         request = testing.DummyRequest(post=post_data)
         request.POST = webob.multidict.MultiDict(request.POST)
         result = register(request)
         assert result['form'].errors['user_name'][0] == \
                'This field is required.'
 
     def test_valid_form(self):
         from appenlight.views.index import register
         from ziggurat_foundations.models.services.user import UserService
         request = pyramid.threadlocal.get_current_request()
         post_data = {'user_name': 'foo',
                      'user_password': 'barr',
                      'email': '[e-mail]',
                      'csrf_token': request.session.get_csrf_token()}
         request = testing.DummyRequest(post=post_data)
         request.add_flash_to_headers = mock.Mock()
         request.POST = webob.multidict.MultiDict(request.POST)
         assert UserService.by_user_name('foo') is None
         register(request)
         user = UserService.by_user_name('foo')
         assert user.user_name == 'foo'
-        assert len(user.user_password) == 60
+        assert len(user.user_password) >= 60
 
 
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables',
                          'default_user')
 class TestApplicationCreation(object):
     def test_wrong_data(self):
         import appenlight.views.applications as applications
         from ziggurat_foundations.models.services.user import UserService
         request = pyramid.threadlocal.get_current_request()
         request.user = UserService.by_user_name('testuser')
         request.unsafe_json_body = {}
         request.headers['X-XSRF-TOKEN'] = request.session.get_csrf_token()
         response = applications.application_create(request)
         assert response.code == 422
 
     def test_proper_data(self):
         import appenlight.views.applications as applications
         from ziggurat_foundations.models.services.user import UserService
 
         request = pyramid.threadlocal.get_current_request()
         request.user = UserService.by_user_name('testuser')
         request.unsafe_json_body = {"resource_name": "app name",
                                     "domains": "foo"}
         request.headers['X-XSRF-TOKEN'] = request.session.get_csrf_token()
         app_dict = applications.application_create(request)
         assert app_dict['public_key'] is not None
         assert app_dict['api_key'] is not None
         assert app_dict['resource_name'] == 'app name'
         assert app_dict['owner_group_id'] is None
         assert app_dict['resource_id'] is not None
         assert app_dict['default_grouping'] == 'url_traceback'
         assert app_dict['possible_permissions'] == ('view', 'update_reports')
         assert app_dict['slow_report_threshold'] == 10
         assert app_dict['owner_user_name'] == 'testuser'
         assert app_dict['owner_user_id'] == request.user.id
         assert app_dict['domains'] is 'foo'
         assert app_dict['postprocessing_rules'] == []
         assert app_dict['error_report_threshold'] == 10
         assert app_dict['allow_permanent_storage'] is False
         assert app_dict['resource_type'] == 'application'
         assert app_dict['current_permissions'] == []
 
 
 @pytest.mark.usefixtures('default_application')
 @pytest.mark.usefixtures('base_app', 'with_migrations', 'clean_tables')
 class TestAPISentryView(object):
     def test_no_payload(self, default_application):
         import colander
         from appenlight.models.services.application import ApplicationService
         from appenlight.views.api import sentry_compat
         from appenlight.lib.request import JSONException
 
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/json'})
         request.unsafe_json_body = ''
         request.context = context
         route = mock.Mock()
         route.name = 'api_sentry'
         request.matched_route = route
         with pytest.raises(JSONException):
             sentry_compat(request)
 
     def test_java_client_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         route = mock.Mock()
         route.name = 'api_sentry'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
         request.body = b'eJy1UmFr2zAQ/S0T+7BCLOzYThp/C6xjG6SDLd/GCBf57Ki' \
                        b'RJSHJJiXkv+/UlC7p2kAZA33Ru6f33t1pz3BAHVayZhWr87' \
                        b'JMs+I6q3MsrifFep2vc1iXM1HMpgBTNmIdeg8tEvlmJ9AGa' \
                        b'fQ7goOkQoDOUmGcZpMkLZO0WGZFRadMiaHIR1EVnTMu3k3b' \
                        b'oiMgqJrXpgOpOVjLLTiPkWAVhMa4jih3MAAholfWyUDAksz' \
                        b'm1iopICbg8fWH52B8VWXZVYwHrWfV/jBipD2gW2no8CFMa5' \
                        b'JButCDSjoQG6mR6LgLDojPPn/7sbydL25ep34HGl+y3DiE+' \
                        b'lH0xXBXjMzFBsXW99SS7pWKYXRw91zqgK4BgZ4/DZVVP/cs' \
                        b'3NuzSZPfAKqP2Cdj4tw7U/cKH0fEFeiWQFqE2FIHAmMPjaN' \
                        b'Y/kHvbzY/JqdHUq9o/KxqQHkcsabX4piDuT4aK+pXG1ZNi/' \
                        b'IwOpEyruXC1LiB3vPO3BmOOxTUCIqv5LIg5H12oh9cf0l+P' \
                        b'MvP5P8kddgoFIEvMGzM5cRSD2aLJ6qTdHKm6nv9pPcRFba0' \
                        b'Kd0eleeCFuGN+9JZ9TaXIn/V5JYMBvxXg3L6PwzSE4dkfOb' \
                        b'w7CtfWmP85SdCs8OvA53fUV19cg=='
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_ruby_client_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_RUBY_ENCODED
         route = mock.Mock()
         route.name = 'api_sentry'
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/octet-stream',
                      'User-Agent': 'sentry-ruby/1.0.0',
                      'X-Sentry-Auth': 'Sentry sentry_version=5, '
                                       'sentry_client=raven-ruby/1.0.0, '
                                       'sentry_timestamp=1462378483, '
                                       'sentry_key=xxx, sentry_secret=xxx'
                      })
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
         request.body = SENTRY_RUBY_ENCODED
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_python_client_decoded_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_PYTHON_PAYLOAD_7
         route = mock.Mock()
         route.name = 'api_sentry'
         request = pyramid.threadlocal.get_current_request()
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
         request.body = json.dumps(SENTRY_PYTHON_PAYLOAD_7).encode('utf8')
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1
 
     def test_python_client_encoded_payload(self):
         from appenlight.views.api import sentry_compat
         from appenlight.models.services.application import ApplicationService
         from appenlight.models.report_group import ReportGroup
         from appenlight.tests.payload_examples import SENTRY_PYTHON_ENCODED
         route = mock.Mock()
         route.name = 'api_sentry'
         request = testing.DummyRequest(
             headers={'Content-Type': 'application/octet-stream',
                      'Content-Encoding': 'deflate',
                      'User-Agent': 'sentry-ruby/1.0.0',
                      'X-Sentry-Auth': 'Sentry sentry_version=5, '
                                       'sentry_client=raven-ruby/1.0.0, '
                                       'sentry_timestamp=1462378483, '
                                       'sentry_key=xxx, sentry_secret=xxx'
                      })
         context = DummyContext()
         context.resource = ApplicationService.by_id(1)
         context.resource.allow_permanent_storage = True
         request.context = context
         request.matched_route = route
         request.body = SENTRY_PYTHON_ENCODED
         sentry_compat(request)
         query = DBSession.query(ReportGroup)
         report = query.first()
         assert query.count() == 1
         assert report.total_reports == 1

backend/src/appenlight/views/admin/users.py

@@ -1,36 +1,36 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 from pyramid import security
-from appenlight.models.user import User
+from ziggurat_foundations.models.services.user import UserService
 
 import logging
 
 log = logging.getLogger(__name__)
 
 
 @view_config(route_name='section_view', permission='root_administration',
              match_param=['section=admin_section', 'view=relogin_user'],
              renderer='json', request_method='GET')
 def relogin_to_user(request):
-    user = User.by_id(request.GET.get('user_id'))
+    user = UserService.by_id(request.GET.get('user_id'))
     if not user:
         return HTTPNotFound()
     headers = security.remember(request, user.id)
     return HTTPFound(location=request.route_url('/'),
                      headers=headers)

backend/src/appenlight/views/applications.py

@@ -1,755 +1,760 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import json
 import logging
 import six
 
 from datetime import datetime, timedelta
 
 import colander
 from pyramid.httpexceptions import HTTPFound, HTTPUnprocessableEntity
 from pyramid.view import view_config
 from webob.multidict import MultiDict
 from zope.sqlalchemy import mark_changed
 from ziggurat_foundations.permissions import ANY_PERMISSION
 
 import appenlight.forms as forms
 from appenlight.models import DBSession
 from appenlight.models.resource import Resource
 from appenlight.models.application import Application
 from appenlight.models.application_postprocess_conf import \
     ApplicationPostprocessConf
-from appenlight.models.user import User
+from ziggurat_foundations.models.services.user import UserService
+from ziggurat_foundations.models.services.resource import ResourceService
+from ziggurat_foundations.models.services.user_resource_permission import UserResourcePermissionService
 from appenlight.models.user_resource_permission import UserResourcePermission
 from appenlight.models.group_resource_permission import GroupResourcePermission
 from appenlight.models.services.application import ApplicationService
 from appenlight.models.services.application_postprocess_conf import \
     ApplicationPostprocessConfService
 from appenlight.models.services.group import GroupService
 from appenlight.models.services.group_resource_permission import \
     GroupResourcePermissionService
 from appenlight.models.services.request_metric import RequestMetricService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.slow_call import SlowCallService
 from appenlight.lib import helpers as h
 from appenlight.lib.utils import build_filter_settings_from_query_dict
 from appenlight.security import RootFactory
 from appenlight.models.report import REPORT_TYPE_MATRIX
 from appenlight.validators import build_rule_schema
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 
 def app_not_found(request, id):
     """
     Redirects on non found and sets a flash message
     """
     request.session.flash(_('Application not found'), 'warning')
     return HTTPFound(
         location=request.route_url('applications', action='index'))
 
 
 @view_config(route_name='applications_no_id',
              renderer='json', request_method="GET", permission='authenticated')
 def applications_list(request):
     """
     Applications list
 
     if query params contain ?type=foo, it will list applications
     with one of those permissions for user,
     otherwise only list of owned applications will
     be returned
 
     appending ?root_list while being administration will allow to list all
     applications in the system
 
     """
     is_root = request.has_permission('root_administration',
                                      RootFactory(request))
     if is_root and request.GET.get('root_list'):
         resources = Resource.all().order_by(Resource.resource_name)
         resource_type = request.GET.get('resource_type', 'application')
         if resource_type:
             resources = resources.filter(
                 Resource.resource_type == resource_type)
     else:
         permissions = request.params.getall('permission')
         if permissions:
-            resources = request.user.resources_with_perms(
+            resources = UserService.resources_with_perms(
+                request.user,
                 permissions,
                 resource_types=[request.GET.get('resource_type',
                                                 'application')])
         else:
             resources = request.user.resources.filter(
                 Application.resource_type == request.GET.get(
                     'resource_type',
                     'application'))
     return [r.get_dict(include_keys=['resource_id', 'resource_name', 'domains',
                                      'owner_user_name', 'owner_group_name'])
             for
             r in resources]
 
 
 @view_config(route_name='applications', renderer='json',
              request_method="GET", permission='view')
 def application_GET(request):
     resource = request.context.resource
     include_sensitive_info = False
     if request.has_permission('edit'):
         include_sensitive_info = True
     resource_dict = resource.get_dict(
         include_perms=include_sensitive_info,
         include_processing_rules=include_sensitive_info)
     return resource_dict
 
 
 @view_config(route_name='applications_no_id', request_method="POST",
              renderer='json', permission='create_resources')
 def application_create(request):
     """
     Creates new application instances
     """
     user = request.user
     form = forms.ApplicationCreateForm(MultiDict(request.unsafe_json_body),
                                        csrf_context=request)
     if form.validate():
         session = DBSession()
         resource = Application()
         DBSession.add(resource)
         form.populate_obj(resource)
         resource.api_key = resource.generate_api_key()
         user.resources.append(resource)
         request.session.flash(_('Application created'))
         DBSession.flush()
         mark_changed(session)
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
     return resource.get_dict()
 
 
 @view_config(route_name='applications', request_method="PATCH",
              renderer='json', permission='edit')
 def application_update(request):
     """
     Updates main application configuration
     """
     resource = request.context.resource
     if not resource:
         return app_not_found()
 
     # disallow setting permanent storage by non-admins
     # use default/non-resource based context for this check
     req_dict = copy.copy(request.unsafe_json_body)
     if not request.has_permission('root_administration', RootFactory(request)):
         req_dict['allow_permanent_storage'] = ''
     if not req_dict.get('uptime_url'):
         # needed cause validator is still triggered by default
         req_dict.pop('uptime_url', '')
     application_form = forms.ApplicationUpdateForm(MultiDict(req_dict),
                                                    csrf_context=request)
     if application_form.validate():
         application_form.populate_obj(resource)
         request.session.flash(_('Application updated'))
     else:
         return HTTPUnprocessableEntity(body=application_form.errors_json)
 
     include_sensitive_info = False
     if request.has_permission('edit'):
         include_sensitive_info = True
     resource_dict = resource.get_dict(
         include_perms=include_sensitive_info,
         include_processing_rules=include_sensitive_info)
     return resource_dict
 
 
 @view_config(route_name='applications_property', match_param='key=api_key',
              request_method="POST", renderer='json',
              permission='delete')
 def application_regenerate_key(request):
     """
     Regenerates API keys for application
     """
     resource = request.context.resource
 
     form = forms.CheckPasswordForm(MultiDict(request.unsafe_json_body),
                                    csrf_context=request)
     form.password.user = request.user
 
     if form.validate():
         resource.api_key = resource.generate_api_key()
         resource.public_key = resource.generate_api_key()
         msg = 'API keys regenerated - please update your application config.'
         request.session.flash(_(msg))
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
     if request.has_permission('edit'):
         include_sensitive_info = True
     resource_dict = resource.get_dict(
         include_perms=include_sensitive_info,
         include_processing_rules=include_sensitive_info)
     return resource_dict
 
 
 @view_config(route_name='applications_property',
              match_param='key=delete_resource',
              request_method="PATCH", renderer='json', permission='delete')
 def application_remove(request):
     """
     Removes application resources
     """
     resource = request.context.resource
     # we need polymorphic object here, to properly launch sqlalchemy events
     resource = ApplicationService.by_id(resource.resource_id)
     form = forms.CheckPasswordForm(MultiDict(request.safe_json_body or {}),
                                    csrf_context=request)
     form.password.user = request.user
     if form.validate():
         DBSession.delete(resource)
         request.session.flash(_('Application removed'))
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
     return True
 
 
 @view_config(route_name='applications_property', match_param='key=owner',
              request_method="PATCH", renderer='json', permission='delete')
 def application_ownership_transfer(request):
     """
     Allows application owner to transfer application ownership to other user
     """
     resource = request.context.resource
     form = forms.ChangeApplicationOwnerForm(
         MultiDict(request.safe_json_body or {}), csrf_context=request)
     form.password.user = request.user
     if form.validate():
-        user = User.by_user_name(form.user_name.data)
+        user = UserService.by_user_name(form.user_name.data)
         user.resources.append(resource)
         # remove integrations to not leak security data of external applications
         for integration in resource.integrations[:]:
             resource.integrations.remove(integration)
         request.session.flash(_('Application transfered'))
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
     return True
 
 
 @view_config(route_name='applications_property',
              match_param='key=postprocessing_rules', renderer='json',
              request_method='POST', permission='edit')
 def applications_postprocess_POST(request):
     """
     Creates new postprocessing rules for applications
     """
     resource = request.context.resource
     conf = ApplicationPostprocessConf()
     conf.do = 'postprocess'
     conf.new_value = '1'
     resource.postprocess_conf.append(conf)
     DBSession.flush()
     return conf.get_dict()
 
 
 @view_config(route_name='applications_property',
              match_param='key=postprocessing_rules', renderer='json',
              request_method='PATCH', permission='edit')
 def applications_postprocess_PATCH(request):
     """
     Creates new postprocessing rules for applications
     """
     json_body = request.unsafe_json_body
 
     schema = build_rule_schema(json_body['rule'], REPORT_TYPE_MATRIX)
     try:
         schema.deserialize(json_body['rule'])
     except colander.Invalid as exc:
         return HTTPUnprocessableEntity(body=json.dumps(exc.asdict()))
 
     resource = request.context.resource
     conf = ApplicationPostprocessConfService.by_pkey_and_resource_id(
         json_body['pkey'], resource.resource_id)
     conf.rule = request.unsafe_json_body['rule']
     # for now hardcode int since we dont support anything else so far
     conf.new_value = int(request.unsafe_json_body['new_value'])
     return conf.get_dict()
 
 
 @view_config(route_name='applications_property',
              match_param='key=postprocessing_rules', renderer='json',
              request_method='DELETE', permission='edit')
 def applications_postprocess_DELETE(request):
     """
     Removes application postprocessing rules
     """
     form = forms.ReactorForm(request.POST, csrf_context=request)
     resource = request.context.resource
     if form.validate():
         for postprocess_conf in resource.postprocess_conf:
             if postprocess_conf.pkey == int(request.GET['pkey']):
                 # remove rule
                 DBSession.delete(postprocess_conf)
     return True
 
 
 @view_config(route_name='applications_property',
              match_param='key=report_graphs', renderer='json',
              permission='view')
 @view_config(route_name='applications_property',
              match_param='key=slow_report_graphs', renderer='json',
              permission='view')
 def get_application_report_stats(request):
     query_params = request.GET.mixed()
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     if not filter_settings.get('start_date'):
         delta = timedelta(hours=1)
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     result = ReportGroupService.get_report_stats(request, filter_settings)
     return result
 
 
 @view_config(route_name='applications_property',
              match_param='key=metrics_graphs', renderer='json',
              permission='view')
 def metrics_graphs(request):
     """
     Handles metric dashboard graphs
     Returns information for time/tier breakdown
     """
     query_params = request.GET.mixed()
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
 
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     delta = timedelta(hours=1)
     if not filter_settings.get('start_date'):
         filter_settings['start_date'] = filter_settings['end_date'] - delta
     if filter_settings['end_date'] <= filter_settings['start_date']:
         filter_settings['end_date'] = filter_settings['start_date']
 
     delta = filter_settings['end_date'] - filter_settings['start_date']
     if delta < h.time_deltas.get('12h')['delta']:
         divide_by_min = 1
     elif delta <= h.time_deltas.get('3d')['delta']:
         divide_by_min = 5.0
     elif delta >= h.time_deltas.get('2w')['delta']:
         divide_by_min = 60.0 * 24
     else:
         divide_by_min = 60.0
 
     results = RequestMetricService.get_metrics_stats(
         request, filter_settings)
     # because requests are PER SECOND / we divide 1 min stats by 60
     # requests are normalized to 1 min average
     # results are average seconds time spent per request in specific area
     for point in results:
         if point['requests']:
             point['main'] = (point['main'] - point['sql'] -
                              point['nosql'] - point['remote'] -
                              point['tmpl'] -
                              point['custom']) / point['requests']
             point['sql'] = point['sql'] / point['requests']
             point['nosql'] = point['nosql'] / point['requests']
             point['remote'] = point['remote'] / point['requests']
             point['tmpl'] = point['tmpl'] / point['requests']
             point['custom'] = point['custom'] / point['requests']
             point['requests_2'] = point['requests'] / 60.0 / divide_by_min
 
     selected_types = ['main', 'sql', 'nosql', 'remote', 'tmpl', 'custom']
 
     for point in results:
         for stat_type in selected_types:
             point[stat_type] = round(point.get(stat_type, 0), 3)
 
     return results
 
 
 @view_config(route_name='applications_property',
              match_param='key=response_graphs', renderer='json',
              permission='view')
 def response_graphs(request):
     """
     Handles dashboard infomation for avg. response time split by today,
     2 days ago and week ago
     """
     query_params = request.GET.mixed()
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
 
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     delta = timedelta(hours=1)
     if not filter_settings.get('start_date'):
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     result_now = RequestMetricService.get_metrics_stats(
         request, filter_settings)
 
     filter_settings_2d = filter_settings.copy()
     filter_settings_2d['start_date'] = filter_settings['start_date'] - \
                                        timedelta(days=2)
     filter_settings_2d['end_date'] = filter_settings['end_date'] - \
                                      timedelta(days=2)
     result_2d = RequestMetricService.get_metrics_stats(
         request, filter_settings_2d)
 
     filter_settings_7d = filter_settings.copy()
     filter_settings_7d['start_date'] = filter_settings['start_date'] - \
                                        timedelta(days=7)
     filter_settings_7d['end_date'] = filter_settings['end_date'] - \
                                      timedelta(days=7)
     result_7d = RequestMetricService.get_metrics_stats(
         request, filter_settings_7d)
 
     plot_data = []
 
     for item in result_now:
         point = {'x': item['x'], 'today': 0, 'days_ago_2': 0,
                  'days_ago_7': 0}
         if item['requests']:
             point['today'] = round(item['main'] / item['requests'], 3)
         plot_data.append(point)
 
     for i, item in enumerate(result_2d[:len(plot_data)]):
         plot_data[i]['days_ago_2'] = 0
         point = result_2d[i]
         if point['requests']:
             plot_data[i]['days_ago_2'] = round(point['main'] /
                                                point['requests'], 3)
 
     for i, item in enumerate(result_7d[:len(plot_data)]):
         plot_data[i]['days_ago_7'] = 0
         point = result_7d[i]
         if point['requests']:
             plot_data[i]['days_ago_7'] = round(point['main'] /
                                                point['requests'], 3)
 
     return plot_data
 
 
 @view_config(route_name='applications_property',
              match_param='key=requests_graphs', renderer='json',
              permission='view')
 def requests_graphs(request):
     """
     Handles dashboard infomation for avg. response time split by today,
     2 days ago and week ago
     """
     query_params = request.GET.mixed()
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
 
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     delta = timedelta(hours=1)
     if not filter_settings.get('start_date'):
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     result_now = RequestMetricService.get_metrics_stats(
         request, filter_settings)
 
     delta = filter_settings['end_date'] - filter_settings['start_date']
     if delta < h.time_deltas.get('12h')['delta']:
         seconds = h.time_deltas['1m']['minutes'] * 60.0
     elif delta <= h.time_deltas.get('3d')['delta']:
         seconds = h.time_deltas['5m']['minutes'] * 60.0
     elif delta >= h.time_deltas.get('2w')['delta']:
         seconds = h.time_deltas['24h']['minutes'] * 60.0
     else:
         seconds = h.time_deltas['1h']['minutes'] * 60.0
 
     for item in result_now:
         if item['requests']:
             item['requests'] = round(item['requests'] / seconds, 3)
     return result_now
 
 
 @view_config(route_name='applications_property',
              match_param='key=apdex_stats', renderer='json',
              permission='view')
 def get_apdex_stats(request):
     """
     Returns information and calculates APDEX score per server for dashboard
     server information (upper right stats boxes)
     """
     query_params = request.GET.mixed()
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
     # make sure we have only one resource here to don't produce
     # weird results when we have wrong app in app selector
     filter_settings['resource'] = [filter_settings['resource'][0]]
 
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     delta = timedelta(hours=1)
     if not filter_settings.get('start_date'):
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     return RequestMetricService.get_apdex_stats(request, filter_settings)
 
 
 @view_config(route_name='applications_property', match_param='key=slow_calls',
              renderer='json', permission='view')
 def get_slow_calls(request):
     """
     Returns information for time consuming calls in specific time interval
     """
     query_params = request.GET.mixed()
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
 
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     delta = timedelta(hours=1)
     if not filter_settings.get('start_date'):
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     return SlowCallService.get_time_consuming_calls(request, filter_settings)
 
 
 @view_config(route_name='applications_property',
              match_param='key=requests_breakdown',
              renderer='json', permission='view')
 def get_requests_breakdown(request):
     """
     Used on dashboard to get information which views are most used in
     a time interval
     """
     query_params = request.GET.mixed()
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     if not filter_settings.get('start_date'):
         delta = timedelta(hours=1)
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     series = RequestMetricService.get_requests_breakdown(
         request, filter_settings)
 
     results = []
     for row in series:
         d_row = {'avg_response': round(row['main'] / row['requests'], 3),
                  'requests': row['requests'],
                  'main': row['main'],
                  'view_name': row['key'],
                  'latest_details': row['latest_details'],
                  'percentage': round(row['percentage'] * 100, 1)}
 
         results.append(d_row)
 
     return results
 
 
 @view_config(route_name='applications_property',
              match_param='key=trending_reports', renderer='json',
              permission='view')
 def trending_reports(request):
     """
     Returns exception/slow reports trending for specific time interval
     """
     query_params = request.GET.mixed().copy()
     # pop report type to rewrite it to tag later
     report_type = query_params.pop('report_type', None)
     if report_type:
         query_params['type'] = report_type
 
     query_params['resource'] = (request.context.resource.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
 
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     if not filter_settings.get('start_date'):
         delta = timedelta(hours=1)
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     results = ReportGroupService.get_trending(request, filter_settings)
 
     trending = []
     for occurences, group in results:
         report_group = group.get_dict(request)
         # show the occurences in time range instead of global ones
         report_group['occurences'] = occurences
         trending.append(report_group)
 
     return trending
 
 
 @view_config(route_name='applications_property',
              match_param='key=integrations',
              renderer='json', permission='view')
 def integrations(request):
     """
     Integration list for given application
     """
     application = request.context.resource
     return {'resource': application}
 
 
 @view_config(route_name='applications_property',
              match_param='key=user_permissions', renderer='json',
              permission='owner', request_method='POST')
 def user_resource_permission_create(request):
     """
     Set new permissions for user for a resource
     """
     resource = request.context.resource
     user_name = request.unsafe_json_body.get('user_name')
-    user = User.by_user_name(user_name)
+    user = UserService.by_user_name(user_name)
     if not user:
-        user = User.by_email(user_name)
+        user = UserService.by_email(user_name)
     if not user:
         return False
 
     for perm_name in request.unsafe_json_body.get('permissions', []):
-        permission = UserResourcePermission.by_resource_user_and_perm(
+        permission = UserResourcePermissionService.by_resource_user_and_perm(
             user.id, perm_name, resource.resource_id)
         if not permission:
             permission = UserResourcePermission(perm_name=perm_name,
                                                 user_id=user.id)
             resource.user_permissions.append(permission)
     DBSession.flush()
-    perms = [p.perm_name for p in resource.perms_for_user(user)
+    perms = [p.perm_name for p in ResourceService.perms_for_user(resource, user)
              if p.type == 'user']
     result = {'user_name': user.user_name,
               'permissions': list(set(perms))}
     return result
 
 
 @view_config(route_name='applications_property',
              match_param='key=user_permissions', renderer='json',
              permission='owner', request_method='DELETE')
 def user_resource_permission_delete(request):
     """
     Removes user permission from specific resource
     """
     resource = request.context.resource
 
-    user = User.by_user_name(request.GET.get('user_name'))
+    user = UserService.by_user_name(request.GET.get('user_name'))
     if not user:
         return False
 
     for perm_name in request.GET.getall('permissions'):
-        permission = UserResourcePermission.by_resource_user_and_perm(
+        permission = UserResourcePermissionService.by_resource_user_and_perm(
             user.id, perm_name, resource.resource_id)
         resource.user_permissions.remove(permission)
     DBSession.flush()
-    perms = [p.perm_name for p in resource.perms_for_user(user)
+    perms = [p.perm_name for p in ResourceService.perms_for_user(resource, user)
              if p.type == 'user']
     result = {'user_name': user.user_name,
               'permissions': list(set(perms))}
     return result
 
 
 @view_config(route_name='applications_property',
              match_param='key=group_permissions', renderer='json',
              permission='owner', request_method='POST')
 def group_resource_permission_create(request):
     """
     Set new permissions for group for a resource
     """
     resource = request.context.resource
     group = GroupService.by_id(request.unsafe_json_body.get('group_id'))
     if not group:
         return False
 
     for perm_name in request.unsafe_json_body.get('permissions', []):
         permission = GroupResourcePermissionService.by_resource_group_and_perm(
             group.id, perm_name, resource.resource_id)
         if not permission:
             permission = GroupResourcePermission(perm_name=perm_name,
                                                  group_id=group.id)
             resource.group_permissions.append(permission)
     DBSession.flush()
-    perm_tuples = resource.groups_for_perm(
+    perm_tuples = ResourceService.groups_for_perm(
+        resource,
         ANY_PERMISSION,
         limit_group_permissions=True,
         group_ids=[group.id])
     perms = [p.perm_name for p in perm_tuples if p.type == 'group']
     result = {'group': group.get_dict(),
               'permissions': list(set(perms))}
     return result
 
 
 @view_config(route_name='applications_property',
              match_param='key=group_permissions', renderer='json',
              permission='owner', request_method='DELETE')
 def group_resource_permission_delete(request):
     """
     Removes group permission from specific resource
     """
     form = forms.ReactorForm(request.POST, csrf_context=request)
     form.validate()
     resource = request.context.resource
     group = GroupService.by_id(request.GET.get('group_id'))
     if not group:
         return False
 
     for perm_name in request.GET.getall('permissions'):
         permission = GroupResourcePermissionService.by_resource_group_and_perm(
             group.id, perm_name, resource.resource_id)
         resource.group_permissions.remove(permission)
     DBSession.flush()
-    perm_tuples = resource.groups_for_perm(
+    perm_tuples = ResourceService.groups_for_perm(
+        resource,
         ANY_PERMISSION,
         limit_group_permissions=True,
         group_ids=[group.id])
     perms = [p.perm_name for p in perm_tuples if p.type == 'group']
     result = {'group': group.get_dict(),
               'permissions': list(set(perms))}
     return result

backend/src/appenlight/views/events.py

@@ -1,59 +1,60 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from appenlight.lib.helpers import gen_pagination_headers
 from appenlight.models.services.event import EventService
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound
+from ziggurat_foundations.models.services.user import UserService
 
 
 @view_config(route_name='events_no_id',
              renderer='json', permission='authenticated')
 def fetch_events(request):
     """
     Returns list of log entries from Elasticsearch
     """
     event_paginator = EventService.get_paginator(
         user=request.user,
         page=1,
         items_per_page=100
     )
     headers = gen_pagination_headers(request, event_paginator)
     request.response.headers.update(headers)
 
     return [ev.get_dict() for ev in event_paginator.items]
 
 
 @view_config(route_name='events', renderer='json', request_method='PATCH',
              permission='authenticated')
 def event_PATCH(request):
-    resources = request.user.resources_with_perms(
-        ['view'], resource_types=request.registry.resource_types)
+    resources = UserService.resources_with_perms(
+        request.user, ['view'], resource_types=request.registry.resource_types)
     event = EventService.for_resource(
         [r.resource_id for r in resources],
         event_id=request.matchdict['event_id']).first()
     if not event:
         return HTTPNotFound()
     allowed_keys = ['status']
     for k, v in request.unsafe_json_body.items():
         if k in allowed_keys:
             if k == 'status':
                 event.close()
             else:
                 setattr(event, k, v)
         else:
             return HTTPBadRequest()
     return event.get_dict()

backend/src/appenlight/views/groups.py

@@ -1,191 +1,192 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPUnprocessableEntity, HTTPNotFound
 
+from ziggurat_foundations.models.services.user import UserService
 from appenlight.lib.utils import permission_tuple_to_dict
 from appenlight.models.services.config import ConfigService
 from appenlight.models.group import Group
 from appenlight.models.services.group import GroupService
 from appenlight.models.user import User
 from appenlight.models import DBSession
 from appenlight import forms
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 _ = str
 
 
 @view_config(route_name='groups_no_id', renderer='json',
              request_method="GET", permission='authenticated')
 def groups_list(request):
     """
     Returns groups list
     """
     groups = Group.all().order_by(Group.group_name)
     list_groups = ConfigService.by_key_and_section(
         'list_groups_to_non_admins', 'global')
     if list_groups.value or request.has_permission('root_administration'):
         return [g.get_dict() for g in groups]
     else:
         return []
 
 
 @view_config(route_name='groups_no_id', renderer='json',
              request_method="POST", permission='root_administration')
 def groups_create(request):
     """
     Returns groups list
     """
     form = forms.GroupCreateForm(
         MultiDict(request.safe_json_body or {}), csrf_context=request)
     if form.validate():
         log.info('registering group')
         group = Group()
         # insert new group here
         DBSession.add(group)
         form.populate_obj(group)
         request.session.flash(_('Group created'))
         DBSession.flush()
         return group.get_dict(include_perms=True)
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
 
 @view_config(route_name='groups', renderer='json',
              request_method="DELETE", permission='root_administration')
 def groups_DELETE(request):
     """
     Removes a groups permanently from db
     """
     msg = _('You cannot remove administrator group from the system')
     group = GroupService.by_id(request.matchdict.get('group_id'))
     if group:
         if group.id == 1:
             request.session.flash(msg, 'warning')
         else:
             DBSession.delete(group)
             request.session.flash(_('Group removed'))
             return True
     request.response.status = 422
     return False
 
 
 @view_config(route_name='groups', renderer='json',
              request_method="GET", permission='root_administration')
 @view_config(route_name='groups', renderer='json',
              request_method="PATCH", permission='root_administration')
 def group_update(request):
     """
     Updates group object
     """
     group = GroupService.by_id(request.matchdict.get('group_id'))
     if not group:
         return HTTPNotFound()
 
     if request.method == 'PATCH':
         form = forms.GroupCreateForm(
             MultiDict(request.unsafe_json_body), csrf_context=request)
         form._modified_group = group
         if form.validate():
             form.populate_obj(group)
         else:
             return HTTPUnprocessableEntity(body=form.errors_json)
     return group.get_dict(include_perms=True)
 
 
 @view_config(route_name='groups_property',
              match_param='key=resource_permissions',
              renderer='json', permission='root_administration')
 def groups_resource_permissions_list(request):
     """
     Get list of permissions assigned to specific resources
     """
     group = GroupService.by_id(request.matchdict.get('group_id'))
     if not group:
         return HTTPNotFound()
     return [permission_tuple_to_dict(perm) for perm in
-            group.resources_with_possible_perms()]
+            GroupService.resources_with_possible_perms(group)]
 
 
 @view_config(route_name='groups_property',
              match_param='key=users', request_method="GET",
              renderer='json', permission='root_administration')
 def groups_users_list(request):
     """
     Get list of permissions assigned to specific resources
     """
     group = GroupService.by_id(request.matchdict.get('group_id'))
     if not group:
         return HTTPNotFound()
     props = ['user_name', 'id', 'first_name', 'last_name', 'email',
              'last_login_date', 'status']
     users_dicts = []
     for user in group.users:
         u_dict = user.get_dict(include_keys=props)
-        u_dict['gravatar_url'] = user.gravatar_url(s=20)
+        u_dict['gravatar_url'] = UserService.gravatar_url(user, s=20)
         users_dicts.append(u_dict)
     return users_dicts
 
 
 @view_config(route_name='groups_property',
              match_param='key=users', request_method="DELETE",
              renderer='json', permission='root_administration')
 def groups_users_remove(request):
     """
     Get list of permissions assigned to specific resources
     """
     group = GroupService.by_id(request.matchdict.get('group_id'))
-    user = User.by_user_name(request.GET.get('user_name'))
+    user = UserService.by_user_name(request.GET.get('user_name'))
     if not group or not user:
         return HTTPNotFound()
     if len(group.users) > 1:
         group.users.remove(user)
         msg = "User removed from group"
         request.session.flash(msg)
         group.member_count = group.users_dynamic.count()
         return True
     msg = "Administrator group needs to contain at least one user"
     request.session.flash(msg, 'warning')
     return False
 
 
 @view_config(route_name='groups_property',
              match_param='key=users', request_method="POST",
              renderer='json', permission='root_administration')
 def groups_users_add(request):
     """
     Get list of permissions assigned to specific resources
     """
     group = GroupService.by_id(request.matchdict.get('group_id'))
-    user = User.by_user_name(request.unsafe_json_body.get('user_name'))
+    user = UserService.by_user_name(request.unsafe_json_body.get('user_name'))
     if not user:
-        user = User.by_email(request.unsafe_json_body.get('user_name'))
+        user = UserService.by_email(request.unsafe_json_body.get('user_name'))
 
     if not group or not user:
         return HTTPNotFound()
     if user not in group.users:
         group.users.append(user)
         group.member_count = group.users_dynamic.count()
     props = ['user_name', 'id', 'first_name', 'last_name', 'email',
              'last_login_date', 'status']
     u_dict = user.get_dict(include_keys=props)
-    u_dict['gravatar_url'] = user.gravatar_url(s=20)
+    u_dict['gravatar_url'] = UserService.gravatar_url(user, s=20)
     return u_dict

backend/src/appenlight/views/index.py

@@ -1,241 +1,242 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import datetime
 import logging
 import uuid
 
 import pyramid.security as security
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPFound
 from pyramid.response import Response
 from pyramid.security import NO_PERMISSION_REQUIRED
 from ziggurat_foundations.ext.pyramid.sign_in import ZigguratSignInSuccess
 from ziggurat_foundations.ext.pyramid.sign_in import ZigguratSignInBadAuth
 from ziggurat_foundations.ext.pyramid.sign_in import ZigguratSignOut
+from ziggurat_foundations.models.services.user import UserService
 
 from appenlight.lib.social import handle_social_data
 from appenlight.models import DBSession
 from appenlight.models.user import User
 from appenlight.models.services.user import UserService
 from appenlight.subscribers import _
 from appenlight import forms
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 
 @view_config(context=ZigguratSignInSuccess, permission=NO_PERMISSION_REQUIRED)
 def sign_in(request):
     """
     Performs sign in by sending proper user identification headers
     Regenerates CSRF token
     """
     user = request.context.user
     if user.status == 1:
         request.session.new_csrf_token()
         user.last_login_date = datetime.datetime.utcnow()
         social_data = request.session.get('zigg.social_auth')
         if social_data:
             handle_social_data(request, user, social_data)
     else:
         request.session.flash(_('Account got disabled'))
 
     if request.context.came_from != '/':
         return HTTPFound(location=request.context.came_from,
                          headers=request.context.headers)
     else:
         return HTTPFound(location=request.route_url('/'),
                          headers=request.context.headers)
 
 
 @view_config(context=ZigguratSignInBadAuth, permission=NO_PERMISSION_REQUIRED)
 def bad_auth(request):
     """
     Handles incorrect login flow
     """
     request.session.flash(_('Incorrect username or password'), 'warning')
     return HTTPFound(location=request.route_url('register'),
                      headers=request.context.headers)
 
 
 @view_config(context=ZigguratSignOut, permission=NO_PERMISSION_REQUIRED)
 def sign_out(request):
     """
     Removes user identification cookie
     """
     return HTTPFound(location=request.route_url('register'),
                      headers=request.context.headers)
 
 
 @view_config(route_name='lost_password',
              renderer='appenlight:templates/user/lost_password.jinja2',
              permission=NO_PERMISSION_REQUIRED)
 def lost_password(request):
     """
     Presents lost password page - sends password reset link to
     specified email address.
     This link is valid only for 10 minutes
     """
     form = forms.LostPasswordForm(request.POST, csrf_context=request)
     if request.method == 'POST' and form.validate():
-        user = User.by_email(form.email.data)
+        user = UserService.by_email(form.email.data)
         if user:
-            user.regenerate_security_code()
+            UserService.regenerate_security_code(user)
             user.security_code_date = datetime.datetime.utcnow()
             email_vars = {
                 'user': user,
                 'request': request,
                 'email_title': "AppEnlight :: New password request"
             }
             UserService.send_email(
                 request, recipients=[user.email],
                 variables=email_vars,
                 template='/email_templates/lost_password.jinja2')
             msg = 'Password reset email had been sent. ' \
                   'Please check your mailbox for further instructions.'
             request.session.flash(_(msg))
             return HTTPFound(location=request.route_url('lost_password'))
     return {"form": form}
 
 
 @view_config(route_name='lost_password_generate',
              permission=NO_PERMISSION_REQUIRED,
              renderer='appenlight:templates/user/lost_password_generate.jinja2')
 def lost_password_generate(request):
     """
     Shows new password form - perform time check and set new password for user
     """
-    user = User.by_user_name_and_security_code(
+    user = UserService.by_user_name_and_security_code(
         request.GET.get('user_name'), request.GET.get('security_code'))
     if user:
         delta = datetime.datetime.utcnow() - user.security_code_date
 
     if user and delta.total_seconds() < 600:
         form = forms.NewPasswordForm(request.POST, csrf_context=request)
         if request.method == "POST" and form.validate():
-            user.set_password(form.new_password.data)
+            UserService.set_password(user, form.new_password.data)
             request.session.flash(_('You can sign in with your new password.'))
             return HTTPFound(location=request.route_url('register'))
         else:
             return {"form": form}
     else:
         return Response('Security code expired')
 
 
 @view_config(route_name='register',
              renderer='appenlight:templates/user/register.jinja2',
              permission=NO_PERMISSION_REQUIRED)
 def register(request):
     """
     Render register page with form
     Also handles oAuth flow for registration
     """
     login_url = request.route_url('ziggurat.routes.sign_in')
     if request.query_string:
         query_string = '?%s' % request.query_string
     else:
         query_string = ''
     referrer = '%s%s' % (request.path, query_string)
 
     if referrer in [login_url, '/register', '/register?sign_in=1']:
         referrer = '/'  # never use the login form itself as came_from
     sign_in_form = forms.SignInForm(
         came_from=request.params.get('came_from', referrer),
         csrf_context=request)
 
     # populate form from oAuth session data returned by authomatic
     social_data = request.session.get('zigg.social_auth')
     if request.method != 'POST' and social_data:
         log.debug(social_data)
         user_name = social_data['user'].get('user_name', '').split('@')[0]
         form_data = {
             'user_name': user_name,
             'email': social_data['user'].get('email')
         }
         form_data['user_password'] = str(uuid.uuid4())
         form = forms.UserRegisterForm(MultiDict(form_data),
                                       csrf_context=request)
         form.user_password.widget.hide_value = False
     else:
         form = forms.UserRegisterForm(request.POST, csrf_context=request)
     if request.method == 'POST' and form.validate():
         log.info('registering user')
         # insert new user here
         if request.registry.settings['appenlight.disable_registration']:
             request.session.flash(_('Registration is currently disabled.'))
             return HTTPFound(location=request.route_url('/'))
 
         new_user = User()
         DBSession.add(new_user)
         form.populate_obj(new_user)
-        new_user.regenerate_security_code()
+        UserService.regenerate_security_code(new_user)
         new_user.status = 1
-        new_user.set_password(new_user.user_password)
+        UserService.set_password(new_user, new_user.user_password)
         new_user.registration_ip = request.environ.get('REMOTE_ADDR')
 
         if social_data:
             handle_social_data(request, new_user, social_data)
 
         email_vars = {'user': new_user,
                       'request': request,
                       'email_title': "AppEnlight :: Start information"}
         UserService.send_email(
             request, recipients=[new_user.email], variables=email_vars,
             template='/email_templates/registered.jinja2')
         request.session.flash(_('You have successfully registered.'))
         DBSession.flush()
         headers = security.remember(request, new_user.id)
         return HTTPFound(location=request.route_url('/'),
                          headers=headers)
     settings = request.registry.settings
     social_plugins = {}
     if settings.get('authomatic.pr.twitter.key', ''):
         social_plugins['twitter'] = True
     if settings.get('authomatic.pr.google.key', ''):
         social_plugins['google'] = True
     if settings.get('authomatic.pr.github.key', ''):
         social_plugins['github'] = True
     if settings.get('authomatic.pr.bitbucket.key', ''):
         social_plugins['bitbucket'] = True
 
     return {
         "form": form,
         "sign_in_form": sign_in_form,
         "social_plugins": social_plugins
     }
 
 
 @view_config(route_name='/',
              renderer='appenlight:templates/app.jinja2',
              permission=NO_PERMISSION_REQUIRED)
 @view_config(route_name='angular_app_ui',
              renderer='appenlight:templates/app.jinja2',
              permission=NO_PERMISSION_REQUIRED)
 @view_config(route_name='angular_app_ui_ix',
              renderer='appenlight:templates/app.jinja2',
              permission=NO_PERMISSION_REQUIRED)
 def app_main_index(request):
     """
     Render dashoard/report browser page page along with:
      - flash messages
      - application list
      - assigned reports
      - latest events
      (those last two come from subscribers.py that sets global renderer variables)
     """
     return {}

backend/src/appenlight/views/reports.py

@@ -1,294 +1,297 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import logging
 
 from datetime import datetime, timedelta
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPUnprocessableEntity
 
+from ziggurat_foundations.models.services.resource import ResourceService
+from ziggurat_foundations.models.services.user import UserService
+
 from appenlight.models import DBSession
 from appenlight.models.user import User
 from appenlight.models.report_comment import ReportComment
 from appenlight.models.report_assignment import ReportAssignment
 from appenlight.models.services.user import UserService
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight import forms
 from appenlight.lib.enums import ReportType
 from appenlight.lib.helpers import gen_pagination_headers
 from appenlight.lib.utils import build_filter_settings_from_query_dict
 from appenlight.validators import ReportSearchSchema, TagListSchema, \
     accepted_search_params
 from webob.multidict import MultiDict
 
 _ = str
 
 log = logging.getLogger(__name__)
 
 section_filters_key = 'appenlight:reports:filter:%s'
 
 
 @view_config(route_name='reports', renderer='json', permission='authenticated')
 @view_config(route_name='slow_reports', renderer='json',
              permission='authenticated')
 def index(request):
     """
     Returns list of report groups based on user search query
     """
     if request.user:
         request.user.last_login_date = datetime.utcnow()
 
-    applications = request.user.resources_with_perms(
-        ['view'], resource_types=['application'])
+    applications = UserService.resources_with_perms(
+        request.user, ['view'], resource_types=['application'])
 
     search_params = request.GET.mixed()
 
     all_possible_app_ids = set([app.resource_id for app in applications])
     schema = ReportSearchSchema().bind(resources=all_possible_app_ids)
     tag_schema = TagListSchema()
     filter_settings = schema.deserialize(search_params)
     tag_list = [{"name": k, "value": v} for k, v in filter_settings.items()
                 if k not in accepted_search_params]
     tags = tag_schema.deserialize(tag_list)
     filter_settings['tags'] = tags
     if request.matched_route.name == 'slow_reports':
         filter_settings['report_type'] = [ReportType.slow]
     else:
         filter_settings['report_type'] = [ReportType.error]
 
     reports_paginator = ReportGroupService.get_paginator_by_app_ids(
         app_ids=filter_settings['resource'],
         page=filter_settings['page'],
         filter_settings=filter_settings
     )
     reports = []
     include_keys = ('id', 'http_status', 'report_type', 'resource_name',
                     'front_url', 'resource_id', 'error', 'url_path', 'tags',
                     'duration')
     for report in reports_paginator.sa_items:
         reports.append(report.get_dict(request, include_keys=include_keys))
     headers = gen_pagination_headers(request, reports_paginator)
     request.response.headers.update(headers)
     return reports
 
 
 @view_config(route_name='report_groups', renderer='json', permission='view',
              request_method="GET")
 def view_report(request):
     """
     Show individual detailed report group along with latest report
     """
     report_group = request.context.report_group
     if not report_group.read:
         report_group.read = True
 
     report_id = request.params.get('reportId', request.params.get('report_id'))
     report_dict = report_group.get_report(report_id).get_dict(request,
                                                               details=True)
     # disallow browsing other occurences by anonymous
     if not request.user:
         report_dict.pop('group_next_report', None)
         report_dict.pop('group_previous_report', None)
     return report_dict
 
 
 @view_config(route_name='report_groups', renderer='json',
              permission='update_reports', request_method='DELETE')
 def remove(request):
     """
     Used to remove reourt groups from database
     """
     report = request.context.report_group
     form = forms.ReactorForm(request.POST, csrf_context=request)
     form.validate()
     DBSession.delete(report)
     return True
 
 
 @view_config(route_name='report_groups_property', match_param='key=comments',
              renderer='json', permission='view', request_method="POST")
 def comment_create(request):
     """
     Creates user comments for report group, sends email notifications
     of said comments
     """
     report_group = request.context.report_group
     application = request.context.resource
     form = forms.CommentForm(MultiDict(request.unsafe_json_body),
                              csrf_context=request)
     if request.method == 'POST' and form.validate():
         comment = ReportComment(owner_id=request.user.id,
                                 report_time=report_group.first_timestamp)
         form.populate_obj(comment)
         report_group.comments.append(comment)
-        perm_list = application.users_for_perm('view')
+        perm_list = ResourceService.users_for_perm(application, 'view')
         uids_to_notify = []
         users_to_notify = []
         for perm in perm_list:
             user = perm.user
             if ('@{}'.format(user.user_name) in comment.body and
                         user.id not in uids_to_notify):
                 uids_to_notify.append(user.id)
                 users_to_notify.append(user)
 
         commenters = ReportGroupService.users_commenting(
             report_group, exclude_user_id=request.user.id)
         for user in commenters:
             if user.id not in uids_to_notify:
                 uids_to_notify.append(user.id)
                 users_to_notify.append(user)
 
         for user in users_to_notify:
             email_vars = {'user': user,
                           'commenting_user': request.user,
                           'request': request,
                           'application': application,
                           'report_group': report_group,
                           'comment': comment,
                           'email_title': "AppEnlight :: New comment"}
             UserService.send_email(
                 request,
                 recipients=[user.email],
                 variables=email_vars,
                 template='/email_templates/new_comment_report.jinja2')
         request.session.flash(_('Your comment was created'))
         return comment.get_dict()
     else:
         return form.errors
 
 
 @view_config(route_name='report_groups_property',
              match_param='key=assigned_users', renderer='json',
              permission='update_reports', request_method="GET")
 def assigned_users(request):
     """
     Returns list of users a specific report group is assigned for review
     """
     report_group = request.context.report_group
     application = request.context.resource
-    users = set([p.user for p in application.users_for_perm('view')])
+    users = set([p.user for p in ResourceService.users_for_perm(application, 'view')])
     currently_assigned = [u.user_name for u in report_group.assigned_users]
     user_status = {'assigned': [], 'unassigned': []}
     # handle users
     for user in users:
         user_dict = {'user_name': user.user_name,
-                     'gravatar_url': user.gravatar_url(),
+                     'gravatar_url': UserService.gravatar_url(user),
                      'name': '%s %s' % (user.first_name, user.last_name,)}
         if user.user_name in currently_assigned:
             user_status['assigned'].append(user_dict)
         elif user_dict not in user_status['unassigned']:
             user_status['unassigned'].append(user_dict)
     return user_status
 
 
 @view_config(route_name='report_groups_property',
              match_param='key=assigned_users', renderer='json',
              permission='update_reports', request_method="PATCH")
 def assign_users(request):
     """
     Assigns specific report group to user for review - send email notification
     """
     report_group = request.context.report_group
     application = request.context.resource
     currently_assigned = [u.user_name for u in report_group.assigned_users]
     new_assigns = request.unsafe_json_body
 
     # first unassign old users
     for user_name in new_assigns['unassigned']:
         if user_name in currently_assigned:
-            user = User.by_user_name(user_name)
+            user = UserService.by_user_name(user_name)
             report_group.assigned_users.remove(user)
             comment = ReportComment(owner_id=request.user.id,
                                     report_time=report_group.first_timestamp)
             comment.body = 'Unassigned group from @%s' % user_name
             report_group.comments.append(comment)
 
     # assign new users
     for user_name in new_assigns['assigned']:
         if user_name not in currently_assigned:
-            user = User.by_user_name(user_name)
+            user = UserService.by_user_name(user_name)
             if user in report_group.assigned_users:
                 report_group.assigned_users.remove(user)
             DBSession.flush()
             assignment = ReportAssignment(
                 owner_id=user.id,
                 report_time=report_group.first_timestamp,
                 group_id=report_group.id)
             DBSession.add(assignment)
 
             comment = ReportComment(owner_id=request.user.id,
                                     report_time=report_group.first_timestamp)
             comment.body = 'Assigned report_group to @%s' % user_name
             report_group.comments.append(comment)
 
             email_vars = {'user': user,
                           'request': request,
                           'application': application,
                           'report_group': report_group,
                           'email_title': "AppEnlight :: Assigned Report"}
             UserService.send_email(request, recipients=[user.email],
                                    variables=email_vars,
                                    template='/email_templates/assigned_report.jinja2')
 
     return True
 
 
 @view_config(route_name='report_groups_property', match_param='key=history',
              renderer='json', permission='view')
 def history(request):
     """ Separate error graph or similar graph"""
     report_group = request.context.report_group
     query_params = request.GET.mixed()
     query_params['resource'] = (report_group.resource_id,)
 
     filter_settings = build_filter_settings_from_query_dict(request,
                                                             query_params)
     if not filter_settings.get('end_date'):
         end_date = datetime.utcnow().replace(microsecond=0, second=0)
         filter_settings['end_date'] = end_date
 
     if not filter_settings.get('start_date'):
         delta = timedelta(days=30)
         filter_settings['start_date'] = filter_settings['end_date'] - delta
 
     filter_settings['group_id'] = report_group.id
 
     result = ReportGroupService.get_report_stats(request, filter_settings)
 
     plot_data = []
     for row in result:
         point = {
             'x': row['x'],
             'reports': row['report'] + row['slow_report'] + row['not_found']}
         plot_data.append(point)
 
     return plot_data
 
 
 @view_config(route_name='report_groups', renderer='json',
              permission='update_reports', request_method="PATCH")
 def report_groups_PATCH(request):
     """
     Used to update the report group fixed status
     """
     report_group = request.context.report_group
     allowed_keys = ['public', 'fixed']
     for k, v in request.unsafe_json_body.items():
         if k in allowed_keys:
             setattr(report_group, k, v)
         else:
             return HTTPUnprocessableEntity()
     return report_group.get_dict(request)

backend/src/appenlight/views/tests.py

@@ -1,444 +1,446 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import copy
 import logging
 import datetime
 import time
 import random
 import redis
 import six
 import pyramid.renderers
 import requests
+
+from ziggurat_foundations.models.services.user import UserService
+
 import appenlight.celery.tasks
 from pyramid.view import view_config
 from pyramid_mailer.message import Message
 from appenlight_client.timing import time_trace
 from appenlight.models import DBSession, Datastores
 from appenlight.models.user import User
 from appenlight.models.report_group import ReportGroup
 from appenlight.models.event import Event
 from appenlight.models.services.report_group import ReportGroupService
 from appenlight.models.services.event import EventService
 from appenlight.lib.enums import ReportType
 
 log = logging.getLogger(__name__)
 
 GLOBAL_REQ = None
 
 
 @view_config(route_name='test', match_param='action=mail',
              renderer='string', permission='root_administration')
 def mail(request):
     """
     Test email communication
     """
     request.environ['HTTP_HOST'] = 'appenlight.com'
     request.environ['wsgi.url_scheme'] = 'https'
     renderer_vars = {"title": "You have just registered on AppEnlight",
                      "username": "test",
                      "email": "grzegżółka",
                      'firstname': 'dupa'}
     # return vars
     html = pyramid.renderers.render('/email_templates/registered.jinja2',
                                     renderer_vars,
                                     request=request)
     message = Message(subject="hello world %s" % random.randint(1, 9999),
                       sender="[e-mail]",
                       recipients=["[e-mail]"],
                       html=html)
     request.registry.mailer.send(message)
     return html
     return vars
 
 
 @view_config(route_name='test', match_param='action=alerting',
              renderer='appenlight:templates/tests/alerting.jinja2',
              permission='root_administration')
 def alerting_test(request):
     """
     Allows to test send data on various registered alerting channels
     """
-    applications = request.user.resources_with_perms(
-        ['view'], resource_types=['application'])
+    applications = UserService.resources_with_perms(request.user, ['view'], resource_types=['application'])
     # what we can select in total
     all_possible_app_ids = [app.resource_id for app in applications]
     resource = applications[0]
 
     alert_channels = []
     for channel in request.user.alert_channels:
         alert_channels.append(channel.get_dict())
 
     cname = request.params.get('channel_name')
     cvalue = request.params.get('channel_value')
     event_name = request.params.get('event_name')
     if cname and cvalue:
         for channel in request.user.alert_channels:
             if (channel.channel_value == cvalue and
                         channel.channel_name == cname):
                 break
         if event_name in ['error_report_alert', 'slow_report_alert']:
             # opened
             new_event = Event(resource_id=resource.resource_id,
                               event_type=Event.types[event_name],
                               start_date=datetime.datetime.utcnow(),
                               status=Event.statuses['active'],
                               values={'reports': 5,
                                       'threshold': 10}
                               )
             channel.notify_alert(resource=resource,
                                  event=new_event,
                                  user=request.user,
                                  request=request)
 
             # closed
             ev_type = Event.types[event_name.replace('open', 'close')]
             new_event = Event(resource_id=resource.resource_id,
                               event_type=ev_type,
                               start_date=datetime.datetime.utcnow(),
                               status=Event.statuses['closed'],
                               values={'reports': 5,
                                       'threshold': 10})
             channel.notify_alert(resource=resource,
                                  event=new_event,
                                  user=request.user,
                                  request=request)
         elif event_name == 'notify_reports':
             report = ReportGroupService.by_app_ids(all_possible_app_ids) \
                 .filter(ReportGroup.report_type == ReportType.error).first()
             confirmed_reports = [(5, report), (1, report)]
             channel.notify_reports(resource=resource,
                                    user=request.user,
                                    request=request,
                                    since_when=datetime.datetime.utcnow(),
                                    reports=confirmed_reports)
             confirmed_reports = [(5, report)]
             channel.notify_reports(resource=resource,
                                    user=request.user,
                                    request=request,
                                    since_when=datetime.datetime.utcnow(),
                                    reports=confirmed_reports)
         elif event_name == 'notify_uptime':
             new_event = Event(resource_id=resource.resource_id,
                               event_type=Event.types['uptime_alert'],
                               start_date=datetime.datetime.utcnow(),
                               status=Event.statuses['active'],
                               values={"status_code": 500,
                                       "tries": 2,
                                       "response_time": 0})
             channel.notify_uptime_alert(resource=resource,
                                         event=new_event,
                                         user=request.user,
                                         request=request)
         elif event_name == 'chart_alert':
             event = EventService.by_type_and_status(
                 event_types=(Event.types['chart_alert'],),
                 status_types=(Event.statuses['active'],)).first()
             channel.notify_chart_alert(resource=event.resource,
                                        event=event,
                                        user=request.user,
                                        request=request)
         elif event_name == 'daily_digest':
             since_when = datetime.datetime.utcnow() - datetime.timedelta(
                 hours=8)
             filter_settings = {'resource': [resource.resource_id],
                                'tags': [{'name': 'type',
                                          'value': ['error'], 'op': None}],
                                'type': 'error', 'start_date': since_when}
 
             reports = ReportGroupService.get_trending(
                 request, filter_settings=filter_settings, limit=50)
             channel.send_digest(resource=resource,
                                 user=request.user,
                                 request=request,
                                 since_when=datetime.datetime.utcnow(),
                                 reports=reports)
 
     return {'alert_channels': alert_channels,
             'applications': dict([(app.resource_id, app.resource_name)
                                   for app in applications.all()])}
 
 
 @view_config(route_name='test', match_param='action=error',
              renderer='string', permission='root_administration')
 def error(request):
     """
     Raises an internal error with some test data for testing purposes
     """
     request.environ['appenlight.message'] = 'test message'
     request.environ['appenlight.extra']['dupa'] = 'dupa'
     request.environ['appenlight.extra']['message'] = 'message'
     request.environ['appenlight.tags']['action'] = 'test_error'
     request.environ['appenlight.tags']['count'] = 5
     log.debug(chr(960))
     log.debug('debug')
     log.info(chr(960))
     log.info('INFO')
     log.warning('warning')
 
     @time_trace(name='error.foobar', min_duration=0.1)
     def fooobar():
         time.sleep(0.12)
         return 1
 
     fooobar()
 
     def foobar(somearg):
         raise Exception('test')
 
     client = redis.StrictRedis()
     client.setex('testval', 10, 'foo')
     request.environ['appenlight.force_send'] = 1
 
     # stats, result = get_local_storage(local_timing).get_thread_stats()
     # import pprint
     # pprint.pprint(stats)
     # pprint.pprint(result)
     # print 'entries', len(result)
     request.environ['appenlight.username'] = 'ErgO'
     raise Exception(chr(960) + '%s' % random.randint(1, 5))
     return {}
 
 
 @view_config(route_name='test', match_param='action=task',
              renderer='string', permission='root_administration')
 def test_task(request):
     """
     Test erroneous celery task
     """
     import appenlight.celery.tasks
 
     appenlight.celery.tasks.test_exception_task.delay()
     return 'task sent'
 
 
 @view_config(route_name='test', match_param='action=task_retry',
              renderer='string', permission='root_administration')
 def test_task_retry(request):
     """
     Test erroneous celery task
     """
     import appenlight.celery.tasks
 
     appenlight.celery.tasks.test_retry_exception_task.delay()
     return 'task sent'
 
 
 @view_config(route_name='test', match_param='action=celery_emails',
              renderer='string', permission='root_administration')
 def test_celery_emails(request):
     import appenlight.celery.tasks
     appenlight.celery.tasks.alerting.delay()
     return 'task sent'
 
 
 @view_config(route_name='test', match_param='action=daily_digest',
              renderer='string', permission='root_administration')
 def test_celery_daily_digest(request):
     import appenlight.celery.tasks
     appenlight.celery.tasks.daily_digest.delay()
     return 'task sent'
 
 
 @view_config(route_name='test', match_param='action=celery_alerting',
              renderer='string', permission='root_administration')
 def test_celery_alerting(request):
     import appenlight.celery.tasks
     appenlight.celery.tasks.alerting()
     return 'task sent'
 
 
 @view_config(route_name='test', match_param='action=logging',
              renderer='string', permission='root_administration')
 def logs(request):
     """
     Test some in-app logging
     """
     log.debug(chr(960))
     log.debug('debug')
     log.info(chr(960))
     log.info('INFO')
     log.warning('Matched GET /\xc4\x85\xc5\xbc\xc4\x87'
                 '\xc4\x99\xc4\x99\xc4\x85/summary')
     log.warning('XXXXMatched GET /\xc4\x85\xc5\xbc\xc4'
                 '\x87\xc4\x99\xc4\x99\xc4\x85/summary')
     log.warning('DUPA /ążćęęą')
     log.warning("g\u017ceg\u017c\u00f3\u0142ka")
     log.error('TEST Lorem ipsum2',
               extra={'user': 'ergo', 'commit': 'sog8ds0g7sdih12hh1j512h5k'})
     log.fatal('TEST Lorem ipsum3')
     log.warning('TEST Lorem ipsum',
                 extra={"action": 'purchase',
                        "price": random.random() * 100,
                        "quantity": random.randint(1, 99)})
     log.warning('test_pkey',
                 extra={"action": 'test_pkey', "price": random.random() * 100,
                        'ae_primary_key': 1,
                        "quantity": random.randint(1, 99)})
     log.warning('test_pkey2',
                 extra={"action": 'test_pkey', "price": random.random() * 100,
                        'ae_primary_key': 'b',
                        'ae_permanent': 't',
                        "quantity": random.randint(1, 99)})
     log.warning('test_pkey3',
                 extra={"action": 'test_pkey', "price": random.random() * 100,
                        'ae_primary_key': 1,
                        "quantity": random.randint(1, 99)})
     log.warning('test_pkey4',
                 extra={"action": 'test_pkey', "price": random.random() * 100,
                        'ae_primary_key': 'b',
                        'ae_permanent': True,
                        "quantity": random.randint(1, 99)})
     request.environ['appenlight.force_send'] = 1
     return {}
 
 
 @view_config(route_name='test', match_param='action=transaction',
              renderer='string', permission='root_administration')
 def transaction_test(request):
     """
     Test transactions
     """
     try:
         result = DBSession.execute("SELECT 1/0")
     except:
         request.tm.abort()
     result = DBSession.execute("SELECT 1")
     return 'OK'
 
 
 @view_config(route_name='test', match_param='action=slow_request',
              renderer='string', permission='root_administration')
 def slow_request(request):
     """
     Test a request that has some slow entries - including nested calls
     """
     users = DBSession.query(User).all()
     import random
     some_val = random.random()
     import threading
     t_id = id(threading.currentThread())
     log.warning('slow_log %s %s ' % (some_val, t_id))
     log.critical('tid %s' % t_id)
 
     @time_trace(name='baz_func %s' % some_val, min_duration=0.1)
     def baz(arg):
         time.sleep(0.32)
         return arg
 
     requests.get('http://ubuntu.com')
 
     @time_trace(name='foo_func %s %s' % (some_val, t_id), min_duration=0.1)
     def foo(arg):
         time.sleep(0.52)
         log.warning('foo_func %s %s' % (some_val, t_id))
         requests.get('http://ubuntu.com?test=%s' % some_val)
         return bar(arg)
 
     @time_trace(name='bar_func %s %s' % (some_val, t_id), min_duration=0.1)
     def bar(arg):
         log.warning('bar_func %s %s' % (some_val, t_id))
         time.sleep(1.52)
         baz(arg)
         baz(arg)
         return baz(arg)
 
     foo('a')
     return {}
 
 
 @view_config(route_name='test', match_param='action=styling',
              renderer='appenlight:templates/tests/styling.jinja2',
              permission='__no_permission_required__')
 def styling(request):
     """
     Some styling test page
     """
     _ = str
     request.session.flash(_(
         'Your password got updated. '
         'Next time log in with your new credentials.'))
     request.session.flash(_(
         'Something went wrong when we '
         'tried to authorize you via external provider'),
         'warning')
     request.session.flash(_(
         'Unfortunately there was a problem '
         'processing your payment, please try again later.'),
         'error')
     return {}
 
 
 @view_config(route_name='test', match_param='action=js_error',
              renderer='appenlight:templates/tests/js_error.jinja2',
              permission='__no_permission_required__')
 def js(request):
     """
     Used for testing javasctipt client for error catching
     """
     return {}
 
 
 @view_config(route_name='test', match_param='action=js_log',
              renderer='appenlight:templates/tests/js_log.jinja2',
              permission='__no_permission_required__')
 def js_log(request):
     """
     Used for testing javasctipt client for logging
     """
     return {}
 
 
 @view_config(route_name='test', match_param='action=log_requests',
              renderer='string',
              permission='__no_permission_required__')
 def log_requests(request):
     """
     Util view for printing json requests
     """
     return {}
 
 
 @view_config(route_name='test', match_param='action=url', renderer='string',
              permission='__no_permission_required__')
 def log_requests(request):
     """
     I have no fucking clue why I needed that ;-)
     """
     return request.route_url('reports', _app_url='https://appenlight.com')
 
 
 class TestClass(object):
     """
     Used to test if class-based view name resolution works correctly
     """
 
     def __init__(self, request):
         self.request = request
 
     @view_config(route_name='test', match_param='action=test_a',
                  renderer='string', permission='root_administration')
     @view_config(route_name='test', match_param='action=test_c',
                  renderer='string', permission='root_administration')
     @view_config(route_name='test', match_param='action=test_d',
                  renderer='string', permission='root_administration')
     def test_a(self):
         return 'ok'
 
     @view_config(route_name='test', match_param='action=test_b',
                  renderer='string', permission='root_administration')
     def test_b(self):
         return 'ok'

backend/src/appenlight/views/user.py

@@ -1,678 +1,678 b''
 # -*- coding: utf-8 -*-
 
 # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 import colander
 import datetime
 import json
 import logging
 import uuid
 import pyramid.security as security
 import appenlight.lib.helpers as h
 
 from authomatic.adapters import WebObAdapter
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPFound, HTTPUnprocessableEntity
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest
 from pyramid.security import NO_PERMISSION_REQUIRED
 from ziggurat_foundations.models.services.external_identity import \
     ExternalIdentityService
+from ziggurat_foundations.models.services.user import UserService
 
 from appenlight.lib import generate_random_string
 from appenlight.lib.social import handle_social_data
 from appenlight.lib.utils import channelstream_request, add_cors_headers, \
     permission_tuple_to_dict
 from appenlight.models import DBSession
 from appenlight.models.alert_channels.email import EmailAlertChannel
 from appenlight.models.alert_channel_action import AlertChannelAction
 from appenlight.models.services.alert_channel import AlertChannelService
 from appenlight.models.services.alert_channel_action import \
     AlertChannelActionService
 from appenlight.models.auth_token import AuthToken
 from appenlight.models.report import REPORT_TYPE_MATRIX
 from appenlight.models.user import User
 from appenlight.models.services.user import UserService
 from appenlight.subscribers import _
 from appenlight.validators import build_rule_schema
 from appenlight import forms
 from webob.multidict import MultiDict
 
 log = logging.getLogger(__name__)
 
 
 @view_config(route_name='users_no_id', renderer='json',
              request_method="GET", permission='root_administration')
 def users_list(request):
     """
     Returns users list
     """
     props = ['user_name', 'id', 'first_name', 'last_name', 'email',
              'last_login_date', 'status']
     users = UserService.all()
     users_dicts = []
     for user in users:
         u_dict = user.get_dict(include_keys=props)
-        u_dict['gravatar_url'] = user.gravatar_url(s=20)
+        u_dict['gravatar_url'] = UserService.gravatar_url(user, s=20)
         users_dicts.append(u_dict)
     return users_dicts
 
 
 @view_config(route_name='users_no_id', renderer='json',
              request_method="POST", permission='root_administration')
 def users_create(request):
     """
     Returns users list
     """
     form = forms.UserCreateForm(MultiDict(request.safe_json_body or {}),
                                 csrf_context=request)
     if form.validate():
         log.info('registering user')
         # probably not needed in the future since this requires root anyways
         # lets keep this here in case we lower view permission in the future
         # if request.registry.settings['appenlight.disable_registration']:
         #     return HTTPUnprocessableEntity(body={'error': 'Registration is currently disabled.'})
         user = User()
         # insert new user here
         DBSession.add(user)
         form.populate_obj(user)
-        user.regenerate_security_code()
-        user.set_password(user.user_password)
+        UserService.regenerate_security_code(user)
+        UserService.set_password(user, user.user_password)
         user.status = 1 if form.status.data else 0
         request.session.flash(_('User created'))
         DBSession.flush()
         return user.get_dict(exclude_keys=['security_code_date', 'notes',
                                            'security_code', 'user_password'])
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
 
 
 @view_config(route_name='users', renderer='json',
              request_method="GET", permission='root_administration')
 @view_config(route_name='users', renderer='json',
              request_method="PATCH", permission='root_administration')
 def users_update(request):
     """
     Updates user object
     """
-    user = User.by_id(request.matchdict.get('user_id'))
+    user = UserService.by_id(request.matchdict.get('user_id'))
     if not user:
         return HTTPNotFound()
     post_data = request.safe_json_body or {}
     if request.method == 'PATCH':
         form = forms.UserUpdateForm(MultiDict(post_data),
                                     csrf_context=request)
         if form.validate():
             form.populate_obj(user, ignore_none=True)
             if form.user_password.data:
-                user.set_password(user.user_password)
+                UserService.set_password(user, user.user_password)
             if form.status.data:
                 user.status = 1
             else:
                 user.status = 0
         else:
             return HTTPUnprocessableEntity(body=form.errors_json)
     return user.get_dict(exclude_keys=['security_code_date', 'notes',
                                        'security_code', 'user_password'])
 
 
 @view_config(route_name='users_property',
              match_param='key=resource_permissions',
              renderer='json', permission='authenticated')
 def users_resource_permissions_list(request):
     """
     Get list of permissions assigned to specific resources
     """
-    user = User.by_id(request.matchdict.get('user_id'))
+    user = UserService.by_id(request.matchdict.get('user_id'))
     if not user:
         return HTTPNotFound()
     return [permission_tuple_to_dict(perm) for perm in
-            user.resources_with_possible_perms()]
+            UserService.resources_with_possible_perms(user)]
 
 
 @view_config(route_name='users', renderer='json',
              request_method="DELETE", permission='root_administration')
 def users_DELETE(request):
     """
     Removes a user permanently from db - makes a check to see if after the
     operation there will be at least one admin left
     """
     msg = _('There needs to be at least one administrator in the system')
-    user = User.by_id(request.matchdict.get('user_id'))
+    user = UserService.by_id(request.matchdict.get('user_id'))
     if user:
-        users = User.users_for_perms(['root_administration']).all()
+        users = UserService.users_for_perms(['root_administration']).all()
         if len(users) < 2 and user.id == users[0].id:
             request.session.flash(msg, 'warning')
         else:
             DBSession.delete(user)
             request.session.flash(_('User removed'))
             return True
     request.response.status = 422
     return False
 
 
 @view_config(route_name='users_self', renderer='json',
              request_method="GET", permission='authenticated')
 @view_config(route_name='users_self', renderer='json',
              request_method="PATCH", permission='authenticated')
 def users_self(request):
     """
     Updates user personal information
     """
 
     if request.method == 'PATCH':
         form = forms.gen_user_profile_form()(
             MultiDict(request.unsafe_json_body),
             csrf_context=request)
         if form.validate():
             form.populate_obj(request.user)
             request.session.flash(_('Your profile got updated.'))
         else:
             return HTTPUnprocessableEntity(body=form.errors_json)
     return request.user.get_dict(
         exclude_keys=['security_code_date', 'notes', 'security_code',
                       'user_password'],
         extended_info=True)
 
 
 @view_config(route_name='users_self_property',
              match_param='key=external_identities', renderer='json',
              request_method='GET', permission='authenticated')
 def users_external_identies(request):
     user = request.user
     identities = [{'provider': ident.provider_name,
                    'id': ident.external_user_name} for ident
                   in user.external_identities.all()]
     return identities
 
 
 @view_config(route_name='users_self_property',
              match_param='key=external_identities', renderer='json',
              request_method='DELETE', permission='authenticated')
 def users_external_identies_DELETE(request):
     """
     Unbinds external identities(google,twitter etc.) from user account
     """
     user = request.user
     for identity in user.external_identities.all():
         log.info('found identity %s' % identity)
         if (identity.provider_name == request.params.get('provider') and
                     identity.external_user_name == request.params.get('id')):
             log.info('remove identity %s' % identity)
             DBSession.delete(identity)
             return True
     return False
 
 
 @view_config(route_name='users_self_property',
              match_param='key=password', renderer='json',
              request_method='PATCH', permission='authenticated')
 def users_password(request):
     """
     Sets new password for user account
     """
     user = request.user
     form = forms.ChangePasswordForm(MultiDict(request.unsafe_json_body),
                                     csrf_context=request)
     form.old_password.user = user
     if form.validate():
-        user.regenerate_security_code()
-        user.set_password(form.new_password.data)
+        UserService.regenerate_security_code(user)
+        UserService.set_password(user, form.new_password.data)
         msg = 'Your password got updated. ' \
               'Next time log in with your new credentials.'
         request.session.flash(_(msg))
         return True
     else:
         return HTTPUnprocessableEntity(body=form.errors_json)
     return False
 
 
 @view_config(route_name='users_self_property', match_param='key=websocket',
              renderer='json', permission='authenticated')
 def users_websocket(request):
     """
     Handle authorization of users trying to connect
     """
     # handle preflight request
     user = request.user
     if request.method == 'OPTIONS':
         res = request.response.body('OK')
         add_cors_headers(res)
         return res
-    applications = user.resources_with_perms(
-        ['view'], resource_types=['application'])
+    applications = UserService.resources_with_perms(user, ['view'], resource_types=['application'])
     channels = ['app_%s' % app.resource_id for app in applications]
     payload = {"username": user.user_name,
                "conn_id": str(uuid.uuid4()),
                "channels": channels
                }
     settings = request.registry.settings
     response = channelstream_request(
         settings['cometd.secret'], '/connect', payload,
         servers=[request.registry.settings['cometd_servers']],
         throw_exceptions=True)
     return payload
 
 
 @view_config(route_name='users_self_property', request_method="GET",
              match_param='key=alert_channels', renderer='json',
              permission='authenticated')
 def alert_channels(request):
     """
     Lists all available alert channels
     """
     user = request.user
     return [c.get_dict(extended_info=True) for c in user.alert_channels]
 
 
 @view_config(route_name='users_self_property', match_param='key=alert_actions',
              request_method="GET", renderer='json', permission='authenticated')
 def alert_actions(request):
     """
     Lists all available alert channels
     """
     user = request.user
     return [r.get_dict(extended_info=True) for r in user.alert_actions]
 
 
 @view_config(route_name='users_self_property', renderer='json',
              match_param='key=alert_channels_rules', request_method='POST',
              permission='authenticated')
 def alert_channels_rule_POST(request):
     """
     Creates new notification rule for specific alert channel
     """
     user = request.user
     alert_action = AlertChannelAction(owner_id=request.user.id,
                                       type='report')
     DBSession.add(alert_action)
     DBSession.flush()
     return alert_action.get_dict()
 
 
 @view_config(route_name='users_self_property', permission='authenticated',
              match_param='key=alert_channels_rules',
              renderer='json', request_method='DELETE')
 def alert_channels_rule_DELETE(request):
     """
     Removes specific alert channel rule
     """
     user = request.user
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
         user.id,
         request.GET.get('pkey'))
     if rule_action:
         DBSession.delete(rule_action)
         return True
     return HTTPNotFound()
 
 
 @view_config(route_name='users_self_property', permission='authenticated',
              match_param='key=alert_channels_rules',
              renderer='json', request_method='PATCH')
 def alert_channels_rule_PATCH(request):
     """
     Removes specific alert channel rule
     """
     user = request.user
     json_body = request.unsafe_json_body
 
     schema = build_rule_schema(json_body['rule'], REPORT_TYPE_MATRIX)
     try:
         schema.deserialize(json_body['rule'])
     except colander.Invalid as exc:
         return HTTPUnprocessableEntity(body=json.dumps(exc.asdict()))
 
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
         user.id,
         request.GET.get('pkey'))
 
     if rule_action:
         rule_action.rule = json_body['rule']
         rule_action.resource_id = json_body['resource_id']
         rule_action.action = json_body['action']
         return rule_action.get_dict()
     return HTTPNotFound()
 
 
 @view_config(route_name='users_self_property', permission='authenticated',
              match_param='key=alert_channels',
              renderer='json', request_method='PATCH')
 def alert_channels_PATCH(request):
     user = request.user
     channel_name = request.GET.get('channel_name')
     channel_value = request.GET.get('channel_value')
     # iterate over channels
     channel = None
     for channel in user.alert_channels:
         if (channel.channel_name == channel_name and
                     channel.channel_value == channel_value):
             break
     if not channel:
         return HTTPNotFound()
 
     allowed_keys = ['daily_digest', 'send_alerts']
     for k, v in request.unsafe_json_body.items():
         if k in allowed_keys:
             setattr(channel, k, v)
         else:
             return HTTPBadRequest()
     return channel.get_dict()
 
 
 @view_config(route_name='users_self_property', permission='authenticated',
              match_param='key=alert_channels',
              request_method="POST", renderer='json')
 def alert_channels_POST(request):
     """
     Creates a new email alert channel for user, sends a validation email
     """
     user = request.user
     form = forms.EmailChannelCreateForm(MultiDict(request.unsafe_json_body),
                                         csrf_context=request)
     if not form.validate():
         return HTTPUnprocessableEntity(body=form.errors_json)
 
     email = form.email.data.strip()
     channel = EmailAlertChannel()
     channel.channel_name = 'email'
     channel.channel_value = email
     security_code = generate_random_string(10)
     channel.channel_json_conf = {'security_code': security_code}
     user.alert_channels.append(channel)
 
     email_vars = {'user': user,
                   'email': email,
                   'request': request,
                   'security_code': security_code,
                   'email_title': "AppEnlight :: "
                                  "Please authorize your email"}
 
     UserService.send_email(request, recipients=[email],
                            variables=email_vars,
                            template='/email_templates/authorize_email.jinja2')
     request.session.flash(_('Your alert channel was '
                             'added to the system.'))
     request.session.flash(
         _('You need to authorize your email channel, a message was '
           'sent containing necessary information.'),
         'warning')
     DBSession.flush()
     channel.get_dict()
 
 
 @view_config(route_name='section_view',
              match_param=['section=user_section',
                           'view=alert_channels_authorize'],
              renderer='string', permission='authenticated')
 def alert_channels_authorize(request):
     """
     Performs alert channel authorization based on auth code sent in email
     """
     user = request.user
     for channel in user.alert_channels:
         security_code = request.params.get('security_code', '')
         if channel.channel_json_conf['security_code'] == security_code:
             channel.channel_validated = True
             request.session.flash(_('Your email was authorized.'))
     return HTTPFound(location=request.route_url('/'))
 
 
 @view_config(route_name='users_self_property', request_method="DELETE",
              match_param='key=alert_channels', renderer='json',
              permission='authenticated')
 def alert_channel_DELETE(request):
     """
     Removes alert channel from users channel
     """
     user = request.user
     channel = None
     for chan in user.alert_channels:
         if (chan.channel_name == request.params.get('channel_name') and
                     chan.channel_value == request.params.get('channel_value')):
             channel = chan
             break
     if channel:
         user.alert_channels.remove(channel)
         request.session.flash(_('Your channel was removed.'))
         return True
     return False
 
 
 @view_config(route_name='users_self_property', permission='authenticated',
              match_param='key=alert_channels_actions_binds',
              renderer='json', request_method="POST")
 def alert_channels_actions_binds_POST(request):
     """
     Adds alert action to users channels
     """
     user = request.user
     json_body = request.unsafe_json_body
     channel = AlertChannelService.by_owner_id_and_pkey(
         user.id,
         json_body.get('channel_pkey'))
 
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
         user.id,
         json_body.get('action_pkey'))
 
     if channel and rule_action:
         if channel.pkey not in [c.pkey for c in rule_action.channels]:
             rule_action.channels.append(channel)
             return rule_action.get_dict(extended_info=True)
     return HTTPUnprocessableEntity()
 
 
 @view_config(route_name='users_self_property', request_method="DELETE",
              match_param='key=alert_channels_actions_binds',
              renderer='json', permission='authenticated')
 def alert_channels_actions_binds_DELETE(request):
     """
     Removes alert action from users channels
     """
     user = request.user
     channel = AlertChannelService.by_owner_id_and_pkey(
         user.id,
         request.GET.get('channel_pkey'))
 
     rule_action = AlertChannelActionService.by_owner_id_and_pkey(
         user.id,
         request.GET.get('action_pkey'))
 
     if channel and rule_action:
         if channel.pkey in [c.pkey for c in rule_action.channels]:
             rule_action.channels.remove(channel)
             return rule_action.get_dict(extended_info=True)
     return HTTPUnprocessableEntity()
 
 
 @view_config(route_name='social_auth_abort',
              renderer='string', permission=NO_PERMISSION_REQUIRED)
 def oauth_abort(request):
     """
     Handles problems with authorization via velruse
     """
 
 
 @view_config(route_name='social_auth', permission=NO_PERMISSION_REQUIRED)
 def social_auth(request):
     # Get the internal provider name URL variable.
     provider_name = request.matchdict.get('provider')
 
     # Start the login procedure.
     adapter = WebObAdapter(request, request.response)
     result = request.authomatic.login(adapter, provider_name)
     if result:
         if result.error:
             return handle_auth_error(request, result)
         elif result.user:
             return handle_auth_success(request, result)
     return request.response
 
 
 def handle_auth_error(request, result):
     # Login procedure finished with an error.
     request.session.pop('zigg.social_auth', None)
     request.session.flash(_('Something went wrong when we tried to '
                             'authorize you via external provider. '
                             'Please try again.'), 'warning')
 
     return HTTPFound(location=request.route_url('/'))
 
 
 def handle_auth_success(request, result):
     # Hooray, we have the user!
     # OAuth 2.0 and OAuth 1.0a provide only limited user data on login,
     # We need to update the user to get more info.
     if result.user:
         result.user.update()
 
     social_data = {
         'user': {'data': result.user.data},
         'credentials': result.user.credentials
     }
     # normalize data
     social_data['user']['id'] = result.user.id
     user_name = result.user.username or ''
     # use email name as username for google
     if (social_data['credentials'].provider_name == 'google' and
             result.user.email):
         user_name = result.user.email
     social_data['user']['user_name'] = user_name
     social_data['user']['email'] = result.user.email or ''
 
     request.session['zigg.social_auth'] = social_data
     # user is logged so bind his external identity with account
     if request.user:
         handle_social_data(request, request.user, social_data)
         request.session.pop('zigg.social_auth', None)
         return HTTPFound(location=request.route_url('/'))
     else:
         user = ExternalIdentityService.user_by_external_id_and_provider(
             social_data['user']['id'],
             social_data['credentials'].provider_name
         )
         # fix legacy accounts with wrong google ID
         if not user and social_data['credentials'].provider_name == 'google':
             user = ExternalIdentityService.user_by_external_id_and_provider(
                 social_data['user']['email'],
                 social_data['credentials'].provider_name)
 
         # user tokens are already found in our db
         if user:
             handle_social_data(request, user, social_data)
             headers = security.remember(request, user.id)
             request.session.pop('zigg.social_auth', None)
             return HTTPFound(location=request.route_url('/'), headers=headers)
         else:
             msg = 'You need to finish registration ' \
                   'process to bind your external identity to your account ' \
                   'or sign in to existing account'
             request.session.flash(msg)
             return HTTPFound(location=request.route_url('register'))
 
 
 @view_config(route_name='section_view', permission='authenticated',
              match_param=['section=users_section', 'view=search_users'],
              renderer='json')
 def search_users(request):
     """
     Returns a list of users for autocomplete
     """
     user = request.user
     items_returned = []
     like_condition = request.params.get('user_name', '') + '%'
     # first append used if email is passed
-    found_user = User.by_email(request.params.get('user_name', ''))
+    found_user = UserService.by_email(request.params.get('user_name', ''))
     if found_user:
         name = '{} {}'.format(found_user.first_name, found_user.last_name)
         items_returned.append({'user': found_user.user_name, 'name': name})
-    for found_user in User.user_names_like(like_condition).limit(20):
+    for found_user in UserService.user_names_like(like_condition).limit(20):
         name = '{} {}'.format(found_user.first_name, found_user.last_name)
         items_returned.append({'user': found_user.user_name, 'name': name})
     return items_returned
 
 
 @view_config(route_name='users_self_property', match_param='key=auth_tokens',
              request_method="GET", renderer='json', permission='authenticated')
 @view_config(route_name='users_property', match_param='key=auth_tokens',
              request_method="GET", renderer='json', permission='authenticated')
 def auth_tokens_list(request):
     """
     Lists all available alert channels
     """
     if request.matched_route.name == 'users_self_property':
         user = request.user
     else:
-        user = User.by_id(request.matchdict.get('user_id'))
+        user = UserService.by_id(request.matchdict.get('user_id'))
         if not user:
             return HTTPNotFound()
     return [c.get_dict() for c in user.auth_tokens]
 
 
 @view_config(route_name='users_self_property', match_param='key=auth_tokens',
              request_method="POST", renderer='json',
              permission='authenticated')
 @view_config(route_name='users_property', match_param='key=auth_tokens',
              request_method="POST", renderer='json',
              permission='authenticated')
 def auth_tokens_POST(request):
     """
     Lists all available alert channels
     """
     if request.matched_route.name == 'users_self_property':
         user = request.user
     else:
-        user = User.by_id(request.matchdict.get('user_id'))
+        user = UserService.by_id(request.matchdict.get('user_id'))
         if not user:
             return HTTPNotFound()
 
     req_data = request.safe_json_body or {}
     if not req_data.get('expires'):
         req_data.pop('expires', None)
     form = forms.AuthTokenCreateForm(MultiDict(req_data), csrf_context=request)
     if not form.validate():
         return HTTPUnprocessableEntity(body=form.errors_json)
     token = AuthToken()
     form.populate_obj(token)
     if token.expires:
         interval = h.time_deltas.get(token.expires)['delta']
         token.expires = datetime.datetime.utcnow() + interval
     user.auth_tokens.append(token)
     DBSession.flush()
     return token.get_dict()
 
 
 @view_config(route_name='users_self_property', match_param='key=auth_tokens',
              request_method="DELETE", renderer='json',
              permission='authenticated')
 @view_config(route_name='users_property', match_param='key=auth_tokens',
              request_method="DELETE", renderer='json',
              permission='authenticated')
 def auth_tokens_DELETE(request):
     """
     Lists all available alert channels
     """
     if request.matched_route.name == 'users_self_property':
         user = request.user
     else:
-        user = User.by_id(request.matchdict.get('user_id'))
+        user = UserService.by_id(request.matchdict.get('user_id'))
         if not user:
             return HTTPNotFound()
 
     for token in user.auth_tokens:
         if token.token == request.params.get('token'):
             user.auth_tokens.remove(token)
             return True
     return False