project: initial commit

.coveragerc

@@ -0,0 +1,3 b''
+[run]
+branch = True
+omit = appenlight/tests/*

.hgignore

@@ -0,0 +1,154 b''
+# Created by .ignore support plugin (hsz.mobi)
+syntax: glob
+
+### Example user template template
+### Example user template
+
+# IntelliJ project files
+.idea
+*.iml
+out
+gen### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask instance folder
+instance/
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+
+syntax: regexp
+^\.idea$
+syntax: regexp
+^\.settings$
+syntax: regexp
+^data$
+syntax: regexp
+^webassets$
+syntax: regexp
+^dist$
+syntax: regexp
+^\.project$
+syntax: regexp
+^\.pydevproject$
+syntax: regexp
+^private$
+syntax: regexp
+^appenlight_frontend/build$
+syntax: regexp
+^appenlight_frontend/bower_components$
+syntax: regexp
+^appenlight_frontend/node_modules$
+^src/node_modules$
+syntax: regexp
+^\.pydevproject$
+syntax: regexp
+appenlight\.egg-info$
+syntax: regexp
+\.pyc$
+syntax: regexp
+\celerybeat.*
+syntax: regexp
+\.iml$
+syntax: regexp
+^frontend/build$
+syntax: regexp
+^frontend/bower_components$
+syntax: regexp
+^frontend/node_modules$
+^frontend/src/node_modules$
+^frontend/build$
+
+syntax: regexp
+\.db$
+
+syntax: regexp
+packer_cache
+
+syntax: regexp
+packer/packer
+
+syntax: regexp
+install_appenlight_production.yaml

LICENSE.txt

@@ -0,0 +1,704 b''
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License, version 3
+(only), as published by the Free Software Foundation.
+
+
+This program incorporates work covered by the following copyright and
+permission notice:
+
+   Copyright (c) 2014-2016 - packaging
+       file:
+   Copyright (c) 2008-2011 - msgpack-python
+       file:licenses/msgpack_license.txt
+   Copyright (c) 2007-2008 - amqp
+       file:licenses/amqp_license.txt
+   Copyright (c) 2013 - bcrypt
+       file:licenses/bcrypt_license.txt
+   Copyright (c) 2015 - elasticsearch
+       file:licenses/elasticsearch_license.txt
+   Copyright (c) 2011-2013 - gevent-websocket
+       file:licenses/gevent_websocket_license.txt
+   Copyright (c) 2015 - python-editor
+       file:licenses/python_editor_license.txt
+   Copyright (c) 2015 - requests
+       file:licenses/requests_license.txt
+   Copyright (c) 2014 - requests-toolbelt
+       file:licenses/requests_toolbelt_license.txt
+
+Both licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+imitations under the License.
+
+
+Below is the full text of GNU Affero General Public License, version 3
+
+
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.

alembic.ini

@@ -0,0 +1,62 b''
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = appenlight:migrations
+version_table = alembic_appenlight_version
+
+appenlight_config_file = development.ini
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+sqlalchemy.url = postgresql://test:test@localhost/appenlight
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

backend/CHANGES.txt

@@ -0,0 +1,4 b''
+0.0
+---
+
+-  Initial version

backend/MANIFEST.in

@@ -0,0 +1,2 b''
+include *.txt *.ini *.cfg *.rst *.md
+recursive-include appenlight *.ico *.png *.css *.gif *.jpg *.pt *.txt *.mak *.mako *.js *.html *.xml *.jinja2 *.rst *.otf *.ttf *.svg *.woff *.eot

backend/README.md

@@ -0,0 +1,39 b''
+# appenlight README
+
+
+To run the app you need to have meet prerequusites:
+
+- running elasticsearch (2.2 tested)
+- running postgresql 9.5
+- running redis
+
+# Setup basics
+
+Set up the basic application database schema:
+
+    appenlight_initialize_db config.ini
+
+Set up basic elasticsearch schema:
+
+    appenlight-reindex-elasticsearch -c config.ini -t all
+
+
+# Running
+
+To run the application itself:
+
+    pserve --reload development.ini
+
+To run celery queue processing:
+
+    celery worker -A appenlight.celery -Q "reports,logs,metrics,default" --ini=development.ini
+
+
+# Testing
+
+To run test suite:
+
+    py.test appenlight/tests/tests.py --cov appenlight (this looks for testing.ini in repo root)
+
+WARNING!!!
+Some tests will insert data into elasticsearch or redis based on testing.ini

backend/requirements.txt

@@ -0,0 +1,49 b''
+repoze.sendmail==4.1
+pyramid==1.7
+pyramid_tm==0.12
+pyramid_debugtoolbar
+pyramid_authstack==1.0.1
+SQLAlchemy==1.0.12
+alembic==0.8.6
+webhelpers2==2.0
+transaction==1.4.3
+zope.sqlalchemy==0.7.6
+pyramid_mailer==0.14.1
+redis==2.10.5
+redlock-py==1.0.8
+pyramid_jinja2==2.6.2
+psycopg2==2.6.1
+wtforms==2.1
+celery==3.1.23
+formencode==1.3.0
+psutil==2.1.2
+ziggurat_foundations>=0.6.7
+bcrypt==2.0.0
+appenlight_client
+markdown==2.5
+colander==1.2
+defusedxml==0.4.1
+dogpile.cache==0.5.7
+pyramid_redis_sessions==1.0.1
+simplejson==3.8.2
+waitress==0.9.0
+gunicorn==19.4.5
+requests==2.9.1
+requests_oauthlib==0.6.1
+gevent==1.1.1
+gevent-websocket==0.9.5
+pygments==2.1.3
+lxml==3.6.0
+paginate==0.5.4
+paginate-sqlalchemy==0.2.0
+pyelasticsearch==1.4
+six==1.8.0
+mock==1.0.1
+itsdangerous==0.24
+camplight==0.9.6
+jira==0.41
+python-dateutil==2.5.3
+authomatic==0.1.0.post1
+cryptography==1.2.3
+webassets==0.11.1
+

backend/setup.py

@@ -0,0 +1,60 b''
+import os
+import sys
+import re
+
+from setuptools import setup, find_packages
+
+here = os.path.abspath(os.path.dirname(__file__))
+README = open(os.path.join(here, 'README.md')).read()
+CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()
+
+REQUIREMENTS = open(os.path.join(here, 'requirements.txt')).readlines()
+
+compiled = re.compile('([^=><]*).*')
+
+
+def parse_req(req):
+    return compiled.search(req).group(1).strip()
+
+
+requires = [_f for _f in map(parse_req, REQUIREMENTS) if _f]
+
+if sys.version_info[:3] < (2, 5, 0):
+    requires.append('pysqlite')
+
+found_packages = find_packages('src')
+found_packages.append('appenlight.migrations.versions')
+setup(name='appenlight',
+      version='0.1',
+      description='appenlight',
+      long_description=README + '\n\n' + CHANGES,
+      classifiers=[
+          "Programming Language :: Python",
+          "Framework :: Pylons",
+          "Topic :: Internet :: WWW/HTTP",
+          "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
+      ],
+      author='',
+      author_email='',
+      url='',
+      keywords='web wsgi bfg pylons pyramid',
+      package_dir={'': 'src'},
+      packages=found_packages,
+      include_package_data=True,
+      zip_safe=False,
+      test_suite='appenlight',
+      install_requires=requires,
+      entry_points={
+          'paste.app_factory': [
+              'main = appenlight:main'
+          ],
+          'console_scripts': [
+              'appenlight-cleanup = appenlight.scripts.cleanup:main',
+              'appenlight-initializedb = appenlight.scripts.initialize_db:main',
+              'appenlight-migratedb = appenlight.scripts.migratedb:main',
+              'appenlight-reindex-elasticsearch = appenlight.scripts.reindex_elasticsearch:main',
+              'appenlight-static = appenlight.scripts.static:main',
+              'appenlight-make-config = appenlight.scripts.make_config:main',
+          ]
+      }
+      )

backend/src/appenlight/__init__.py

@@ -0,0 +1,285 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import datetime
+import logging
+import pyelasticsearch
+import redis
+import os
+from pkg_resources import iter_entry_points
+
+import appenlight.lib.jinja2_filters as jinja2_filters
+import appenlight.lib.encryption as encryption
+
+from authomatic.providers import oauth2, oauth1
+from authomatic import Authomatic
+from pyramid.config import Configurator, PHASE3_CONFIG
+from pyramid.authentication import AuthTktAuthenticationPolicy
+from pyramid.authorization import ACLAuthorizationPolicy
+from pyramid_mailer.mailer import Mailer
+from pyramid.renderers import JSON
+from pyramid_redis_sessions import session_factory_from_settings
+from pyramid.settings import asbool, aslist
+from pyramid.security import AllPermissionsList
+from pyramid_authstack import AuthenticationStackPolicy
+from redlock import Redlock
+from sqlalchemy import engine_from_config
+
+from appenlight.celery import configure_celery
+from appenlight.lib import cache_regions
+from appenlight.lib.ext_json import json
+from appenlight.security import groupfinder, AuthTokenAuthenticationPolicy
+
+json_renderer = JSON(serializer=json.dumps, indent=4)
+
+log = logging.getLogger(__name__)
+
+
+def datetime_adapter(obj, request):
+    return obj.isoformat()
+
+
+def all_permissions_adapter(obj, request):
+    return '__all_permissions__'
+
+
+json_renderer.add_adapter(datetime.datetime, datetime_adapter)
+json_renderer.add_adapter(AllPermissionsList, all_permissions_adapter)
+
+
+def main(global_config, **settings):
+    """ This function returns a Pyramid WSGI application.
+    """
+    auth_tkt_policy = AuthTktAuthenticationPolicy(
+        settings['authtkt.secret'],
+        hashalg='sha512',
+        callback=groupfinder,
+        max_age=2592000,
+        secure=asbool(settings.get('authtkt.secure', 'false')))
+    auth_token_policy = AuthTokenAuthenticationPolicy(
+        callback=groupfinder
+    )
+    authorization_policy = ACLAuthorizationPolicy()
+    authentication_policy = AuthenticationStackPolicy()
+    authentication_policy.add_policy('auth_tkt', auth_tkt_policy)
+    authentication_policy.add_policy('auth_token', auth_token_policy)
+    # set crypto key
+    encryption.ENCRYPTION_SECRET = settings.get('encryption_secret')
+    # import this later so encyption key can be monkeypatched
+    from appenlight.models import DBSession, register_datastores
+    # update config with cometd info
+    settings['cometd_servers'] = {'server': settings['cometd.server'],
+                                  'secret': settings['cometd.secret']}
+
+    # Create the Pyramid Configurator.
+    settings['_mail_url'] = settings['mailing.app_url']
+    config = Configurator(settings=settings,
+                          authentication_policy=authentication_policy,
+                          authorization_policy=authorization_policy,
+                          root_factory='appenlight.security.RootFactory',
+                          default_permission='view')
+    config.set_default_csrf_options(require_csrf=True, header='X-XSRF-TOKEN')
+    config.add_view_deriver('appenlight.predicates.csrf_view',
+                            name='csrf_view')
+
+
+    # later, when config is available
+    dogpile_config = {'url': settings['redis.url'],
+                      "redis_expiration_time": 86400,
+                      "redis_distributed_lock": True}
+    cache_regions.regions = cache_regions.CacheRegions(dogpile_config)
+    config.registry.cache_regions = cache_regions.regions
+    engine = engine_from_config(settings, 'sqlalchemy.',
+                                json_serializer=json.dumps)
+    DBSession.configure(bind=engine)
+
+    # json rederer that serializes datetime
+    config.add_renderer('json', json_renderer)
+    config.set_request_property('appenlight.lib.request.es_conn', 'es_conn')
+    config.set_request_property('appenlight.lib.request.get_user', 'user',
+                                reify=True)
+    config.set_request_property('appenlight.lib.request.get_csrf_token',
+                                'csrf_token', reify=True)
+    config.set_request_property('appenlight.lib.request.safe_json_body',
+                                'safe_json_body', reify=True)
+    config.set_request_property('appenlight.lib.request.unsafe_json_body',
+                                'unsafe_json_body', reify=True)
+    config.add_request_method('appenlight.lib.request.add_flash_to_headers',
+                              'add_flash_to_headers')
+
+    config.include('pyramid_redis_sessions')
+    config.include('pyramid_tm')
+    config.include('pyramid_jinja2')
+    config.include('appenlight_client.ext.pyramid_tween')
+    config.include('ziggurat_foundations.ext.pyramid.sign_in')
+    config.registry.es_conn = pyelasticsearch.ElasticSearch(
+        settings['elasticsearch.nodes'])
+    config.registry.redis_conn = redis.StrictRedis.from_url(
+        settings['redis.url'])
+
+    config.registry.redis_lockmgr = Redlock([settings['redis.redlock.url'], ],
+                                            retry_count=0, retry_delay=0)
+    # mailer
+    config.registry.mailer = Mailer.from_settings(settings)
+
+    # Configure sessions
+    session_factory = session_factory_from_settings(settings)
+    config.set_session_factory(session_factory)
+
+    # Configure renderers and event subscribers
+    config.add_jinja2_extension('jinja2.ext.loopcontrols')
+    config.add_jinja2_search_path('appenlight:templates')
+    # event subscribers
+    config.add_subscriber("appenlight.subscribers.application_created",
+                          "pyramid.events.ApplicationCreated")
+    config.add_subscriber("appenlight.subscribers.add_renderer_globals",
+                          "pyramid.events.BeforeRender")
+    config.add_subscriber('appenlight.subscribers.new_request',
+                          'pyramid.events.NewRequest')
+    config.add_view_predicate('context_type_class',
+                              'appenlight.predicates.contextTypeClass')
+
+    register_datastores(es_conn=config.registry.es_conn,
+                        redis_conn=config.registry.redis_conn,
+                        redis_lockmgr=config.registry.redis_lockmgr)
+
+    # base stuff and scan
+
+    # need to ensure webassets exists otherwise config.override_asset()
+    # throws exception
+    if not os.path.exists(settings['webassets.dir']):
+        os.mkdir(settings['webassets.dir'])
+    config.add_static_view(path='appenlight:webassets',
+                           name='static', cache_max_age=3600)
+    config.override_asset(to_override='appenlight:webassets/',
+                          override_with=settings['webassets.dir'])
+
+    config.include('appenlight.views')
+    config.include('appenlight.views.admin')
+    config.scan(ignore=['appenlight.migrations',
+                        'appenlight.scripts',
+                        'appenlight.tests'])
+
+    # authomatic social auth
+    authomatic_conf = {
+        # callback http://yourapp.com/social_auth/twitter
+        'twitter': {
+            'class_': oauth1.Twitter,
+            'consumer_key': settings.get('authomatic.pr.twitter.key', 'X'),
+            'consumer_secret': settings.get('authomatic.pr.twitter.secret',
+                                            'X'),
+        },
+        # callback http://yourapp.com/social_auth/facebook
+        'facebook': {
+            'class_': oauth2.Facebook,
+            'consumer_key': settings.get('authomatic.pr.facebook.app_id', 'X'),
+            'consumer_secret': settings.get('authomatic.pr.facebook.secret',
+                                            'X'),
+            'scope': ['email'],
+        },
+        # callback http://yourapp.com/social_auth/google
+        'google': {
+            'class_': oauth2.Google,
+            'consumer_key': settings.get('authomatic.pr.google.key', 'X'),
+            'consumer_secret': settings.get(
+                'authomatic.pr.google.secret', 'X'),
+            'scope': ['profile', 'email'],
+        },
+        'github': {
+            'class_': oauth2.GitHub,
+            'consumer_key': settings.get('authomatic.pr.github.key', 'X'),
+            'consumer_secret': settings.get(
+                'authomatic.pr.github.secret', 'X'),
+            'scope': ['repo', 'public_repo', 'user:email'],
+            'access_headers': {'User-Agent': 'AppEnlight'},
+        },
+        'bitbucket': {
+            'class_': oauth1.Bitbucket,
+            'consumer_key': settings.get('authomatic.pr.bitbucket.key', 'X'),
+            'consumer_secret': settings.get(
+                'authomatic.pr.bitbucket.secret', 'X')
+        }
+    }
+    config.registry.authomatic = Authomatic(
+        config=authomatic_conf, secret=settings['authomatic.secret'])
+
+    # resource type information
+    config.registry.resource_types = ['resource', 'application']
+
+    # plugin information
+    config.registry.appenlight_plugins = {}
+
+    def register_appenlight_plugin(config, plugin_name, plugin_config):
+        def register():
+            log.warning('Registering plugin: {}'.format(plugin_name))
+            if plugin_name not in config.registry.appenlight_plugins:
+                config.registry.appenlight_plugins[plugin_name] = {
+                    'javascript': None,
+                    'static': None,
+                    'css': None,
+                    'top_nav': None,
+                    'celery_tasks': None,
+                    'celery_beats': None,
+                    'fulltext_indexer': None,
+                    'sqlalchemy_migrations': None,
+                    'default_values_setter': None,
+                    'resource_types': [],
+                    'url_gen': None
+                }
+            config.registry.appenlight_plugins[plugin_name].update(
+                plugin_config)
+            # inform AE what kind of resource types we have available
+            # so we can avoid failing when a plugin is removed but data
+            # is still present in the db
+            if plugin_config.get('resource_types'):
+                config.registry.resource_types.extend(
+                    plugin_config['resource_types'])
+
+        config.action('appenlight_plugin={}'.format(plugin_name), register)
+
+    config.add_directive('register_appenlight_plugin',
+                         register_appenlight_plugin)
+
+    for entry_point in iter_entry_points(group='appenlight.plugins'):
+        plugin = entry_point.load()
+        plugin.includeme(config)
+
+    # include other appenlight plugins explictly if needed
+    includes = aslist(settings.get('appenlight.includes', []))
+    for inc in includes:
+        config.include(inc)
+
+    # run this after everything registers in configurator
+
+    def pre_commit():
+        jinja_env = config.get_jinja2_environment()
+        jinja_env.filters['tojson'] = json.dumps
+        jinja_env.filters['toJSONUnsafe'] = jinja2_filters.toJSONUnsafe
+
+    config.action(None, pre_commit, order=PHASE3_CONFIG + 999)
+
+    def wrap_config_celery():
+        configure_celery(config.registry)
+
+    config.action(None, wrap_config_celery, order=PHASE3_CONFIG + 999)
+
+    app = config.make_wsgi_app()
+    return app

backend/src/appenlight/celery/__init__.py

@@ -0,0 +1,161 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+from datetime import timedelta
+from celery import Celery
+from celery.bin import Option
+from celery.schedules import crontab
+from celery.signals import worker_init, task_revoked, user_preload_options
+from celery.signals import task_prerun, task_retry, task_failure, task_success
+from kombu.serialization import register
+from pyramid.paster import bootstrap
+from pyramid.request import Request
+from pyramid.scripting import prepare
+from pyramid.settings import asbool
+from pyramid.threadlocal import get_current_request
+
+from appenlight.celery.encoders import json_dumps, json_loads
+from appenlight_client.ext.celery import register_signals
+
+log = logging.getLogger(__name__)
+
+register('date_json', json_dumps, json_loads,
+         content_type='application/x-date_json',
+         content_encoding='utf-8')
+
+celery = Celery()
+
+celery.user_options['preload'].add(
+    Option('--ini', dest='ini', default=None,
+           help='Specifies pyramid configuration file location.')
+)
+
+@user_preload_options.connect
+def on_preload_parsed(options, **kwargs):
+    """
+    This actually configures celery from pyramid config file
+    """
+    celery.conf['INI_PYRAMID'] = options['ini']
+    import appenlight_client.client as e_client
+    ini_location = options['ini']
+    if not ini_location:
+        raise Exception('You need to pass pyramid ini location using '
+                        '--ini=filename.ini argument to the worker')
+    env = bootstrap(ini_location)
+    api_key = env['request'].registry.settings['appenlight.api_key']
+    tr_config = env['request'].registry.settings.get(
+        'appenlight.transport_config')
+    CONFIG = e_client.get_config({'appenlight.api_key': api_key})
+    if tr_config:
+        CONFIG['appenlight.transport_config'] = tr_config
+    APPENLIGHT_CLIENT = e_client.Client(CONFIG)
+    # log.addHandler(APPENLIGHT_CLIENT.log_handler)
+    register_signals(APPENLIGHT_CLIENT)
+    celery.pyramid = env
+
+
+celery_config = {
+    'CELERY_IMPORTS': ["appenlight.celery.tasks",],
+    'CELERYD_TASK_TIME_LIMIT': 60,
+    'CELERYD_MAX_TASKS_PER_CHILD': 1000,
+    'CELERY_IGNORE_RESULT': True,
+    'CELERY_ACCEPT_CONTENT': ['date_json'],
+    'CELERY_TASK_SERIALIZER': 'date_json',
+    'CELERY_RESULT_SERIALIZER': 'date_json',
+    'BROKER_URL': None,
+    'CELERYD_CONCURRENCY': None,
+    'CELERY_TIMEZONE': None,
+    'CELERYBEAT_SCHEDULE': {
+        'alerting': {
+            'task': 'appenlight.celery.tasks.alerting',
+            'schedule': timedelta(seconds=60)
+        },
+        'daily_digest': {
+            'task': 'appenlight.celery.tasks.daily_digest',
+            'schedule': crontab(minute=1, hour='4,12,20')
+        },
+    }
+}
+celery.config_from_object(celery_config)
+
+def configure_celery(pyramid_registry):
+    settings = pyramid_registry.settings
+    celery_config['BROKER_URL'] = settings['celery.broker_url']
+    celery_config['CELERYD_CONCURRENCY'] = settings['celery.concurrency']
+    celery_config['CELERY_TIMEZONE'] = settings['celery.timezone']
+    if asbool(settings.get('celery.always_eager')):
+        celery_config['CELERY_ALWAYS_EAGER'] = True
+        celery_config['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
+
+    for plugin in pyramid_registry.appenlight_plugins.values():
+        if plugin.get('celery_tasks'):
+            celery_config['CELERY_IMPORTS'].extend(plugin['celery_tasks'])
+        if plugin.get('celery_beats'):
+            for name, config in plugin['celery_beats']:
+                celery_config['CELERYBEAT_SCHEDULE'][name] = config
+    celery.config_from_object(celery_config)
+
+
+@task_prerun.connect
+def task_prerun_signal(task_id, task, args, kwargs, **kwaargs):
+    if hasattr(celery, 'pyramid'):
+        env = celery.pyramid
+        env = prepare(registry=env['request'].registry)
+        proper_base_url = env['request'].registry.settings['mailing.app_url']
+        tmp_request = Request.blank('/', base_url=proper_base_url)
+        # ensure tasks generate url for right domain from config
+        env['request'].environ['HTTP_HOST'] = tmp_request.environ['HTTP_HOST']
+        env['request'].environ['SERVER_PORT'] = tmp_request.environ['SERVER_PORT']
+        env['request'].environ['SERVER_NAME'] = tmp_request.environ['SERVER_NAME']
+        env['request'].environ['wsgi.url_scheme'] = tmp_request.environ[
+            'wsgi.url_scheme']
+    get_current_request().tm.begin()
+
+
+@task_success.connect
+def task_success_signal(result, **kwargs):
+    get_current_request().tm.commit()
+    if hasattr(celery, 'pyramid'):
+        celery.pyramid["closer"]()
+
+
+@task_retry.connect
+def task_retry_signal(request, reason, einfo, **kwargs):
+    get_current_request().tm.abort()
+    if hasattr(celery, 'pyramid'):
+        celery.pyramid["closer"]()
+
+
+@task_failure.connect
+def task_failure_signal(task_id, exception, args, kwargs, traceback, einfo,
+                        **kwaargs):
+    get_current_request().tm.abort()
+    if hasattr(celery, 'pyramid'):
+        celery.pyramid["closer"]()
+
+
+@task_revoked.connect
+def task_revoked_signal(request, terminated, signum, expired, **kwaargs):
+    get_current_request().tm.abort()
+    if hasattr(celery, 'pyramid'):
+        celery.pyramid["closer"]()

backend/src/appenlight/celery/encoders.py

@@ -0,0 +1,65 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import json
+from datetime import datetime, date, timedelta
+
+DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+
+
+class DateEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, datetime):
+            return {
+                '__type__': '__datetime__',
+                'iso': obj.strftime(DATE_FORMAT)
+            }
+        elif isinstance(obj, date):
+            return {
+                '__type__': '__date__',
+                'iso': obj.strftime(DATE_FORMAT)
+            }
+        elif isinstance(obj, timedelta):
+            return {
+                '__type__': '__timedelta__',
+                'seconds': obj.total_seconds()
+            }
+        else:
+            return json.JSONEncoder.default(self, obj)
+
+
+def date_decoder(dct):
+    if '__type__' in dct:
+        if dct['__type__'] == '__datetime__':
+            return datetime.strptime(dct['iso'], DATE_FORMAT)
+        elif dct['__type__'] == '__date__':
+            return datetime.strptime(dct['iso'], DATE_FORMAT).date()
+        elif dct['__type__'] == '__timedelta__':
+            return timedelta(seconds=dct['seconds'])
+    return dct
+
+
+def json_dumps(obj):
+    return json.dumps(obj, cls=DateEncoder)
+
+
+def json_loads(obj):
+    return json.loads(obj.decode('utf8'), object_hook=date_decoder)

backend/src/appenlight/celery/tasks.py

@@ -0,0 +1,610 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import bisect
+import collections
+import math
+from datetime import datetime, timedelta
+
+import sqlalchemy as sa
+import pyelasticsearch
+
+from celery.utils.log import get_task_logger
+from zope.sqlalchemy import mark_changed
+from pyramid.threadlocal import get_current_request, get_current_registry
+from appenlight.celery import celery
+from appenlight.models.report_group import ReportGroup
+from appenlight.models import DBSession, Datastores
+from appenlight.models.report import Report
+from appenlight.models.log import Log
+from appenlight.models.request_metric import Metric
+from appenlight.models.event import Event
+
+from appenlight.models.services.application import ApplicationService
+from appenlight.models.services.event import EventService
+from appenlight.models.services.log import LogService
+from appenlight.models.services.report import ReportService
+from appenlight.models.services.report_group import ReportGroupService
+from appenlight.models.services.user import UserService
+from appenlight.models.tag import Tag
+from appenlight.lib import print_traceback
+from appenlight.lib.utils import parse_proto, in_batches
+from appenlight.lib.ext_json import json
+from appenlight.lib.redis_keys import REDIS_KEYS
+from appenlight.lib.enums import ReportType
+
+log = get_task_logger(__name__)
+
+sample_boundries = list(range(100, 10000, 100))
+
+
+def pick_sample(total_occurences, report_type=1):
+    every = 1.0
+    position = bisect.bisect_left(sample_boundries, total_occurences)
+    if position > 0:
+        # 404
+        if report_type == 2:
+            divide = 10.0
+        else:
+            divide = 100.0
+        every = sample_boundries[position - 1] / divide
+    return total_occurences % every == 0
+
+
[e-mail](queue="default", default_retry_delay=1, max_retries=2)
+def test_exception_task():
+    log.error('test celery log', extra={'location': 'celery'})
+    log.warning('test celery log', extra={'location': 'celery'})
+    raise Exception('Celery exception test')
+
+
[e-mail](queue="default", default_retry_delay=1, max_retries=2)
+def test_retry_exception_task():
+    try:
+        import time
+
+        time.sleep(1.3)
+        log.error('test retry celery log', extra={'location': 'celery'})
+        log.warning('test retry celery log', extra={'location': 'celery'})
+        raise Exception('Celery exception test')
+    except Exception as exc:
+        test_retry_exception_task.retry(exc=exc)
+
+
[e-mail](queue="reports", default_retry_delay=600, max_retries=999)
+def add_reports(resource_id, params, dataset, environ=None, **kwargs):
+    proto_version = parse_proto(params.get('protocol_version', ''))
+    current_time = datetime.utcnow().replace(second=0, microsecond=0)
+    try:
+        # we will store solr docs here for single insert
+        es_report_docs = {}
+        es_report_group_docs = {}
+        resource = ApplicationService.by_id(resource_id)
+        reports = []
+
+        if proto_version.major < 1 and proto_version.minor < 5:
+            for report_data in dataset:
+                report_details = report_data.get('report_details', [])
+                for i, detail_data in enumerate(report_details):
+                    report_data.update(detail_data)
+                    report_data.pop('report_details')
+                    traceback = report_data.get('traceback')
+                    if traceback is None:
+                        report_data['traceback'] = report_data.get('frameinfo')
+                    # for 0.3 api
+                    error = report_data.pop('error_type', '')
+                    if error:
+                        report_data['error'] = error
+                    if proto_version.minor < 4:
+                        # convert "Unknown" slow reports to
+                        # '' (from older clients)
+                        if (report_data['error'] and
+                                    report_data['http_status'] < 500):
+                            report_data['error'] = ''
+                    message = report_data.get('message')
+                    if 'extra' not in report_data:
+                        report_data['extra'] = []
+                    if message:
+                        report_data['extra'] = [('message', message), ]
+                    reports.append(report_data)
+        else:
+            reports = dataset
+
+        tags = []
+        es_slow_calls_docs = {}
+        es_reports_stats_rows = {}
+        for report_data in reports:
+            # build report details for later
+            added_details = 0
+            report = Report()
+            report.set_data(report_data, resource, proto_version)
+            report._skip_ft_index = True
+
+            report_group = ReportGroupService.by_hash_and_resource(
+                report.resource_id,
+                report.grouping_hash
+            )
+            occurences = report_data.get('occurences', 1)
+            if not report_group:
+                # total reports will be +1 moment later
+                report_group = ReportGroup(grouping_hash=report.grouping_hash,
+                                           occurences=0, total_reports=0,
+                                           last_report=0,
+                                           priority=report.priority,
+                                           error=report.error,
+                                           first_timestamp=report.start_time)
+                report_group._skip_ft_index = True
+                report_group.report_type = report.report_type
+            report.report_group_time = report_group.first_timestamp
+            add_sample = pick_sample(report_group.occurences,
+                                     report_type=report_group.report_type)
+            if add_sample:
+                resource.report_groups.append(report_group)
+                report_group.reports.append(report)
+                added_details += 1
+                DBSession.flush()
+                if report.partition_id not in es_report_docs:
+                    es_report_docs[report.partition_id] = []
+                es_report_docs[report.partition_id].append(report.es_doc())
+                tags.extend(list(report.tags.items()))
+                slow_calls = report.add_slow_calls(report_data, report_group)
+                DBSession.flush()
+                for s_call in slow_calls:
+                    if s_call.partition_id not in es_slow_calls_docs:
+                        es_slow_calls_docs[s_call.partition_id] = []
+                    es_slow_calls_docs[s_call.partition_id].append(
+                        s_call.es_doc())
+                    # try generating new stat rows if needed
+            else:
+                # required for postprocessing to not fail later
+                report.report_group = report_group
+
+            stat_row = ReportService.generate_stat_rows(
+                report, resource, report_group)
+            if stat_row.partition_id not in es_reports_stats_rows:
+                es_reports_stats_rows[stat_row.partition_id] = []
+            es_reports_stats_rows[stat_row.partition_id].append(
+                stat_row.es_doc())
+
+            # see if we should mark 10th occurence of report
+            last_occurences_10 = int(math.floor(report_group.occurences / 10))
+            curr_occurences_10 = int(math.floor(
+                (report_group.occurences + report.occurences) / 10))
+            last_occurences_100 = int(
+                math.floor(report_group.occurences / 100))
+            curr_occurences_100 = int(math.floor(
+                (report_group.occurences + report.occurences) / 100))
+            notify_occurences_10 = last_occurences_10 != curr_occurences_10
+            notify_occurences_100 = last_occurences_100 != curr_occurences_100
+            report_group.occurences = ReportGroup.occurences + occurences
+            report_group.last_timestamp = report.start_time
+            report_group.summed_duration = ReportGroup.summed_duration + report.duration
+            summed_duration = ReportGroup.summed_duration + report.duration
+            summed_occurences = ReportGroup.occurences + occurences
+            report_group.average_duration = summed_duration / summed_occurences
+            report_group.run_postprocessing(report)
+            if added_details:
+                report_group.total_reports = ReportGroup.total_reports + 1
+                report_group.last_report = report.id
+            report_group.set_notification_info(notify_10=notify_occurences_10,
+                                               notify_100=notify_occurences_100)
+            DBSession.flush()
+            report_group.get_report().notify_channel(report_group)
+            if report_group.partition_id not in es_report_group_docs:
+                es_report_group_docs[report_group.partition_id] = []
+            es_report_group_docs[report_group.partition_id].append(
+                report_group.es_doc())
+
+            action = 'REPORT'
+            log_msg = '%s: %s %s, client: %s, proto: %s' % (
+                action,
+                report_data.get('http_status', 'unknown'),
+                str(resource),
+                report_data.get('client'),
+                proto_version)
+            log.info(log_msg)
+        total_reports = len(dataset)
+        key = REDIS_KEYS['counters']['reports_per_minute'].format(current_time)
+        Datastores.redis.incr(key, total_reports)
+        Datastores.redis.expire(key, 3600 * 24)
+        key = REDIS_KEYS['counters']['reports_per_minute_per_app'].format(
+            resource_id, current_time)
+        Datastores.redis.incr(key, total_reports)
+        Datastores.redis.expire(key, 3600 * 24)
+
+        add_reports_es(es_report_group_docs, es_report_docs)
+        add_reports_slow_calls_es(es_slow_calls_docs)
+        add_reports_stats_rows_es(es_reports_stats_rows)
+        return True
+    except Exception as exc:
+        print_traceback(log)
+        add_reports.retry(exc=exc)
+
+
[e-mail](queue="es", default_retry_delay=600, max_retries=999)
+def add_reports_es(report_group_docs, report_docs):
+    for k, v in report_group_docs.items():
+        Datastores.es.bulk_index(k, 'report_group', v, id_field="_id")
+    for k, v in report_docs.items():
+        Datastores.es.bulk_index(k, 'report', v, id_field="_id",
+                                 parent_field='_parent')
+
+
[e-mail](queue="es", default_retry_delay=600, max_retries=999)
+def add_reports_slow_calls_es(es_docs):
+    for k, v in es_docs.items():
+        Datastores.es.bulk_index(k, 'log', v)
+
+
[e-mail](queue="es", default_retry_delay=600, max_retries=999)
+def add_reports_stats_rows_es(es_docs):
+    for k, v in es_docs.items():
+        Datastores.es.bulk_index(k, 'log', v)
+
+
[e-mail](queue="logs", default_retry_delay=600, max_retries=999)
+def add_logs(resource_id, request, dataset, environ=None, **kwargs):
+    proto_version = request.get('protocol_version')
+    current_time = datetime.utcnow().replace(second=0, microsecond=0)
+
+    try:
+        es_docs = collections.defaultdict(list)
+        application = ApplicationService.by_id(resource_id)
+        ns_pairs = []
+        for entry in dataset:
+            # gather pk and ns so we can remove older versions of row later
+            if entry['primary_key'] is not None:
+                ns_pairs.append({"pk": entry['primary_key'],
+                                 "ns": entry['namespace']})
+            log_entry = Log()
+            log_entry.set_data(entry, resource=application)
+            log_entry._skip_ft_index = True
+            application.logs.append(log_entry)
+            DBSession.flush()
+            # insert non pk rows first
+            if entry['primary_key'] is None:
+                es_docs[log_entry.partition_id].append(log_entry.es_doc())
+
+        # 2nd pass to delete all log entries from db foe same pk/ns pair
+        if ns_pairs:
+            ids_to_delete = []
+            es_docs = collections.defaultdict(list)
+            es_docs_to_delete = collections.defaultdict(list)
+            found_pkey_logs = LogService.query_by_primary_key_and_namespace(
+                list_of_pairs=ns_pairs)
+            log_dict = {}
+            for log_entry in found_pkey_logs:
+                log_key = (log_entry.primary_key, log_entry.namespace)
+                if log_key not in log_dict:
+                    log_dict[log_key] = []
+                log_dict[log_key].append(log_entry)
+
+            for ns, entry_list in log_dict.items():
+                entry_list = sorted(entry_list, key=lambda x: x.timestamp)
+                # newest row needs to be indexed in es
+                log_entry = entry_list[-1]
+                # delete everything from pg and ES, leave the last row in pg
+                for e in entry_list[:-1]:
+                    ids_to_delete.append(e.log_id)
+                    es_docs_to_delete[e.partition_id].append(e.delete_hash)
+
+                es_docs_to_delete[log_entry.partition_id].append(
+                    log_entry.delete_hash)
+
+                es_docs[log_entry.partition_id].append(log_entry.es_doc())
+
+            if ids_to_delete:
+                query = DBSession.query(Log).filter(
+                    Log.log_id.in_(ids_to_delete))
+                query.delete(synchronize_session=False)
+            if es_docs_to_delete:
+                # batch this to avoid problems with default ES bulk limits
+                for es_index in es_docs_to_delete.keys():
+                    for batch in in_batches(es_docs_to_delete[es_index], 20):
+                        query = {'terms': {'delete_hash': batch}}
+
+                        try:
+                            Datastores.es.delete_by_query(
+                                es_index, 'log', query)
+                        except pyelasticsearch.ElasticHttpNotFoundError as exc:
+                            log.error(exc)
+
+        total_logs = len(dataset)
+
+        log_msg = 'LOG_NEW: %s, entries: %s, proto:%s' % (
+            str(application),
+            total_logs,
+            proto_version)
+        log.info(log_msg)
+        # mark_changed(session)
+        key = REDIS_KEYS['counters']['logs_per_minute'].format(current_time)
+        Datastores.redis.incr(key, total_logs)
+        Datastores.redis.expire(key, 3600 * 24)
+        key = REDIS_KEYS['counters']['logs_per_minute_per_app'].format(
+            resource_id, current_time)
+        Datastores.redis.incr(key, total_logs)
+        Datastores.redis.expire(key, 3600 * 24)
+        add_logs_es(es_docs)
+        return True
+    except Exception as exc:
+        print_traceback(log)
+        add_logs.retry(exc=exc)
+
+
[e-mail](queue="es", default_retry_delay=600, max_retries=999)
+def add_logs_es(es_docs):
+    for k, v in es_docs.items():
+        Datastores.es.bulk_index(k, 'log', v)
+
+
[e-mail](queue="metrics", default_retry_delay=600, max_retries=999)
+def add_metrics(resource_id, request, dataset, proto_version):
+    current_time = datetime.utcnow().replace(second=0, microsecond=0)
+    try:
+        application = ApplicationService.by_id_cached()(resource_id)
+        application = DBSession.merge(application, load=False)
+        es_docs = []
+        rows = []
+        for metric in dataset:
+            tags = dict(metric['tags'])
+            server_n = tags.get('server_name', metric['server_name']).lower()
+            tags['server_name'] = server_n or 'unknown'
+            new_metric = Metric(
+                timestamp=metric['timestamp'],
+                resource_id=application.resource_id,
+                namespace=metric['namespace'],
+                tags=tags)
+            rows.append(new_metric)
+            es_docs.append(new_metric.es_doc())
+        session = DBSession()
+        session.bulk_save_objects(rows)
+        session.flush()
+
+        action = 'METRICS'
+        metrics_msg = '%s: %s, metrics: %s, proto:%s' % (
+            action,
+            str(application),
+            len(dataset),
+            proto_version
+        )
+        log.info(metrics_msg)
+
+        mark_changed(session)
+        key = REDIS_KEYS['counters']['metrics_per_minute'].format(current_time)
+        Datastores.redis.incr(key, len(rows))
+        Datastores.redis.expire(key, 3600 * 24)
+        key = REDIS_KEYS['counters']['metrics_per_minute_per_app'].format(
+            resource_id, current_time)
+        Datastores.redis.incr(key, len(rows))
+        Datastores.redis.expire(key, 3600 * 24)
+        add_metrics_es(es_docs)
+        return True
+    except Exception as exc:
+        print_traceback(log)
+        add_metrics.retry(exc=exc)
+
+
[e-mail](queue="es", default_retry_delay=600, max_retries=999)
+def add_metrics_es(es_docs):
+    for doc in es_docs:
+        partition = 'rcae_m_%s' % doc['timestamp'].strftime('%Y_%m_%d')
+        Datastores.es.index(partition, 'log', doc)
+
+
[e-mail](queue="default", default_retry_delay=5, max_retries=2)
+def check_user_report_notifications(resource_id, since_when=None):
+    try:
+        request = get_current_request()
+        application = ApplicationService.by_id(resource_id)
+        if not application:
+            return
+        error_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
+            ReportType.error, resource_id)
+        slow_key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
+            ReportType.slow, resource_id)
+        error_group_ids = Datastores.redis.smembers(error_key)
+        slow_group_ids = Datastores.redis.smembers(slow_key)
+        Datastores.redis.delete(error_key)
+        Datastores.redis.delete(slow_key)
+        err_gids = [int(g_id) for g_id in error_group_ids]
+        slow_gids = [int(g_id) for g_id in list(slow_group_ids)]
+        group_ids = err_gids + slow_gids
+        occurence_dict = {}
+        for g_id in group_ids:
+            key = REDIS_KEYS['counters']['report_group_occurences'].format(
+                g_id)
+            val = Datastores.redis.get(key)
+            Datastores.redis.delete(key)
+            if val:
+                occurence_dict[g_id] = int(val)
+            else:
+                occurence_dict[g_id] = 1
+        report_groups = ReportGroupService.by_ids(group_ids)
+        report_groups.options(sa.orm.joinedload(ReportGroup.last_report_ref))
+
+        ApplicationService.check_for_groups_alert(
+            application, 'alert', report_groups=report_groups,
+            occurence_dict=occurence_dict, since_when=since_when)
+        users = set([p.user for p in application.users_for_perm('view')])
+        report_groups = report_groups.all()
+        for user in users:
+            UserService.report_notify(user, request, application,
+                                      report_groups=report_groups,
+                                      occurence_dict=occurence_dict,
+                                      since_when=since_when)
+        for group in report_groups:
+            # marks report_groups as notified
+            if not group.notified:
+                group.notified = True
+    except Exception as exc:
+        print_traceback(log)
+        raise
+
+
[e-mail](queue="default", default_retry_delay=1, max_retries=2)
+def close_alerts(since_when=None):
+    log.warning('Checking alerts')
+    try:
+        event_types = [Event.types['error_report_alert'],
+                       Event.types['slow_report_alert'], ]
+        statuses = [Event.statuses['active']]
+        # get events older than 5 min
+        events = EventService.by_type_and_status(
+            event_types,
+            statuses,
+            older_than=(since_when - timedelta(minutes=5)))
+        for event in events:
+            # see if we can close them
+            event.validate_or_close(
+                since_when=(since_when - timedelta(minutes=1)))
+    except Exception as exc:
+        print_traceback(log)
+        raise
+
+
[e-mail](queue="default", default_retry_delay=600, max_retries=999)
+def update_tag_counter(tag_name, tag_value, count):
+    try:
+        query = DBSession.query(Tag).filter(Tag.name == tag_name).filter(
+            sa.cast(Tag.value, sa.types.TEXT) == sa.cast(json.dumps(tag_value),
+                                                         sa.types.TEXT))
+        query.update({'times_seen': Tag.times_seen + count,
+                      'last_timestamp': datetime.utcnow()},
+                     synchronize_session=False)
+        session = DBSession()
+        mark_changed(session)
+        return True
+    except Exception as exc:
+        print_traceback(log)
+        update_tag_counter.retry(exc=exc)
+
+
[e-mail](queue="default")
+def update_tag_counters():
+    """
+    Sets task to update counters for application tags
+    """
+    tags = Datastores.redis.lrange(REDIS_KEYS['seen_tag_list'], 0, -1)
+    Datastores.redis.delete(REDIS_KEYS['seen_tag_list'])
+    c = collections.Counter(tags)
+    for t_json, count in c.items():
+        tag_info = json.loads(t_json)
+        update_tag_counter.delay(tag_info[0], tag_info[1], count)
+
+
[e-mail](queue="default")
+def daily_digest():
+    """
+    Sends daily digest with top 50 error reports
+    """
+    request = get_current_request()
+    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
+    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
+    since_when = datetime.utcnow() - timedelta(hours=8)
+    log.warning('Generating daily digests')
+    for resource_id in apps:
+        resource_id = resource_id.decode('utf8')
+        end_date = datetime.utcnow().replace(microsecond=0, second=0)
+        filter_settings = {'resource': [resource_id],
+                           'tags': [{'name': 'type',
+                                     'value': ['error'], 'op': None}],
+                           'type': 'error', 'start_date': since_when,
+                           'end_date': end_date}
+
+        reports = ReportGroupService.get_trending(
+            request, filter_settings=filter_settings, limit=50)
+
+        application = ApplicationService.by_id(resource_id)
+        if application:
+            users = set([p.user for p in application.users_for_perm('view')])
+            for user in users:
+                user.send_digest(request, application, reports=reports,
+                                 since_when=since_when)
+
+
[e-mail](queue="default")
+def alerting():
+    """
+    Loop that checks redis for info and then issues new tasks to celery to
+    perform the following:
+    - which applications should have new alerts opened
+    - which currently opened alerts should be closed
+    """
+    start_time = datetime.utcnow()
+    # transactions are needed for mailer
+    apps = Datastores.redis.smembers(REDIS_KEYS['apps_that_had_reports'])
+    Datastores.redis.delete(REDIS_KEYS['apps_that_had_reports'])
+    for app in apps:
+        log.warning('Notify for app: %s' % app)
+        check_user_report_notifications.delay(app.decode('utf8'))
+    # clear app ids from set
+    close_alerts.delay(since_when=start_time)
+
+
[e-mail](queue="default", soft_time_limit=3600 * 4, hard_time_limit=3600 * 4,
+      max_retries=999)
+def logs_cleanup(resource_id, filter_settings):
+    request = get_current_request()
+    request.tm.begin()
+    es_query = {
+        "_source": False,
+        "size": 5000,
+        "query": {
+            "filtered": {
+                "filter": {
+                    "and": [{"term": {"resource_id": resource_id}}]
+                }
+            }
+        }
+    }
+
+    query = DBSession.query(Log).filter(Log.resource_id == resource_id)
+    if filter_settings['namespace']:
+        query = query.filter(Log.namespace == filter_settings['namespace'][0])
+        es_query['query']['filtered']['filter']['and'].append(
+            {"term": {"namespace": filter_settings['namespace'][0]}}
+        )
+    query.delete(synchronize_session=False)
+    request.tm.commit()
+    result = request.es_conn.search(es_query, index='rcae_l_*',
+                                    doc_type='log', es_scroll='1m',
+                                    es_search_type='scan')
+    scroll_id = result['_scroll_id']
+    while True:
+        log.warning('log_cleanup, app:{} ns:{} batch'.format(
+            resource_id,
+            filter_settings['namespace']
+        ))
+        es_docs_to_delete = []
+        result = request.es_conn.send_request(
+            'POST', ['_search', 'scroll'],
+            body=scroll_id, query_params={"scroll": '1m'})
+        scroll_id = result['_scroll_id']
+        if not result['hits']['hits']:
+            break
+        for doc in result['hits']['hits']:
+            es_docs_to_delete.append({"id": doc['_id'],
+                                      "index": doc['_index']})
+
+        for batch in in_batches(es_docs_to_delete, 10):
+            Datastores.es.bulk([Datastores.es.delete_op(doc_type='log',
+                                                        **to_del)
+                                for to_del in batch])

backend/src/appenlight/fil.py

@@ -0,0 +1,24 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+def filter_callable(structure, section=None):
+    structure['SOMEVAL'] = '***REMOVED***'
+    return structure

backend/src/appenlight/forms.py

@@ -0,0 +1,903 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import wtforms
+import formencode
+import re
+import pyramid.threadlocal
+import datetime
+import appenlight.lib.helpers as h
+
+from appenlight.models.user import User
+from appenlight.models.group import Group
+from appenlight.models import DBSession
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.integrations import IntegrationException
+from appenlight.models.integrations.campfire import CampfireIntegration
+from appenlight.models.integrations.bitbucket import BitbucketIntegration
+from appenlight.models.integrations.github import GithubIntegration
+from appenlight.models.integrations.flowdock import FlowdockIntegration
+from appenlight.models.integrations.hipchat import HipchatIntegration
+from appenlight.models.integrations.jira import JiraClient
+from appenlight.models.integrations.slack import SlackIntegration
+from appenlight.lib.ext_json import json
+from wtforms.ext.csrf.form import SecureForm
+from wtforms.compat import iteritems
+from collections import defaultdict
+
+_ = str
+
+strip_filter = lambda x: x.strip() if x else None
+uppercase_filter = lambda x: x.upper() if x else None
+
+FALSE_VALUES = ('false', '', False, None)
+
+
+class CSRFException(Exception):
+    pass
+
+
+class ReactorForm(SecureForm):
+    def __init__(self, formdata=None, obj=None, prefix='', csrf_context=None,
+                 **kwargs):
+        super(ReactorForm, self).__init__(formdata=formdata, obj=obj,
+                                          prefix=prefix,
+                                          csrf_context=csrf_context, **kwargs)
+        self._csrf_context = csrf_context
+
+    def generate_csrf_token(self, csrf_context):
+        return csrf_context.session.get_csrf_token()
+
+    def validate_csrf_token(self, field):
+        request = self._csrf_context or pyramid.threadlocal.get_current_request()
+        is_from_auth_token = 'auth:auth_token' in request.effective_principals
+        if is_from_auth_token:
+            return True
+
+        if field.data != field.current_token:
+            # try to save the day by using token from angular
+            if request.headers.get('X-XSRF-TOKEN') != field.current_token:
+                raise CSRFException('Invalid CSRF token')
+
+    @property
+    def errors_dict(self):
+        r_dict = defaultdict(list)
+        for k, errors in self.errors.items():
+            r_dict[k].extend([str(e) for e in errors])
+        return r_dict
+
+    @property
+    def errors_json(self):
+        return json.dumps(self.errors_dict)
+
+    def populate_obj(self, obj, ignore_none=False):
+        """
+        Populates the attributes of the passed `obj` with data from the form's
+        fields.
+
+        :note: This is a destructive operation; Any attribute with the same name
+               as a field will be overridden. Use with caution.
+        """
+        if ignore_none:
+            for name, field in iteritems(self._fields):
+                if field.data is not None:
+                    field.populate_obj(obj, name)
+        else:
+            for name, field in iteritems(self._fields):
+                field.populate_obj(obj, name)
+
+    css_classes = {}
+    ignore_labels = {}
+
+
+class SignInForm(ReactorForm):
+    came_from = wtforms.HiddenField()
+    sign_in_user_name = wtforms.StringField(_('User Name'))
+    sign_in_user_password = wtforms.PasswordField(_('Password'))
+
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+    html_attrs = {'sign_in_user_name': {'placeholder': 'Your login'},
+                  'sign_in_user_password': {
+                      'placeholder': 'Your password'}}
+
+
+from wtforms.widgets import html_params, HTMLString
+
+
+def select_multi_checkbox(field, ul_class='set', **kwargs):
+    """Render a multi-checkbox widget"""
+    kwargs.setdefault('type', 'checkbox')
+    field_id = kwargs.pop('id', field.id)
+    html = ['<ul %s>' % html_params(id=field_id, class_=ul_class)]
+    for value, label, checked in field.iter_choices():
+        choice_id = '%s-%s' % (field_id, value)
+        options = dict(kwargs, name=field.name, value=value, id=choice_id)
+        if checked:
+            options['checked'] = 'checked'
+        html.append('<li><input %s /> ' % html_params(**options))
+        html.append('<label for="%s">%s</label></li>' % (choice_id, label))
+    html.append('</ul>')
+    return HTMLString(''.join(html))
+
+
+def button_widget(field, button_cls='ButtonField btn btn-default', **kwargs):
+    """Render a button widget"""
+    kwargs.setdefault('type', 'button')
+    field_id = kwargs.pop('id', field.id)
+    kwargs.setdefault('value', field.label.text)
+    html = ['<button %s>%s</button>' % (html_params(id=field_id,
+                                                     class_=button_cls),
+                                         kwargs['value'],)]
+    return HTMLString(''.join(html))
+
+
+def clean_whitespace(value):
+    if value:
+        return value.strip()
+    return value
+
+
+def found_username_validator(form, field):
+    user = User.by_user_name(field.data)
+    # sets user to recover in email validator
+    form.field_user = user
+    if not user:
+        raise wtforms.ValidationError('This username does not exist')
+
+
+def found_username_email_validator(form, field):
+    user = User.by_email(field.data)
+    if not user:
+        raise wtforms.ValidationError('Email is incorrect')
+
+
+def unique_username_validator(form, field):
+    user = User.by_user_name(field.data)
+    if user:
+        raise wtforms.ValidationError('This username already exists in system')
+
+
+def unique_groupname_validator(form, field):
+    group = Group.by_group_name(field.data)
+    mod_group = getattr(form, '_modified_group', None)
+    if group and (not mod_group or mod_group.id != group.id):
+        raise wtforms.ValidationError(
+            'This group name already exists in system')
+
+
+def unique_email_validator(form, field):
+    user = User.by_email(field.data)
+    if user:
+        raise wtforms.ValidationError('This email already exists in system')
+
+
+def email_validator(form, field):
+    validator = formencode.validators.Email()
+    try:
+        validator.to_python(field.data)
+    except formencode.Invalid as e:
+        raise wtforms.ValidationError(e)
+
+
+def unique_alert_email_validator(form, field):
+    q = DBSession.query(AlertChannel)
+    q = q.filter(AlertChannel.channel_name == 'email')
+    q = q.filter(AlertChannel.channel_value == field.data)
+    email = q.first()
+    if email:
+        raise wtforms.ValidationError(
+            'This email already exists in alert system')
+
+
+def blocked_email_validator(form, field):
+    blocked_emails = [
+        'goood-mail.org',
+        'shoeonlineblog.com',
+        'louboutinemart.com',
+        'guccibagshere.com',
+        'nikeshoesoutletforsale.com'
+    ]
+    data = field.data or ''
+    domain = data.split('@')[-1]
+    if domain in blocked_emails:
+        raise wtforms.ValidationError('Don\'t spam')
+
+
+def old_password_validator(form, field):
+    if not field.user.check_password(field.data or ''):
+        raise wtforms.ValidationError('You need to enter correct password')
+
+
+class UserRegisterForm(ReactorForm):
+    user_name = wtforms.StringField(
+        _('User Name'),
+        filters=[strip_filter],
+        validators=[
+            wtforms.validators.Length(min=2, max=30),
+            wtforms.validators.Regexp(
+                re.compile(r'^[\.\w-]+$', re.UNICODE),
+                message="Invalid characters used"),
+            unique_username_validator,
+            wtforms.validators.DataRequired()
+        ])
+
+    user_password = wtforms.PasswordField(_('User Password'),
+                                          filters=[strip_filter],
+                                          validators=[
+                                              wtforms.validators.Length(min=4),
+                                              wtforms.validators.DataRequired()
+                                          ])
+
+    email = wtforms.StringField(_('Email Address'),
+                                filters=[strip_filter],
+                                validators=[email_validator,
+                                            unique_email_validator,
+                                            blocked_email_validator,
+                                            wtforms.validators.DataRequired()],
+                                description=_("We promise we will not share "
+                                              "your email with anyone"))
+    first_name = wtforms.HiddenField(_('First Name'))
+    last_name = wtforms.HiddenField(_('Last Name'))
+
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+    html_attrs = {'user_name': {'placeholder': 'Your login'},
+                  'user_password': {'placeholder': 'Your password'},
+                  'email': {'placeholder': 'Your email'}}
+
+
+class UserCreateForm(UserRegisterForm):
+    status = wtforms.BooleanField('User status',
+                                  false_values=FALSE_VALUES)
+
+
+class UserUpdateForm(UserCreateForm):
+    user_name = None
+    user_password = wtforms.PasswordField(_('User Password'),
+                                          filters=[strip_filter],
+                                          validators=[
+                                              wtforms.validators.Length(min=4),
+                                              wtforms.validators.Optional()
+                                          ])
+    email = wtforms.StringField(_('Email Address'),
+                                filters=[strip_filter],
+                                validators=[email_validator,
+                                            wtforms.validators.DataRequired()])
+
+
+class LostPasswordForm(ReactorForm):
+    email = wtforms.StringField(_('Email Address'),
+                                filters=[strip_filter],
+                                validators=[email_validator,
+                                            found_username_email_validator,
+                                            wtforms.validators.DataRequired()])
+
+    submit = wtforms.SubmitField(_('Reset password'))
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+class ChangePasswordForm(ReactorForm):
+    old_password = wtforms.PasswordField(
+        'Old Password',
+        filters=[strip_filter],
+        validators=[old_password_validator,
+                    wtforms.validators.DataRequired()])
+
+    new_password = wtforms.PasswordField(
+        'New Password',
+        filters=[strip_filter],
+        validators=[wtforms.validators.Length(min=4),
+                    wtforms.validators.DataRequired()])
+    new_password_confirm = wtforms.PasswordField(
+        'Confirm Password',
+        filters=[strip_filter],
+        validators=[wtforms.validators.EqualTo('new_password'),
+                    wtforms.validators.DataRequired()])
+    submit = wtforms.SubmitField('Change Password')
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+class CheckPasswordForm(ReactorForm):
+    password = wtforms.PasswordField(
+        'Password',
+        filters=[strip_filter],
+        validators=[old_password_validator,
+                    wtforms.validators.DataRequired()])
+
+
+class NewPasswordForm(ReactorForm):
+    new_password = wtforms.PasswordField(
+        'New Password',
+        filters=[strip_filter],
+        validators=[wtforms.validators.Length(min=4),
+                    wtforms.validators.DataRequired()])
+    new_password_confirm = wtforms.PasswordField(
+        'Confirm Password',
+        filters=[strip_filter],
+        validators=[wtforms.validators.EqualTo('new_password'),
+                    wtforms.validators.DataRequired()])
+    submit = wtforms.SubmitField('Set Password')
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+class CORSTextAreaField(wtforms.StringField):
+    """
+    This field represents an HTML ``<textarea>`` and can be used to take
+    multi-line input.
+    """
+    widget = wtforms.widgets.TextArea()
+
+    def process_formdata(self, valuelist):
+        self.data = []
+        if valuelist:
+            data = [x.strip() for x in valuelist[0].split('\n')]
+            for d in data:
+                if not d:
+                    continue
+                if d.startswith('www.'):
+                    d = d[4:]
+                if data:
+                    self.data.append(d)
+        else:
+            self.data = []
+        self.data = '\n'.join(self.data)
+
+
+class ApplicationCreateForm(ReactorForm):
+    resource_name = wtforms.StringField(
+        _('Application name'),
+        filters=[strip_filter],
+        validators=[wtforms.validators.Length(min=1),
+                    wtforms.validators.DataRequired()])
+
+    domains = CORSTextAreaField(
+        _('Domain names for CORS headers '),
+        validators=[wtforms.validators.Length(min=1),
+                    wtforms.validators.Optional()],
+        description='Required for Javascript error '
+                    'tracking (one line one domain, skip http:// part)')
+
+    submit = wtforms.SubmitField(_('Create Application'))
+
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+    html_attrs = {'resource_name': {'placeholder': 'Application Name'},
+                  'uptime_url': {'placeholder': 'http://somedomain.com'}}
+
+
+class ApplicationUpdateForm(ApplicationCreateForm):
+    default_grouping = wtforms.SelectField(
+        _('Default grouping for errors'),
+        choices=[('url_type', 'Error Type + location',),
+                 ('url_traceback', 'Traceback + location',),
+                 ('traceback_server', 'Traceback + Server',)],
+        default='url_traceback')
+
+    error_report_threshold = wtforms.IntegerField(
+        _('Alert on error reports'),
+        validators=[
+            wtforms.validators.NumberRange(min=1),
+            wtforms.validators.DataRequired()
+        ],
+        description='Application requires to send at least this amount of '
+                    'error reports per minute to open alert'
+    )
+
+    slow_report_threshold = wtforms.IntegerField(
+        _('Alert on slow reports'),
+        validators=[wtforms.validators.NumberRange(min=1),
+                    wtforms.validators.DataRequired()],
+        description='Application requires to send at least this amount of '
+                    'slow reports per minute to open alert')
+
+    allow_permanent_storage = wtforms.BooleanField(
+        _('Permanent logs'),
+        false_values=FALSE_VALUES,
+        description=_(
+            'Allow permanent storage of logs in separate DB partitions'))
+
+    submit = wtforms.SubmitField(_('Create Application'))
+
+
+class UserSearchSchemaForm(ReactorForm):
+    user_name = wtforms.StringField('User Name',
+                                    filters=[strip_filter], )
+
+    submit = wtforms.SubmitField(_('Search User'))
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+    '<li class="user_exists"><span></span></li>'
+
+
+class YesNoForm(ReactorForm):
+    no = wtforms.SubmitField('No', default='')
+    yes = wtforms.SubmitField('Yes', default='')
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+status_codes = [('', 'All',), ('500', '500',), ('404', '404',)]
+
+priorities = [('', 'All',)]
+for i in range(1, 11):
+    priorities.append((str(i), str(i),))
+
+report_status_choices = [('', 'All',),
+                         ('never_reviewed', 'Never revieved',),
+                         ('reviewed', 'Revieved',),
+                         ('public', 'Public',),
+                         ('fixed', 'Fixed',), ]
+
+
+class ReportBrowserForm(ReactorForm):
+    applications = wtforms.SelectMultipleField('Applications',
+                                               widget=select_multi_checkbox)
+    http_status = wtforms.SelectField('HTTP Status', choices=status_codes)
+    priority = wtforms.SelectField('Priority', choices=priorities, default='')
+    start_date = wtforms.DateField('Start Date')
+    end_date = wtforms.DateField('End Date')
+    error = wtforms.StringField('Error')
+    url_path = wtforms.StringField('URL Path')
+    url_domain = wtforms.StringField('URL Domain')
+    report_status = wtforms.SelectField('Report status',
+                                        choices=report_status_choices,
+                                        default='')
+    submit = wtforms.SubmitField('<span class="glyphicon glyphicon-search">'
+                                 '</span> Filter results',
+                                 widget=button_widget)
+
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+slow_report_status_choices = [('', 'All',),
+                              ('never_reviewed', 'Never revieved',),
+                              ('reviewed', 'Revieved',),
+                              ('public', 'Public',), ]
+
+
+class BulkOperationForm(ReactorForm):
+    applications = wtforms.SelectField('Applications')
+    start_date = wtforms.DateField(
+        'Start Date',
+        default=lambda: datetime.datetime.utcnow() - datetime.timedelta(
+            days=90))
+    end_date = wtforms.DateField('End Date')
+    confirm = wtforms.BooleanField(
+        'Confirm operation',
+        validators=[wtforms.validators.DataRequired()])
+
+
+class LogBrowserForm(ReactorForm):
+    applications = wtforms.SelectMultipleField('Applications',
+                                               widget=select_multi_checkbox)
+    start_date = wtforms.DateField('Start Date')
+    log_level = wtforms.StringField('Log level')
+    message = wtforms.StringField('Message')
+    namespace = wtforms.StringField('Namespace')
+    submit = wtforms.SubmitField(
+        '<span class="glyphicon glyphicon-search"></span> Filter results',
+        widget=button_widget)
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+class CommentForm(ReactorForm):
+    body = wtforms.TextAreaField('Comment', validators=[
+        wtforms.validators.Length(min=1),
+        wtforms.validators.DataRequired()
+    ])
+    submit = wtforms.SubmitField('Comment', )
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+class EmailChannelCreateForm(ReactorForm):
+    email = wtforms.StringField(_('Email Address'),
+                                filters=[strip_filter],
+                                validators=[email_validator,
+                                            unique_alert_email_validator,
+                                            wtforms.validators.DataRequired()])
+    submit = wtforms.SubmitField('Add email channel', )
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+def gen_user_profile_form():
+    class UserProfileForm(ReactorForm):
+        email = wtforms.StringField(
+            _('Email Address'),
+            validators=[email_validator, wtforms.validators.DataRequired()])
+        first_name = wtforms.StringField(_('First Name'))
+        last_name = wtforms.StringField(_('Last Name'))
+        company_name = wtforms.StringField(_('Company Name'))
+        company_address = wtforms.TextAreaField(_('Company Address'))
+        zip_code = wtforms.StringField(_('ZIP code'))
+        city = wtforms.StringField(_('City'))
+        notifications = wtforms.BooleanField('Account notifications',
+                                             false_values=FALSE_VALUES)
+        submit = wtforms.SubmitField(_('Update Account'))
+        ignore_labels = ['submit']
+        css_classes = {'submit': 'btn btn-primary'}
+
+    return UserProfileForm
+
+
+class PurgeAppForm(ReactorForm):
+    resource_id = wtforms.HiddenField(
+        'App Id',
+        validators=[wtforms.validators.DataRequired()])
+    days = wtforms.IntegerField(
+        'Days',
+        validators=[wtforms.validators.DataRequired()])
+    password = wtforms.PasswordField(
+        'Admin Password',
+        validators=[old_password_validator, wtforms.validators.DataRequired()])
+    submit = wtforms.SubmitField(_('Purge Data'))
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+class IntegrationRepoForm(ReactorForm):
+    host_name = wtforms.StringField("Service Host", default='')
+    user_name = wtforms.StringField(
+        "User Name",
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired(),
+                    wtforms.validators.Length(min=1)])
+    repo_name = wtforms.StringField(
+        "Repo Name",
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired(),
+                    wtforms.validators.Length(min=1)])
+
+
+class IntegrationBitbucketForm(IntegrationRepoForm):
+    host_name = wtforms.StringField("Service Host",
+                                    default='https://bitbucket.org')
+
+    def validate_user_name(self, field):
+        try:
+            request = pyramid.threadlocal.get_current_request()
+            client = BitbucketIntegration.create_client(
+                request,
+                self.user_name.data,
+                self.repo_name.data)
+            client.get_assignees()
+        except IntegrationException as e:
+            raise wtforms.validators.ValidationError(str(e))
+
+
+class IntegrationGithubForm(IntegrationRepoForm):
+    host_name = wtforms.StringField("Service Host",
+                                    default='https://github.com')
+
+    def validate_user_name(self, field):
+        try:
+            request = pyramid.threadlocal.get_current_request()
+            client = GithubIntegration.create_client(
+                request,
+                self.user_name.data,
+                self.repo_name.data)
+            client.get_assignees()
+        except IntegrationException as e:
+            raise wtforms.validators.ValidationError(str(e))
+            raise wtforms.validators.ValidationError(str(e))
+
+
+def filter_rooms(data):
+    if data is not None:
+        rooms = data.split(',')
+        return ','.join([r.strip() for r in rooms])
+
+
+class IntegrationCampfireForm(ReactorForm):
+    account = wtforms.StringField(
+        'Account',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    api_token = wtforms.StringField(
+        'Api Token',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    rooms = wtforms.StringField('Room ID list', filters=[filter_rooms])
+
+    def validate_api_token(self, field):
+        try:
+            client = CampfireIntegration.create_client(self.api_token.data,
+                                                       self.account.data)
+            client.get_account()
+        except IntegrationException as e:
+            raise wtforms.validators.ValidationError(str(e))
+
+    def validate_rooms(self, field):
+        if not field.data:
+            return
+        client = CampfireIntegration.create_client(self.api_token.data,
+                                                   self.account.data)
+
+        try:
+            room_list = [r['id'] for r in client.get_rooms()]
+        except IntegrationException as e:
+            raise wtforms.validators.ValidationError(str(e))
+
+        rooms = field.data.split(',')
+        if len(rooms) > 3:
+            msg = 'You can use up to 3 room ids'
+            raise wtforms.validators.ValidationError(msg)
+        if rooms:
+            for room_id in rooms:
+                if int(room_id) not in room_list:
+                    msg = "Room %s doesn't exist"
+                    raise wtforms.validators.ValidationError(msg % room_id)
+                if not room_id.strip().isdigit():
+                    msg = 'You must use only integers for room ids'
+                    raise wtforms.validators.ValidationError(msg)
+
+    submit = wtforms.SubmitField(_('Connect to Campfire'))
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+def filter_rooms(data):
+    if data is not None:
+        rooms = data.split(',')
+        return ','.join([r.strip() for r in rooms])
+
+
+class IntegrationHipchatForm(ReactorForm):
+    api_token = wtforms.StringField(
+        'Api Token',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    rooms = wtforms.StringField(
+        'Room ID list',
+        filters=[filter_rooms],
+        validators=[wtforms.validators.DataRequired()])
+
+    def validate_rooms(self, field):
+        if not field.data:
+            return
+        client = HipchatIntegration.create_client(self.api_token.data)
+        rooms = field.data.split(',')
+        if len(rooms) > 3:
+            msg = 'You can use up to 3 room ids'
+            raise wtforms.validators.ValidationError(msg)
+        if rooms:
+            for room_id in rooms:
+                if not room_id.strip().isdigit():
+                    msg = 'You must use only integers for room ids'
+                    raise wtforms.validators.ValidationError(msg)
+                try:
+                    client.send({
+                        "message_format": 'text',
+                        "message": "testing for room existence",
+                        "from": "App Enlight",
+                        "room_id": room_id,
+                        "color": "green"
+                    })
+                except IntegrationException as exc:
+                    msg = 'Room id: %s exception: %s'
+                    raise wtforms.validators.ValidationError(msg % (room_id,
+                                                                    exc))
+
+
+class IntegrationFlowdockForm(ReactorForm):
+    api_token = wtforms.StringField('API Token',
+                                    filters=[strip_filter],
+                                    validators=[
+                                        wtforms.validators.DataRequired()
+                                    ], )
+
+    def validate_api_token(self, field):
+        try:
+            client = FlowdockIntegration.create_client(self.api_token.data)
+            registry = pyramid.threadlocal.get_current_registry()
+            payload = {
+                "source": registry.settings['mailing.from_name'],
+                "from_address": registry.settings['mailing.from_email'],
+                "subject": "Integration test",
+                "content": "If you can see this it was successful",
+                "tags": ["appenlight"],
+                "link": registry.settings['mailing.app_url']
+            }
+            client.send_to_inbox(payload)
+        except IntegrationException as e:
+            raise wtforms.validators.ValidationError(str(e))
+
+
+class IntegrationSlackForm(ReactorForm):
+    webhook_url = wtforms.StringField(
+        'Reports webhook',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+
+    def validate_webhook_url(self, field):
+        registry = pyramid.threadlocal.get_current_registry()
+        client = SlackIntegration.create_client(field.data)
+        link = "<%s|%s>" % (registry.settings['mailing.app_url'],
+                            registry.settings['mailing.from_name'])
+        test_data = {
+            "username": "App Enlight",
+            "icon_emoji": ":fire:",
+            "attachments": [
+                {"fallback": "Testing integration channel: %s" % link,
+                 "pretext": "Testing integration channel:  %s" % link,
+                 "color": "good",
+                 "fields": [
+                     {
+                         "title": "Status",
+                         "value": "Integration is working fine",
+                         "short": False
+                     }
+                 ]}
+            ]
+        }
+        try:
+            client.make_request(data=test_data)
+        except IntegrationException as exc:
+            raise wtforms.validators.ValidationError(str(exc))
+
+
+class IntegrationWebhooksForm(ReactorForm):
+    reports_webhook = wtforms.StringField(
+        'Reports webhook',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    alerts_webhook = wtforms.StringField(
+        'Alerts webhook',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    submit = wtforms.SubmitField(_('Setup webhooks'))
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-primary'}
+
+
+class IntegrationJiraForm(ReactorForm):
+    host_name = wtforms.StringField(
+        'Server URL',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    user_name = wtforms.StringField(
+        'Username',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    password = wtforms.PasswordField(
+        'Password',
+        filters=[strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+    project = wtforms.StringField(
+        'Project key',
+        filters=[uppercase_filter, strip_filter],
+        validators=[wtforms.validators.DataRequired()])
+
+    def validate_project(self, field):
+        if not field.data:
+            return
+        try:
+            client = JiraClient(self.user_name.data,
+                                self.password.data,
+                                self.host_name.data,
+                                self.project.data)
+        except Exception as exc:
+            raise wtforms.validators.ValidationError(str(exc))
+
+        room_list = [r.key.upper() for r in client.get_projects()]
+        if field.data.upper() not in room_list:
+            msg = "Project %s doesn\t exist in your Jira Instance"
+            raise wtforms.validators.ValidationError(msg % field.data)
+
+
+def get_deletion_form(resource):
+    class F(ReactorForm):
+        application_name = wtforms.StringField(
+            'Application Name',
+            filters=[strip_filter],
+            validators=[wtforms.validators.AnyOf([resource.resource_name])])
+        resource_id = wtforms.HiddenField(default=resource.resource_id)
+        submit = wtforms.SubmitField(_('Delete my application'))
+        ignore_labels = ['submit']
+        css_classes = {'submit': 'btn btn-danger'}
+
+    return F
+
+
+class ChangeApplicationOwnerForm(ReactorForm):
+    password = wtforms.PasswordField(
+        'Password',
+        filters=[strip_filter],
+        validators=[old_password_validator,
+                    wtforms.validators.DataRequired()])
+
+    user_name = wtforms.StringField(
+        'New owners username',
+        filters=[strip_filter],
+        validators=[found_username_validator,
+                    wtforms.validators.DataRequired()])
+    submit = wtforms.SubmitField(_('Transfer ownership of application'))
+    ignore_labels = ['submit']
+    css_classes = {'submit': 'btn btn-danger'}
+
+
+def default_filename():
+    return 'Invoice %s' % datetime.datetime.utcnow().strftime('%Y/%m')
+
+
+class FileUploadForm(ReactorForm):
+    title = wtforms.StringField('File Title',
+                                default=default_filename,
+                                validators=[wtforms.validators.DataRequired()])
+    file = wtforms.FileField('File')
+
+    def validate_file(self, field):
+        if not hasattr(field.data, 'file'):
+            raise wtforms.ValidationError('File is missing')
+
+    submit = wtforms.SubmitField(_('Upload'))
+
+
+def get_partition_deletion_form(es_indices, pg_indices):
+    class F(ReactorForm):
+        es_index = wtforms.SelectMultipleField('Elasticsearch',
+                                               choices=[(ix, '') for ix in
+                                                        es_indices])
+        pg_index = wtforms.SelectMultipleField('pg',
+                                               choices=[(ix, '') for ix in
+                                                        pg_indices])
+        confirm = wtforms.TextField('Confirm',
+                                    filters=[uppercase_filter, strip_filter],
+                                    validators=[
+                                        wtforms.validators.AnyOf(['CONFIRM']),
+                                        wtforms.validators.DataRequired()])
+        ignore_labels = ['submit']
+        css_classes = {'submit': 'btn btn-danger'}
+
+    return F
+
+
+class GroupCreateForm(ReactorForm):
+    group_name = wtforms.StringField(
+        _('Group Name'),
+        filters=[strip_filter],
+        validators=[
+            wtforms.validators.Length(min=2, max=50),
+            unique_groupname_validator,
+            wtforms.validators.DataRequired()
+        ])
+    description = wtforms.StringField(_('Group description'))
+
+
+time_choices = [(k, v['label'],) for k, v in h.time_deltas.items()]
+
+
+class AuthTokenCreateForm(ReactorForm):
+    description = wtforms.StringField(_('Token description'))
+    expires = wtforms.SelectField('Expires',
+                                  coerce=lambda x: x,
+                                  choices=time_choices,
+                                  validators=[wtforms.validators.Optional()])

backend/src/appenlight/migrations/README

@@ -0,0 +1,1 b''
+Generic single-database configuration.
  No newline at end of file

backend/src/appenlight/migrations/__init__.py

@@ -0,0 +1,22 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+

backend/src/appenlight/migrations/env.py

@@ -0,0 +1,103 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool, MetaData
+from logging.config import fileConfig
+from appenlight.models import NAMING_CONVENTION
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+
+
+target_metadata = MetaData(naming_convention=NAMING_CONVENTION)
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+VERSION_TABLE_NAME = 'alembic_appenlight_version'
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url, target_metadata=target_metadata,
+                      transaction_per_migration=True,
+                      version_table=VERSION_TABLE_NAME)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,
+        transaction_per_migration=True,
+        version_table=VERSION_TABLE_NAME
+    )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

backend/src/appenlight/migrations/script.py.mako

@@ -0,0 +1,22 b''
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

backend/src/appenlight/migrations/versions/55b6e612672f_initial_tables.py

@@ -0,0 +1,629 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+"""initial tables
+
+Revision ID: 55b6e612672f
+Revises: None
+Create Date: 2014-10-13 23:47:38.295159
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '55b6e612672f'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('users', sa.Column('first_name', sa.Unicode(25)))
+    op.add_column('users', sa.Column('last_name', sa.Unicode(50)))
+    op.add_column('users', sa.Column('company_name', sa.Unicode(255)))
+    op.add_column('users', sa.Column('company_address', sa.Unicode(255)))
+    op.add_column('users', sa.Column('phone1', sa.Unicode(25)))
+    op.add_column('users', sa.Column('phone2', sa.Unicode(25)))
+    op.add_column('users', sa.Column('zip_code', sa.Unicode(25)))
+    op.add_column('users', sa.Column('default_report_sort', sa.Unicode(20), nullable=False, server_default="newest"))
+    op.add_column('users', sa.Column('city', sa.Unicode(128)))
+    op.add_column('users', sa.Column('notes', sa.UnicodeText, server_default=''))
+    op.add_column('users', sa.Column('notifications', sa.Boolean(), nullable=False, server_default='true'))
+    op.add_column('users', sa.Column('registration_ip', sa.Unicode(40), nullable=False, server_default=''))
+
+    op.create_table(
+        'integrations',
+        sa.Column('id', sa.Integer(), primary_key=True),
+        sa.Column('resource_id', sa.Integer(),
+                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
+                                ondelete='cascade')),
+        sa.Column('integration_name', sa.Unicode(64)),
+        sa.Column('config', sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column('modified_date', sa.DateTime(), nullable=False, server_default=sa.func.now()),
+        sa.Column('external_id', sa.Unicode(255)),
+        sa.Column('external_id2', sa.Unicode(255))
+    )
+
+    op.create_table(
+        'alert_channels',
+        sa.Column('owner_id', sa.Integer(),
+                  sa.ForeignKey('users.id', onupdate='cascade',
+                                ondelete='cascade'), nullable=False),
+        sa.Column('channel_name', sa.Unicode(25), nullable=False),
+        sa.Column('channel_value', sa.Unicode(80), nullable=False),
+        sa.Column('channel_json_conf', sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column('channel_validated', sa.Boolean, nullable=False, server_default='False'),
+        sa.Column('send_alerts', sa.Boolean, nullable=False, server_default='True'),
+        sa.Column('notify_only_first', sa.Boolean, nullable=False, server_default='False'),
+        sa.Column('daily_digest', sa.Boolean, nullable=False, server_default='True'),
+        sa.Column('pkey', sa.Integer(), primary_key=True),
+        sa.Column('integration_id', sa.Integer,
+                  sa.ForeignKey('integrations.id', onupdate='cascade',
+                                ondelete='cascade')),
+    )
+    op.create_unique_constraint('uq_alert_channels', 'alert_channels',
+                                ["owner_id", "channel_name", "channel_value"])
+
+    op.create_table(
+        'alert_channels_actions',
+        sa.Column('owner_id', sa.Integer(), nullable=False),
+        sa.Column('resource_id', sa.Integer(),
+                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
+                                ondelete='cascade')),
+        sa.Column('pkey', sa.Integer(), primary_key=True),
+        sa.Column('action', sa.Unicode(10), nullable=False, server_default='always'),
+        sa.Column('rule', sa.dialects.postgresql.JSON),
+        sa.Column('type', sa.Unicode(10), index=True),
+        sa.Column('other_id', sa.Unicode(40), index=True),
+        sa.Column('config', sa.dialects.postgresql.JSON),
+        sa.Column('name', sa.Unicode(255), server_default='')
+    )
+
+
+    op.create_table(
+        'application_postprocess_conf',
+        sa.Column('pkey', sa.Integer(), primary_key=True),
+        sa.Column('do', sa.Unicode(25), nullable=False),
+        sa.Column('new_value', sa.UnicodeText(), nullable=False, server_default=''),
+        sa.Column('resource_id', sa.Integer(),
+                  sa.ForeignKey('resources.resource_id',
+                                onupdate='cascade',
+                                ondelete='cascade'), nullable=False),
+        sa.Column('rule', sa.dialects.postgresql.JSON),
+    )
+
+    op.create_table(
+        'applications',
+        sa.Column('resource_id', sa.Integer(),
+                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
+                                ondelete='cascade'), nullable=False,
+                  primary_key=True, autoincrement=False),
+        sa.Column('domains', sa.UnicodeText, nullable=False),
+        sa.Column('api_key', sa.Unicode(32), nullable=False, index=True),
+        sa.Column('default_grouping', sa.Unicode(20), nullable=False, server_default='url_type'),
+        sa.Column('public_key', sa.Unicode(32), nullable=False, index=True),
+        sa.Column('error_report_threshold', sa.Integer(), server_default='10', nullable=False),
+        sa.Column('slow_report_threshold', sa.Integer(), server_default='10', nullable=False),
+        sa.Column('apdex_threshold', sa.Float(), server_default='0.7', nullable=False),
+        sa.Column('allow_permanent_storage', sa.Boolean(), server_default="false", nullable=False),
+    )
+    op.create_unique_constraint(None, 'applications',
+                                ["public_key"])
+    op.create_unique_constraint(None, 'applications',
+                                ["api_key"])
+
+    op.create_table(
+        'metrics',
+        sa.Column('pkey', sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column('resource_id', sa.Integer(),
+                  sa.ForeignKey('resources.resource_id',
+                                onupdate='cascade',
+                                ondelete='cascade')),
+        sa.Column('timestamp', sa.DateTime),
+        sa.Column('namespace', sa.Unicode(255)),
+        sa.Column('tags', sa.dialects.postgresql.JSON, server_default="{}")
+    )
+
+    op.create_table(
+        'events',
+        sa.Column('id', sa.Integer, nullable=False, primary_key=True),
+        sa.Column('start_date', sa.DateTime, nullable=False, index=True),
+        sa.Column('end_date', sa.DateTime),
+        sa.Column('status', sa.Integer(), nullable=False, index=True),
+        sa.Column('event_type', sa.Integer(), nullable=False, index=True),
+        sa.Column('origin_user_id', sa.Integer()),
+        sa.Column('target_user_id', sa.Integer()),
+        sa.Column('resource_id', sa.Integer(), index=True),
+        sa.Column('text', sa.UnicodeText, server_default=''),
+        sa.Column('values', sa.dialects.postgresql.JSON),
+        sa.Column('target_id', sa.Integer()),
+        sa.Column('target_uuid', sa.Unicode(40), index=True)
+    )
+
+    op.create_table(
+        'logs',
+        sa.Column('log_id', sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column('resource_id', sa.Integer(),
+                  sa.ForeignKey('resources.resource_id',
+                                onupdate='cascade',
+                                ondelete='cascade')),
+        sa.Column('log_level', sa.SmallInteger(), nullable=False),
+        sa.Column('primary_key', sa.Unicode(128), nullable=True),
+        sa.Column('message', sa.UnicodeText, nullable=False, server_default=''),
+        sa.Column('timestamp', sa.DateTime),
+        sa.Column('namespace', sa.Unicode(255)),
+        sa.Column('request_id', sa.Unicode(40)),
+        sa.Column('tags', sa.dialects.postgresql.JSON, server_default="{}"),
+        sa.Column('permanent', sa.Boolean(), server_default="false",
+                  nullable=False)
+    )
+
+    op.create_table(
+        'reports_groups',
+        sa.Column('id', sa.types.BigInteger, primary_key=True),
+        sa.Column('resource_id', sa.Integer,
+                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
+                                ondelete='cascade'), nullable=False),
+        sa.Column('priority', sa.Integer, nullable=False, server_default="5"),
+        sa.Column('first_timestamp', sa.DateTime(), nullable=False, server_default=sa.func.now()),
+        sa.Column('last_timestamp', sa.DateTime()),
+        sa.Column('error', sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column('grouping_hash', sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column('triggered_postprocesses_ids', sa.dialects.postgresql.JSON, nullable=False, server_default="[]"),
+        sa.Column('report_type', sa.Integer, nullable=False, server_default="0"),
+        sa.Column('total_reports', sa.Integer, nullable=False, server_default="0"),
+        sa.Column('last_report', sa.Integer, nullable=False, server_default="0"),
+        sa.Column('occurences', sa.Integer, nullable=False, server_default="1"),
+        sa.Column('average_duration', sa.Float(), nullable=False, server_default="0"),
+        sa.Column('summed_duration', sa.Float(), nullable=False, server_default="0"),
+        sa.Column('notified', sa.Boolean, nullable=False, server_default="False"),
+        sa.Column('fixed', sa.Boolean, nullable=False, server_default="False"),
+        sa.Column('public', sa.Boolean, nullable=False, server_default="False"),
+        sa.Column('read', sa.Boolean, nullable=False, server_default="False"),
+    )
+
+    op.create_table(
+        'reports',
+        sa.Column('id', sa.types.BigInteger, primary_key=True),
+        sa.Column('group_id', sa.types.BigInteger,
+                  sa.ForeignKey('reports_groups.id', onupdate='cascade',
+                                ondelete='cascade'), nullable=False, index=True),
+        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
+        sa.Column('report_type', sa.Integer, nullable=False, server_default="0"),
+        sa.Column('error', sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column('extra', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
+        sa.Column('request', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
+        sa.Column('tags', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
+        sa.Column('ip', sa.Unicode(39), nullable=False, server_default=""),
+        sa.Column('username', sa.Unicode(255), nullable=False, server_default=""),
+        sa.Column('user_agent', sa.Unicode(512), nullable=False, server_default=""),
+        sa.Column('url', sa.UnicodeText, nullable=False, server_default=""),
+        sa.Column('request_id', sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column('request_stats', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
+        sa.Column('traceback', sa.dialects.postgresql.JSON, nullable=False, server_default="{}"),
+        sa.Column('traceback_hash', sa.Unicode(40), nullable=False, server_default=""),
+        sa.Column('start_time', sa.DateTime(), nullable=False, server_default=sa.func.now()),
+        sa.Column('end_time', sa.DateTime()),
+        sa.Column('report_group_time', sa.DateTime, index=True, nullable=False, server_default=sa.func.now()),
+        sa.Column('duration', sa.Float(), nullable=False, server_default="0"),
+        sa.Column('http_status', sa.Integer, index=True),
+        sa.Column('url_domain', sa.Unicode(128)),
+        sa.Column('url_path', sa.UnicodeText),
+        sa.Column('language', sa.Integer, server_default="0"),
+    )
+    op.create_index(None, 'reports',
+                    [sa.text("(tags ->> 'server_name')")])
+    op.create_index(None, 'reports',
+                    [sa.text("(tags ->> 'view_name')")])
+
+    op.create_table(
+        'reports_assignments',
+        sa.Column('group_id', sa.types.BigInteger, nullable=False, primary_key=True),
+        sa.Column('owner_id', sa.Integer,
+            sa.ForeignKey('users.id', onupdate='cascade',ondelete='cascade'),
+            nullable=False, primary_key=True),
+        sa.Column('report_time', sa.DateTime, nullable=False)
+        )
+
+    op.create_table(
+        'reports_comments',
+        sa.Column('comment_id', sa.Integer, primary_key=True),
+        sa.Column('body', sa.UnicodeText, nullable=False, server_default=''),
+        sa.Column('owner_id', sa.Integer,
+                  sa.ForeignKey('users.id', onupdate='cascade',
+                                ondelete='set null'), nullable=True),
+        sa.Column('created_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
+        sa.Column('report_time', sa.DateTime, nullable=False),
+        sa.Column('group_id', sa.types.BigInteger, nullable=False)
+    )
+
+    op.create_table(
+        'reports_stats',
+        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
+        sa.Column('start_interval', sa.DateTime, nullable=False, index=True),
+        sa.Column('group_id', sa.types.BigInteger, index=True),
+        sa.Column('occurences', sa.Integer, nullable=False, server_default='0', index=True),
+        sa.Column('owner_user_id', sa.Integer),
+        sa.Column('type', sa.Integer, index=True, nullable=False),
+        sa.Column('duration', sa.Float(), server_default='0'),
+        sa.Column('server_name', sa.Unicode(128),
+                  server_default=''),
+        sa.Column('view_name', sa.Unicode(128),
+                  server_default=''),
+        sa.Column('id', sa.BigInteger(), nullable=False, primary_key=True),
+    )
+    op.create_index('ix_reports_stats_start_interval_group_id', 'reports_stats',
+                    ["start_interval", "group_id"])
+
+    op.create_table(
+        'slow_calls',
+        sa.Column('id', sa.types.BigInteger, primary_key=True),
+        sa.Column('report_id', sa.types.BigInteger, sa.ForeignKey('reports.id', onupdate='cascade', ondelete='cascade'),
+                  nullable=False, index=True),
+        sa.Column('duration', sa.Float(), nullable=False, server_default="0", index=True),
+        sa.Column('timestamp', sa.DateTime, nullable=False, server_default=sa.func.now(), index=True),
+        sa.Column('report_group_time', sa.DateTime, index=True, nullable=False, server_default=sa.func.now()),
+        sa.Column('type', sa.Unicode(16), nullable=False, index=True),
+        sa.Column('statement', sa.UnicodeText, nullable=False, server_default=''),
+        sa.Column('parameters', sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column('location', sa.UnicodeText, server_default=''),
+        sa.Column('subtype', sa.Unicode(16), nullable=False, index=True),
+        sa.Column('resource_id', sa.Integer, nullable=False, index=True),
+        sa.Column('statement_hash', sa.Unicode(60), index=True)
+    )
+
+    op.create_table(
+        'tags',
+        sa.Column('id', sa.types.BigInteger, primary_key=True),
+        sa.Column('resource_id', sa.Integer,
+                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
+                                ondelete='cascade')),
+        sa.Column('first_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
+        sa.Column('last_timestamp', sa.DateTime, nullable=False, server_default=sa.func.now()),
+        sa.Column('name', sa.Unicode(32), nullable=False),
+        sa.Column('value', sa.dialects.postgresql.JSON, nullable=False),
+        sa.Column('times_seen', sa.Integer, nullable=False, server_default='1')
+    )
+
+    op.create_table(
+        'auth_tokens',
+        sa.Column('id', sa.Integer, nullable=False, primary_key=True),
+        sa.Column('token', sa.Unicode),
+        sa.Column('creation_date', sa.DateTime, nullable=False, server_default=sa.func.now()),
+        sa.Column('expires', sa.DateTime),
+        sa.Column('owner_id', sa.Integer,
+                  sa.ForeignKey('users.id', onupdate='cascade',
+                                ondelete='cascade')),
+        sa.Column('description', sa.Unicode),
+    )
+
+    op.create_table(
+        'channels_actions',
+        sa.Column('channel_pkey', sa.Integer,
+                  sa.ForeignKey('alert_channels.pkey',
+                                ondelete='CASCADE', onupdate='CASCADE')),
+        sa.Column('action_pkey', sa.Integer,
+                  sa.ForeignKey('alert_channels_actions.pkey',
+                                ondelete='CASCADE', onupdate='CASCADE'))
+    )
+
+    op.create_table(
+        'config',
+        sa.Column('key', sa.Unicode(128), primary_key=True),
+        sa.Column('section', sa.Unicode(128), primary_key=True),
+        sa.Column('value', sa.dialects.postgresql.JSON,
+                  server_default="{}")
+    )
+
+    op.create_table(
+        'plugin_configs',
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('plugin_name', sa.Unicode(128)),
+        sa.Column('section', sa.Unicode(128)),
+        sa.Column('config', sa.dialects.postgresql.JSON,
+                  server_default="{}"),
+        sa.Column('resource_id', sa.Integer(),
+                  sa.ForeignKey('resources.resource_id', onupdate='cascade',
+                                ondelete='cascade')),
+        sa.Column('owner_id', sa.Integer(),
+                  sa.ForeignKey('users.id', onupdate='cascade',
+                                ondelete='cascade')))
+
+    op.create_table(
+        'rc_versions',
+        sa.Column('name', sa.Unicode(40), primary_key=True),
+        sa.Column('value', sa.Unicode(40)),
+    )
+    version_table = sa.table('rc_versions',
+                             sa.Column('name', sa.Unicode(40)),
+                             sa.Column('value', sa.Unicode(40)))
+
+    insert = version_table.insert().values(name='es_reports')
+    op.execute(insert)
+    insert = version_table.insert().values(name='es_reports_groups')
+    op.execute(insert)
+    insert = version_table.insert().values(name='es_reports_stats')
+    op.execute(insert)
+    insert = version_table.insert().values(name='es_logs')
+    op.execute(insert)
+    insert = version_table.insert().values(name='es_metrics')
+    op.execute(insert)
+    insert = version_table.insert().values(name='es_slow_calls')
+    op.execute(insert)
+
+
+    op.execute('''
+        CREATE OR REPLACE FUNCTION floor_time_5min(timestamp without time zone)
+          RETURNS timestamp without time zone AS
+        $BODY$SELECT date_trunc('hour', $1) + INTERVAL '5 min' * FLOOR(date_part('minute', $1) / 5.0)$BODY$
+          LANGUAGE sql VOLATILE;
+    ''')
+
+    op.execute('''
+    CREATE OR REPLACE FUNCTION partition_logs() RETURNS trigger
+        LANGUAGE plpgsql SECURITY DEFINER
+        AS $$
+    DECLARE
+    main_table         varchar := 'logs';
+    partitioned_table  varchar := '';
+    BEGIN
+
+    IF NEW.permanent THEN
+        partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp);
+    ELSE
+        partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp) || '_' || DATE_part('day', NEW.timestamp);
+    END IF;
+
+    BEGIN
+    EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+    EXCEPTION
+    WHEN undefined_table THEN
+        RAISE NOTICE 'A partition has been created %', partitioned_table;
+        IF NEW.permanent THEN
+        EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
+                partitioned_table,
+                quote_literal(date_trunc('month', NEW.timestamp)::date) ,
+                quote_literal((date_trunc('month', NEW.timestamp)::date  + interval '1 month')::text),
+                main_table);
+        EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(log_id);', partitioned_table, partitioned_table);
+        EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_namespace_resource_id ON %s (namespace, resource_id);', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_pkey_namespace ON %s (primary_key, namespace);', partitioned_table, partitioned_table);
+        ELSE
+        EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
+                partitioned_table,
+                quote_literal(date_trunc('day', NEW.timestamp)::date) ,
+                quote_literal((date_trunc('day', NEW.timestamp)::date  + interval '1 day')::text),
+                main_table);
+        EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s_ PRIMARY KEY(log_id);', partitioned_table, partitioned_table);
+        EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_namespace_resource_id ON %s (namespace, resource_id);', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_primary_key_namespace ON %s (primary_key,namespace);', partitioned_table, partitioned_table);
+        END IF;
+
+
+        EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+    END;
+
+
+    RETURN NULL;
+    END
+    $$;
+    ''')
+
+    op.execute('''
+    CREATE TRIGGER partition_logs BEFORE INSERT ON logs FOR EACH ROW EXECUTE PROCEDURE partition_logs();
+    ''')
+
+    op.execute('''
+    CREATE OR REPLACE FUNCTION partition_metrics() RETURNS trigger
+        LANGUAGE plpgsql SECURITY DEFINER
+        AS $$
+        DECLARE
+        main_table         varchar := 'metrics';
+        partitioned_table  varchar := '';
+        BEGIN
+
+        partitioned_table := main_table || '_p_' || date_part('year', NEW.timestamp)::TEXT || '_' || DATE_part('month', NEW.timestamp) || '_' || DATE_part('day', NEW.timestamp);
+
+    BEGIN
+    EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+    EXCEPTION
+    WHEN undefined_table THEN
+        RAISE NOTICE 'A partition has been created %', partitioned_table;
+        EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( timestamp >= DATE %s AND timestamp < DATE %s)) INHERITS (%s)',
+                partitioned_table,
+                quote_literal(date_trunc('day', NEW.timestamp)::date) ,
+                quote_literal((date_trunc('day', NEW.timestamp)::date  + interval '1 day')::text),
+                main_table);
+        EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(pkey);', partitioned_table, partitioned_table);
+        EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s (timestamp);', partitioned_table, partitioned_table);
+        EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s (resource_id);', partitioned_table, partitioned_table);
+        EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+    END;
+
+    RETURN NULL;
+    END
+    $$;
+    ''')
+
+    op.execute('''
+    CREATE TRIGGER partition_metrics BEFORE INSERT ON metrics FOR EACH ROW EXECUTE PROCEDURE partition_metrics();
+    ''')
+
+    op.execute('''
+    CREATE FUNCTION partition_reports_stats() RETURNS trigger
+        LANGUAGE plpgsql SECURITY DEFINER
+        AS $$
+        DECLARE
+            main_table         varchar := 'reports_stats';
+            partitioned_table  varchar := '';
+        BEGIN
+
+            partitioned_table := main_table || '_p_' || date_part('year', NEW.start_interval)::TEXT || '_' || DATE_part('month', NEW.start_interval);
+
+            BEGIN
+                EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+            EXCEPTION
+            WHEN undefined_table THEN
+                RAISE NOTICE 'A partition has been created %', partitioned_table;
+                EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( start_interval >= DATE %s AND start_interval < DATE %s )) INHERITS (%s)',
+                    partitioned_table,
+                    quote_literal(date_trunc('month', NEW.start_interval)::date) ,
+                    quote_literal((date_trunc('month', NEW.start_interval)::date  + interval '1 month')::text),
+                    main_table);
+                EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
+                EXECUTE format('CREATE INDEX ix_%s_start_interval ON %s USING btree (start_interval);', partitioned_table, partitioned_table);
+                EXECUTE format('CREATE INDEX ix_%s_type ON %s USING btree (type);', partitioned_table, partitioned_table);
+                EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
+                EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+            END;
+            RETURN NULL;
+        END
+        $$;
+    ''')
+
+    op.execute('''
+    CREATE TRIGGER partition_reports_stats BEFORE INSERT ON reports_stats FOR EACH ROW EXECUTE PROCEDURE partition_reports_stats();
+    ''')
+
+    op.execute('''
+    CREATE OR REPLACE FUNCTION partition_reports_groups() RETURNS trigger
+        LANGUAGE plpgsql SECURITY DEFINER
+        AS $$
+    DECLARE
+        main_table         varchar := 'reports_groups';
+        partitioned_table  varchar := '';
+    BEGIN
+
+        partitioned_table := main_table || '_p_' || date_part('year', NEW.first_timestamp)::TEXT || '_' || DATE_part('month', NEW.first_timestamp);
+
+        BEGIN
+            EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+        EXCEPTION
+        WHEN undefined_table THEN
+            RAISE NOTICE 'A partition has been created %', partitioned_table;
+            EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( first_timestamp >= DATE %s AND first_timestamp < DATE %s )) INHERITS (%s)',
+                partitioned_table,
+                quote_literal(date_trunc('month', NEW.first_timestamp)::date) ,
+                quote_literal((date_trunc('month', NEW.first_timestamp)::date  + interval '1 month')::text),
+                main_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
+            EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+        END;
+        RETURN NULL;
+    END
+    $$;
+    ''')
+
+    op.execute('''
+    CREATE TRIGGER partition_reports_groups BEFORE INSERT ON reports_groups FOR EACH ROW EXECUTE PROCEDURE partition_reports_groups();
+    ''')
+
+    op.execute('''
+    CREATE OR REPLACE FUNCTION partition_reports() RETURNS trigger
+        LANGUAGE plpgsql SECURITY DEFINER
+        AS $$
+    DECLARE
+        main_table         varchar := 'reports';
+        partitioned_table  varchar := '';
+        partitioned_parent_table  varchar := '';
+    BEGIN
+
+        partitioned_table := main_table || '_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
+        partitioned_parent_table := 'reports_groups_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
+
+        BEGIN
+            EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+        EXCEPTION
+        WHEN undefined_table THEN
+            RAISE NOTICE 'A partition has been created %', partitioned_table;
+            EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( report_group_time >= DATE %s AND report_group_time < DATE %s )) INHERITS (%s)',
+                partitioned_table,
+                quote_literal(date_trunc('month', NEW.report_group_time)::date) ,
+                quote_literal((date_trunc('month', NEW.report_group_time)::date  + interval '1 month')::text),
+                main_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_group_id FOREIGN KEY (group_id) REFERENCES %s (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table, partitioned_parent_table);
+            EXECUTE format('CREATE INDEX ix_%s_report_group_time ON %s USING btree (report_group_time);', partitioned_table, partitioned_table);
+            EXECUTE format('CREATE INDEX ix_%s_group_id ON %s USING btree (group_id);', partitioned_table, partitioned_table);
+            EXECUTE format('CREATE INDEX ix_%s_resource_id ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
+            EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+        END;
+        RETURN NULL;
+    END
+    $$;
+    ''')
+
+    op.execute('''
+    CREATE TRIGGER partition_reports BEFORE INSERT ON reports FOR EACH ROW EXECUTE PROCEDURE partition_reports();
+    ''')
+
+
+    op.execute('''
+    CREATE OR REPLACE FUNCTION partition_slow_calls() RETURNS trigger
+        LANGUAGE plpgsql SECURITY DEFINER
+        AS $$
+    DECLARE
+        main_table         varchar := 'slow_calls';
+        partitioned_table  varchar := '';
+        partitioned_parent_table  varchar := '';
+    BEGIN
+
+        partitioned_table := main_table || '_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
+        partitioned_parent_table := 'reports_p_' || date_part('year', NEW.report_group_time)::TEXT || '_' || DATE_part('month', NEW.report_group_time);
+
+        BEGIN
+            EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+        EXCEPTION
+        WHEN undefined_table THEN
+            RAISE NOTICE 'A partition has been created %', partitioned_table;
+            EXECUTE format('CREATE TABLE  IF NOT EXISTS %s ( CHECK( report_group_time >= DATE %s AND report_group_time < DATE %s )) INHERITS (%s)',
+                partitioned_table,
+                quote_literal(date_trunc('month', NEW.report_group_time)::date) ,
+                quote_literal((date_trunc('month', NEW.report_group_time)::date  + interval '1 month')::text),
+                main_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT pk_%s PRIMARY KEY(id);', partitioned_table, partitioned_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_resource_id FOREIGN KEY (resource_id) REFERENCES resources (resource_id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table);
+            EXECUTE format('ALTER TABLE %s ADD CONSTRAINT fk_%s_report_id FOREIGN KEY (report_id) REFERENCES %s (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE;', partitioned_table, partitioned_table, partitioned_parent_table);
+            EXECUTE format('CREATE INDEX ix_%s_resource_id  ON %s USING btree (resource_id);', partitioned_table, partitioned_table);
+            EXECUTE format('CREATE INDEX ix_%s_report_id ON %s USING btree (report_id);', partitioned_table, partitioned_table);
+            EXECUTE format('CREATE INDEX ix_%s_timestamp ON %s USING btree (timestamp);', partitioned_table, partitioned_table);
+            EXECUTE 'INSERT INTO ' || partitioned_table || ' SELECT(' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').*;';
+        END;
+        RETURN NULL;
+    END
+    $$;
+    ''')
+
+    op.execute('''
+    CREATE TRIGGER partition_slow_calls BEFORE INSERT ON slow_calls FOR EACH ROW EXECUTE PROCEDURE partition_slow_calls();
+    ''')
+
+def downgrade():
+    pass

backend/src/appenlight/models/__init__.py

@@ -0,0 +1,135 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import MetaData
+from sqlalchemy.orm import scoped_session
+from sqlalchemy.orm import sessionmaker
+from zope.sqlalchemy import ZopeTransactionExtension
+import ziggurat_foundations
+from ziggurat_foundations.models.base import get_db_session
+
+log = logging.getLogger(__name__)
+
+DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
+
+NAMING_CONVENTION = {
+    "ix": 'ix_%(column_0_label)s',
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s"
+}
+
+metadata = MetaData(naming_convention=NAMING_CONVENTION)
+Base = declarative_base(metadata=metadata)
+
+# optional for request.db approach
+ziggurat_foundations.models.DBSession = DBSession
+
+
+class Datastores(object):
+    redis = None
+    es = None
+
+
+def register_datastores(es_conn, redis_conn, redis_lockmgr):
+    Datastores.es = es_conn
+    Datastores.redis = redis_conn
+    Datastores.lockmgr = redis_lockmgr
+
+
+class SliceableESQuery(object):
+    def __init__(self, query, sort_query=None, aggregations=False, **kwconfig):
+        self.query = query
+        self.sort_query = sort_query
+        self.aggregations = aggregations
+        self.items_per_page = kwconfig.pop('items_per_page', 10)
+        self.page = kwconfig.pop('page', 1)
+        self.kwconfig = kwconfig
+        self.result = None
+
+    def __getitem__(self, index):
+        config = self.kwconfig.copy()
+        config['es_from'] = index.start
+        query = self.query.copy()
+        if self.sort_query:
+            query.update(self.sort_query)
+        self.result = Datastores.es.search(query, size=self.items_per_page,
+                                           **config)
+        if self.aggregations:
+            self.items = self.result.get('aggregations')
+        else:
+            self.items = self.result['hits']['hits']
+
+        return self.items
+
+    def __iter__(self):
+        return self.result
+
+    def __len__(self):
+        config = self.kwconfig.copy()
+        query = self.query.copy()
+        self.result = Datastores.es.search(query, size=self.items_per_page,
+                                           **config)
+        if self.aggregations:
+            self.items = self.result.get('aggregations')
+        else:
+            self.items = self.result['hits']['hits']
+
+        count = int(self.result['hits']['total'])
+        return count if count < 5000 else 5000
+
+
+from appenlight.models.resource import Resource
+from appenlight.models.application import Application
+from appenlight.models.user import User
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.alert_channel_action import AlertChannelAction
+from appenlight.models.request_metric import Metric
+from appenlight.models.application_postprocess_conf import \
+    ApplicationPostprocessConf
+from appenlight.models.auth_token import AuthToken
+from appenlight.models.event import Event
+from appenlight.models.external_identity import ExternalIdentity
+from appenlight.models.group import Group
+from appenlight.models.group_permission import GroupPermission
+from appenlight.models.group_resource_permission import GroupResourcePermission
+from appenlight.models.log import Log
+from appenlight.models.plugin_config import PluginConfig
+from appenlight.models.report import Report
+from appenlight.models.report_group import ReportGroup
+from appenlight.models.report_comment import ReportComment
+from appenlight.models.report_assignment import ReportAssignment
+from appenlight.models.report_stat import ReportStat
+from appenlight.models.slow_call import SlowCall
+from appenlight.models.tag import Tag
+from appenlight.models.user_group import UserGroup
+from appenlight.models.user_permission import UserPermission
+from appenlight.models.user_resource_permission import UserResourcePermission
+from ziggurat_foundations import ziggurat_model_init
+
+ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission,
+                    UserResourcePermission, GroupResourcePermission,
+                    Resource,
+                    ExternalIdentity, passwordmanager=None)

backend/src/appenlight/models/alert_channel.py

@@ -0,0 +1,296 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import sqlalchemy as sa
+import urllib.request, urllib.parse, urllib.error
+from datetime import timedelta
+from appenlight.models import Base
+from appenlight.lib.utils.date_utils import convert_date
+from sqlalchemy.dialects.postgresql import JSON
+from ziggurat_foundations.models.base import BaseModel
+
+log = logging.getLogger(__name__)
+
+#
+channel_rules_m2m_table = sa.Table(
+    'channels_actions', Base.metadata,
+    sa.Column('channel_pkey', sa.Integer,
+              sa.ForeignKey('alert_channels.pkey')),
+    sa.Column('action_pkey', sa.Integer,
+              sa.ForeignKey('alert_channels_actions.pkey'))
+)
+
+DATE_FRMT = '%Y-%m-%dT%H:%M'
+
+
+class AlertChannel(Base, BaseModel):
+    """
+    Stores information about possible alerting options
+    """
+    __tablename__ = 'alert_channels'
+    __possible_channel_names__ = ['email']
+    __mapper_args__ = {
+        'polymorphic_on': 'channel_name',
+        'polymorphic_identity': 'integration'
+    }
+
+    owner_id = sa.Column(sa.Unicode(30),
+                         sa.ForeignKey('users.id', onupdate='CASCADE',
+                                       ondelete='CASCADE'))
+    channel_name = sa.Column(sa.Unicode(25), nullable=False)
+    channel_value = sa.Column(sa.Unicode(80), nullable=False, default='')
+    channel_json_conf = sa.Column(JSON(), nullable=False, default='')
+    channel_validated = sa.Column(sa.Boolean, nullable=False,
+                                  default=False)
+    send_alerts = sa.Column(sa.Boolean, nullable=False,
+                            default=True)
+    daily_digest = sa.Column(sa.Boolean, nullable=False,
+                             default=True)
+    integration_id = sa.Column(sa.Integer, sa.ForeignKey('integrations.id'),
+                               nullable=True)
+    pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
+
+    channel_actions = sa.orm.relationship('AlertChannelAction',
+                                          cascade="all",
+                                          passive_deletes=True,
+                                          passive_updates=True,
+                                          secondary=channel_rules_m2m_table,
+                                          backref='channels')
+
+    @property
+    def channel_visible_value(self):
+        if self.integration:
+            return '{}: {}'.format(
+                self.channel_name,
+                self.integration.resource.resource_name
+            )
+
+        return '{}: {}'.format(
+            self.channel_name,
+            self.channel_value
+        )
+
+    def get_dict(self, exclude_keys=None, include_keys=None,
+                 extended_info=True):
+        """
+        Returns dictionary with required information that will be consumed by
+        angular
+        """
+        instance_dict = super(AlertChannel, self).get_dict(exclude_keys,
+                                                           include_keys)
+        exclude_keys_list = exclude_keys or []
+        include_keys_list = include_keys or []
+
+        instance_dict['supports_report_alerting'] = True
+        instance_dict['channel_visible_value'] = self.channel_visible_value
+
+        if extended_info:
+            instance_dict['actions'] = [
+                rule.get_dict(extended_info=True) for
+                rule in self.channel_actions]
+
+        del instance_dict['channel_json_conf']
+
+        if self.integration:
+            instance_dict[
+                'supports_report_alerting'] = \
+                self.integration.supports_report_alerting
+        d = {}
+        for k in instance_dict.keys():
+            if (k not in exclude_keys_list and
+                    (k in include_keys_list or not include_keys)):
+                d[k] = instance_dict[k]
+        return d
+
+    def __repr__(self):
+        return '<AlertChannel: (%s,%s), user:%s>' % (self.channel_name,
+                                                     self.channel_value,
+                                                     self.user_name,)
+
+    def send_digest(self, **kwargs):
+        """
+        This should implement daily top error report notifications
+        """
+        log.warning('send_digest NOT IMPLEMENTED')
+
+    def notify_reports(self, **kwargs):
+        """
+        This should implement notification of reports that occured in 1 min
+        interval
+        """
+        log.warning('notify_reports NOT IMPLEMENTED')
+
+    def notify_alert(self, **kwargs):
+        """
+        Notify user of report/uptime/chart threshold events based on events alert
+        type
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        alert_name = kwargs['event'].unified_alert_name()
+        if alert_name in ['slow_report_alert', 'error_report_alert']:
+            self.notify_report_alert(**kwargs)
+        elif alert_name == 'uptime_alert':
+            self.notify_uptime_alert(**kwargs)
+        elif alert_name == 'chart_alert':
+            self.notify_chart_alert(**kwargs)
+
+    def notify_chart_alert(self, **kwargs):
+        """
+        This should implement report open/close alerts notifications
+        """
+        log.warning('notify_chart_alert NOT IMPLEMENTED')
+
+    def notify_report_alert(self, **kwargs):
+        """
+        This should implement report open/close alerts notifications
+        """
+        log.warning('notify_report_alert NOT IMPLEMENTED')
+
+    def notify_uptime_alert(self, **kwargs):
+        """
+        This should implement uptime open/close alerts notifications
+        """
+        log.warning('notify_uptime_alert NOT IMPLEMENTED')
+
+    def get_notification_basic_vars(self, kwargs):
+        """
+        Sets most common variables used later for rendering notifications for
+        channel
+        """
+        if 'event' in kwargs:
+            kwargs['since_when'] = kwargs['event'].start_date
+
+        url_start_date = kwargs.get('since_when') - timedelta(minutes=1)
+        url_end_date = kwargs.get('since_when') + timedelta(minutes=4)
+        tmpl_vars = {
+            "timestamp": kwargs['since_when'],
+            "user": kwargs['user'],
+            "since_when": kwargs.get('since_when'),
+            "url_start_date": url_start_date,
+            "url_end_date": url_end_date
+        }
+        tmpl_vars["resource_name"] = kwargs['resource'].resource_name
+        tmpl_vars["resource"] = kwargs['resource']
+
+        if 'event' in kwargs:
+            tmpl_vars['event_values'] = kwargs['event'].values
+            tmpl_vars['alert_type'] = kwargs['event'].unified_alert_name()
+            tmpl_vars['alert_action'] = kwargs['event'].unified_alert_action()
+        return tmpl_vars
+
+    def report_alert_notification_vars(self, kwargs):
+        tmpl_vars = self.get_notification_basic_vars(kwargs)
+        reports = kwargs.get('reports', [])
+        tmpl_vars["reports"] = reports
+        tmpl_vars["confirmed_total"] = len(reports)
+
+        tmpl_vars["report_type"] = "error reports"
+        tmpl_vars["url_report_type"] = 'report'
+
+        alert_type = tmpl_vars.get('alert_type', '')
+        if 'slow_report' in alert_type:
+            tmpl_vars["report_type"] = "slow reports"
+            tmpl_vars["url_report_type"] = 'report/list_slow'
+
+        app_url = kwargs['request'].registry.settings['_mail_url']
+
+        destination_url = kwargs['request'].route_url('/',
+                                                      _app_url=app_url)
+        if alert_type:
+            destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
+                tmpl_vars["url_report_type"],
+                tmpl_vars['resource'].resource_id,
+                tmpl_vars['url_start_date'].strftime(DATE_FRMT),
+                tmpl_vars['url_end_date'].strftime(DATE_FRMT)
+            )
+        else:
+            destination_url += 'ui/{}?resource={}'.format(
+                tmpl_vars["url_report_type"],
+                tmpl_vars['resource'].resource_id
+            )
+        tmpl_vars["destination_url"] = destination_url
+
+        return tmpl_vars
+
+    def uptime_alert_notification_vars(self, kwargs):
+        tmpl_vars = self.get_notification_basic_vars(kwargs)
+        app_url = kwargs['request'].registry.settings['_mail_url']
+        destination_url = kwargs['request'].route_url('/', _app_url=app_url)
+        destination_url += 'ui/{}?resource={}'.format(
+            'uptime',
+            tmpl_vars['resource'].resource_id)
+        tmpl_vars['destination_url'] = destination_url
+
+        reason = ''
+        e_values = tmpl_vars.get('event_values')
+
+        if e_values and e_values.get('response_time') == 0:
+            reason += ' Response time was slower than 20 seconds.'
+        elif e_values:
+            code = e_values.get('status_code')
+            reason += ' Response status code: %s.' % code
+
+        tmpl_vars['reason'] = reason
+        return tmpl_vars
+
+    def chart_alert_notification_vars(self, kwargs):
+        tmpl_vars = self.get_notification_basic_vars(kwargs)
+        tmpl_vars['chart_name'] = tmpl_vars['event_values']['chart_name']
+        tmpl_vars['action_name'] = tmpl_vars['event_values'].get(
+            'action_name') or ''
+        matched_values = tmpl_vars['event_values']['matched_step_values']
+        tmpl_vars['readable_values'] = []
+        for key, value in list(matched_values['values'].items()):
+            matched_label = matched_values['labels'].get(key)
+            if matched_label:
+                tmpl_vars['readable_values'].append({
+                    'label': matched_label['human_label'],
+                    'value': value
+                })
+        tmpl_vars['readable_values'] = sorted(tmpl_vars['readable_values'],
+                                              key=lambda x: x['label'])
+        start_date = convert_date(tmpl_vars['event_values']['start_interval'])
+        end_date = None
+        if tmpl_vars['event_values'].get('end_interval'):
+            end_date = convert_date(tmpl_vars['event_values']['end_interval'])
+
+        app_url = kwargs['request'].registry.settings['_mail_url']
+        destination_url = kwargs['request'].route_url('/', _app_url=app_url)
+        to_encode = {
+            'resource': tmpl_vars['event_values']['resource'],
+            'start_date': start_date.strftime(DATE_FRMT),
+        }
+        if end_date:
+            to_encode['end_date'] = end_date.strftime(DATE_FRMT)
+
+        destination_url += 'ui/{}?{}'.format(
+            'logs',
+            urllib.parse.urlencode(to_encode)
+        )
+        tmpl_vars['destination_url'] = destination_url
+        return tmpl_vars

backend/src/appenlight/models/alert_channel_action.py

@@ -0,0 +1,84 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+
+from appenlight.models.resource import Resource
+from appenlight.models import Base, get_db_session
+from sqlalchemy.orm import validates
+from ziggurat_foundations.models.base import BaseModel
+
+
+class AlertChannelAction(Base, BaseModel):
+    """
+    Stores notifications conditions for user's alert channels
+    This is later used for rule parsing like "alert if http_status == 500"
+    """
+    __tablename__ = 'alert_channels_actions'
+
+    types = ['report', 'chart']
+
+    owner_id = sa.Column(sa.Integer,
+                         sa.ForeignKey('users.id', onupdate='CASCADE',
+                                       ondelete='CASCADE'))
+    resource_id = sa.Column(sa.Integer())
+    action = sa.Column(sa.Unicode(10), nullable=False, default='always')
+    type = sa.Column(sa.Unicode(10), nullable=False)
+    other_id = sa.Column(sa.Unicode(40))
+    pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
+    rule = sa.Column(sa.dialects.postgresql.JSON,
+                     nullable=False, default={'field': 'http_status',
+                                              "op": "ge", "value": "500"})
+    config = sa.Column(sa.dialects.postgresql.JSON)
+    name = sa.Column(sa.Unicode(255))
+
+    @validates('notify_type')
+    def validate_email(self, key, notify_type):
+        assert notify_type in ['always', 'only_first']
+        return notify_type
+
+    def resource_name(self, db_session=None):
+        db_session = get_db_session(db_session)
+        if self.resource_id:
+            return Resource.by_resource_id(self.resource_id,
+                                           db_session=db_session).resource_name
+        else:
+            return 'any resource'
+
+    def get_dict(self, exclude_keys=None, include_keys=None,
+                 extended_info=False):
+        """
+        Returns dictionary with required information that will be consumed by
+        angular
+        """
+        instance_dict = super(AlertChannelAction, self).get_dict()
+        exclude_keys_list = exclude_keys or []
+        include_keys_list = include_keys or []
+        if extended_info:
+            instance_dict['channels'] = [
+                c.get_dict(extended_info=False) for c in self.channels]
+
+        d = {}
+        for k in instance_dict.keys():
+            if (k not in exclude_keys_list and
+                    (k in include_keys_list or not include_keys)):
+                d[k] = instance_dict[k]
+        return d

backend/src/appenlight/models/alert_channels/__init__.py

@@ -0,0 +1,21 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+

backend/src/appenlight/models/alert_channels/campfire.py

@@ -0,0 +1,193 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.integrations.campfire import CampfireIntegration
+from webhelpers2.text import truncate
+
+log = logging.getLogger(__name__)
+
+
+class CampfireAlertChannel(AlertChannel):
+    __mapper_args__ = {
+        'polymorphic_identity': 'campfire'
+    }
+
+    @property
+    def client(self):
+        client = CampfireIntegration.create_client(
+            self.integration.config['api_token'],
+            self.integration.config['account'])
+        return client
+
+    def notify_reports(self, **kwargs):
+        """
+        Notify user of individual reports
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        app_url = kwargs['request'].registry.settings['_mail_url']
+        destination_url = kwargs['request'].route_url('/',
+                                                      app_url=app_url)
+        f_args = ('report',
+                  template_vars['resource'].resource_id,
+                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
+                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
+        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
+            *f_args)
+
+        if template_vars['confirmed_total'] > 1:
+            template_vars["title"] = "%s - %s reports" % (
+                template_vars['resource_name'],
+                template_vars['confirmed_total'],
+            )
+        else:
+            error_title = truncate(template_vars['reports'][0][1].error or
+                                   'slow report', 90)
+            template_vars["title"] = "%s - '%s' report" % (
+                template_vars['resource_name'],
+                error_title)
+
+        template_vars["title"] += ' ' + destination_url
+
+        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+        for room in self.integration.config['rooms'].split(','):
+            self.client.speak_to_room(room.strip(), template_vars["title"])
+
+    def notify_report_alert(self, **kwargs):
+        """
+        Build and send report alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        if kwargs['event'].unified_alert_action() == 'OPEN':
+            title = 'ALERT %s: %s - %s %s %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                kwargs['event'].values['reports'],
+                template_vars['report_type'],
+                template_vars['destination_url']
+            )
+
+        else:
+            title = 'ALERT %s: %s type: %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                template_vars['alert_type'].replace('_', ' '),
+            )
+        for room in self.integration.config['rooms'].split(','):
+            self.client.speak_to_room(room.strip(), title, sound='VUVUZELA')
+
+    def notify_uptime_alert(self, **kwargs):
+        """
+        Build and send uptime alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.uptime_alert_notification_vars(kwargs)
+
+        message = 'ALERT %s: %s has uptime issues %s\n\n' % (
+            template_vars['alert_action'],
+            template_vars['resource_name'],
+            template_vars['destination_url']
+        )
+        message += template_vars['reason']
+
+        for room in self.integration.config['rooms'].split(','):
+            self.client.speak_to_room(room.strip(), message, sound='VUVUZELA')
+
+    def send_digest(self, **kwargs):
+        """
+        Build and send daily digest notification
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+        f_args = (template_vars['resource_name'],
+                  template_vars['confirmed_total'],)
+        message = "Daily report digest: %s - %s reports" % f_args
+        message += '{}\n'.format(template_vars['destination_url'])
+        for room in self.integration.config['rooms'].split(','):
+            self.client.speak_to_room(room.strip(), message)
+
+        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+    def notify_chart_alert(self, **kwargs):
+        """
+        Build and send chart alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.chart_alert_notification_vars(kwargs)
+        message = 'ALERT {}: value in "{}" chart: ' \
+                  'met alert "{}" criteria {} \n'.format(
+            template_vars['alert_action'],
+            template_vars['chart_name'],
+            template_vars['action_name'],
+            template_vars['destination_url']
+        )
+
+        for item in template_vars['readable_values']:
+            message += '{}: {}\n'.format(item['label'], item['value'])
+
+        for room in self.integration.config['rooms'].split(','):
+            self.client.speak_to_room(room.strip(), message, sound='VUVUZELA')

backend/src/appenlight/models/alert_channels/email.py

@@ -0,0 +1,180 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.services.user import UserService
+from webhelpers2.text import truncate
+
+log = logging.getLogger(__name__)
+
+
+class EmailAlertChannel(AlertChannel):
+    """
+    Default email alerting channel
+    """
+
+    __mapper_args__ = {
+        'polymorphic_identity': 'email'
+    }
+
+    def notify_reports(self, **kwargs):
+        """
+        Notify user of individual reports
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        if template_vars['confirmed_total'] > 1:
+            template_vars["title"] = "App Enlight :: %s - %s reports" % (
+                template_vars['resource_name'],
+                template_vars['confirmed_total'],
+            )
+        else:
+            error_title = truncate(template_vars['reports'][0][1].error or
+                                   'slow report', 20)
+            template_vars["title"] = "App Enlight :: %s - '%s' report" % (
+                template_vars['resource_name'],
+                error_title)
+        UserService.send_email(kwargs['request'],
+                               [self.channel_value],
+                               template_vars,
+                               '/email_templates/notify_reports.jinja2')
+        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+    def send_digest(self, **kwargs):
+        """
+        Build and send daily digest notification
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+        title = "App Enlight :: Daily report digest: %s - %s reports"
+        template_vars["email_title"] = title % (
+            template_vars['resource_name'],
+            template_vars['confirmed_total'],
+        )
+
+        UserService.send_email(kwargs['request'],
+                               [self.channel_value],
+                               template_vars,
+                               '/email_templates/notify_reports.jinja2',
+                               immediately=True,
+                               silent=True)
+        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+    def notify_report_alert(self, **kwargs):
+        """
+        Build and send report alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        if kwargs['event'].unified_alert_action() == 'OPEN':
+            title = 'App Enlight :: ALERT %s: %s - %s %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                kwargs['event'].values['reports'],
+                template_vars['report_type'],
+            )
+        else:
+            title = 'App Enlight :: ALERT %s: %s type: %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                template_vars['alert_type'].replace('_', ' '),
+            )
+        template_vars['email_title'] = title
+        UserService.send_email(kwargs['request'], [self.channel_value],
+                               template_vars,
+                               '/email_templates/alert_reports.jinja2')
+
+    def notify_uptime_alert(self, **kwargs):
+        """
+        Build and send uptime alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.uptime_alert_notification_vars(kwargs)
+        title = 'App Enlight :: ALERT %s: %s has uptime issues' % (
+            template_vars['alert_action'],
+            template_vars['resource_name'],
+        )
+        template_vars['email_title'] = title
+
+        UserService.send_email(kwargs['request'], [self.channel_value],
+                               template_vars,
+                               '/email_templates/alert_uptime.jinja2')
+
+    def notify_chart_alert(self, **kwargs):
+        """
+        Build and send chart alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.chart_alert_notification_vars(kwargs)
+
+        title = 'App Enlight :: ALERT {} value in "{}" chart' \
+                ' met alert "{}" criteria'.format(
+            template_vars['alert_action'],
+            template_vars['chart_name'],
+            template_vars['action_name'],
+        )
+        template_vars['email_title'] = title
+        UserService.send_email(kwargs['request'], [self.channel_value],
+                               template_vars,
+                               '/email_templates/alert_chart.jinja2')

backend/src/appenlight/models/alert_channels/flowdock.py

@@ -0,0 +1,238 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.integrations.flowdock import FlowdockIntegration
+from webhelpers2.text import truncate
+
+log = logging.getLogger(__name__)
+
+
+class FlowdockAlertChannel(AlertChannel):
+    __mapper_args__ = {
+        'polymorphic_identity': 'flowdock'
+    }
+
+    def notify_reports(self, **kwargs):
+        """
+        Notify user of individual reports
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        app_url = kwargs['request'].registry.settings['_mail_url']
+        destination_url = kwargs['request'].route_url('/',
+                                                      _app_url=app_url)
+        f_args = ('report',
+                  template_vars['resource'].resource_id,
+                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
+                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
+        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
+            *f_args)
+
+        if template_vars['confirmed_total'] > 1:
+            template_vars["title"] = "%s - %s reports" % (
+                template_vars['resource_name'],
+                template_vars['confirmed_total'],
+            )
+        else:
+            error_title = truncate(template_vars['reports'][0][1].error or
+                                   'slow report', 90)
+            template_vars["title"] = "%s - '%s' report" % (
+                template_vars['resource_name'],
+                error_title)
+
+        log_msg = 'NOTIFY : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+        client = FlowdockIntegration.create_client(
+            self.integration.config['api_token'])
+        payload = {
+            "source": "App Enlight",
+            "from_address": kwargs['request'].registry.settings[
+                'mailing.from_email'],
+            "subject": template_vars["title"],
+            "content": "New report present",
+            "tags": ["appenlight"],
+            "link": destination_url
+        }
+        client.send_to_inbox(payload)
+
+    def notify_report_alert(self, **kwargs):
+        """
+        Build and send report alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        if kwargs['event'].unified_alert_action() == 'OPEN':
+
+            title = 'ALERT %s: %s - %s %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                kwargs['event'].values['reports'],
+                template_vars['report_type'],
+            )
+
+        else:
+            title = 'ALERT %s: %s type: %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                template_vars['alert_type'].replace('_', ' '),
+            )
+
+        client = FlowdockIntegration.create_client(
+            self.integration.config['api_token'])
+        payload = {
+            "source": "App Enlight",
+            "from_address": kwargs['request'].registry.settings[
+                'mailing.from_email'],
+            "subject": title,
+            "content": 'Investigation required',
+            "tags": ["appenlight", "alert", template_vars['alert_type']],
+            "link": template_vars['destination_url']
+        }
+        client.send_to_inbox(payload)
+
+    def notify_uptime_alert(self, **kwargs):
+        """
+        Build and send uptime alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.uptime_alert_notification_vars(kwargs)
+
+        message = 'ALERT %s: %s has uptime issues' % (
+            template_vars['alert_action'],
+            template_vars['resource_name'],
+        )
+        submessage = 'Info: '
+        submessage += template_vars['reason']
+
+        client = FlowdockIntegration.create_client(
+            self.integration.config['api_token'])
+        payload = {
+            "source": "App Enlight",
+            "from_address": kwargs['request'].registry.settings[
+                'mailing.from_email'],
+            "subject": message,
+            "content": submessage,
+            "tags": ["appenlight", "alert", 'uptime'],
+            "link": template_vars['destination_url']
+        }
+        client.send_to_inbox(payload)
+
+    def send_digest(self, **kwargs):
+        """
+        Build and send daily digest notification
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+        message = "Daily report digest: %s - %s reports" % (
+            template_vars['resource_name'], template_vars['confirmed_total'])
+
+        f_args = (template_vars['confirmed_total'],
+                  template_vars['timestamp'])
+
+        payload = {
+            "source": "App Enlight",
+            "from_address": kwargs['request'].registry.settings[
+                'mailing.from_email'],
+            "subject": message,
+            "content": '%s reports in total since %s' % f_args,
+            "tags": ["appenlight", "digest"],
+            "link": template_vars['destination_url']
+        }
+
+        client = FlowdockIntegration.create_client(
+            self.integration.config['api_token'])
+        client.send_to_inbox(payload)
+
+        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+    def notify_chart_alert(self, **kwargs):
+        """
+        Build and send chart alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.chart_alert_notification_vars(kwargs)
+
+        message = 'ALERT {}: value in "{}" chart ' \
+                  'met alert "{}" criteria'.format(
+            template_vars['alert_action'],
+            template_vars['chart_name'],
+            template_vars['action_name'],
+        )
+        submessage = 'Info: '
+        for item in template_vars['readable_values']:
+            submessage += '{}: {}\n'.format(item['label'], item['value'])
+
+        client = FlowdockIntegration.create_client(
+            self.integration.config['api_token'])
+        payload = {
+            "source": "App Enlight",
+            "from_address": kwargs['request'].registry.settings[
+                'mailing.from_email'],
+            "subject": message,
+            "content": submessage,
+            "tags": ["appenlight", "alert", 'chart'],
+            "link": template_vars['destination_url']
+        }
+        client.send_to_inbox(payload)

backend/src/appenlight/models/alert_channels/hipchat.py

@@ -0,0 +1,234 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.integrations.hipchat import HipchatIntegration
+from webhelpers2.text import truncate
+
+log = logging.getLogger(__name__)
+
+
+class HipchatAlertChannel(AlertChannel):
+    __mapper_args__ = {
+        'polymorphic_identity': 'hipchat'
+    }
+
+    def notify_reports(self, **kwargs):
+        """
+        Notify user of individual reports
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        app_url = kwargs['request'].registry.settings['_mail_url']
+        destination_url = kwargs['request'].route_url('/',
+                                                      _app_url=app_url)
+        f_args = ('report',
+                  template_vars['resource'].resource_id,
+                  template_vars['url_start_date'].strftime('%Y-%m-%dT%H:%M'),
+                  template_vars['url_end_date'].strftime('%Y-%m-%dT%H:%M'))
+        destination_url += 'ui/{}?resource={}&start_date={}&end_date={}'.format(
+            *f_args)
+
+        if template_vars['confirmed_total'] > 1:
+            template_vars["title"] = "%s - %s reports" % (
+                template_vars['resource_name'],
+                template_vars['confirmed_total'],
+            )
+        else:
+            error_title = truncate(template_vars['reports'][0][1].error or
+                                   'slow report', 90)
+            template_vars["title"] = "%s - '%s' report" % (
+                template_vars['resource_name'],
+                error_title)
+
+        template_vars["title"] += ' ' + destination_url
+
+        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+        client = HipchatIntegration.create_client(
+            self.integration.config['api_token'])
+        for room in self.integration.config['rooms'].split(','):
+            client.send({
+                "message_format": 'text',
+                "message": template_vars["title"],
+                "from": "App Enlight",
+                "room_id": room.strip(),
+                "color": "yellow"
+            })
+
+    def notify_report_alert(self, **kwargs):
+        """
+        Build and send report alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        if kwargs['event'].unified_alert_action() == 'OPEN':
+
+            title = 'ALERT %s: %s - %s %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                kwargs['event'].values['reports'],
+                template_vars['report_type'],
+            )
+
+        else:
+            title = 'ALERT %s: %s type: %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+                template_vars['alert_type'].replace('_', ' '),
+            )
+
+        title += '\n ' + template_vars['destination_url']
+
+        api_token = self.integration.config['api_token']
+        client = HipchatIntegration.create_client(api_token)
+        for room in self.integration.config['rooms'].split(','):
+            client.send({
+                "message_format": 'text',
+                "message": title,
+                "from": "App Enlight",
+                "room_id": room.strip(),
+                "color": "red",
+                "notify": '1'
+            })
+
+    def notify_uptime_alert(self, **kwargs):
+        """
+        Build and send uptime alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.uptime_alert_notification_vars(kwargs)
+
+        message = 'ALERT %s: %s has uptime issues\n' % (
+            template_vars['alert_action'],
+            template_vars['resource_name'],
+        )
+        message += template_vars['reason']
+        message += '\n{}'.format(template_vars['destination_url'])
+
+        api_token = self.integration.config['api_token']
+        client = HipchatIntegration.create_client(api_token)
+        for room in self.integration.config['rooms'].split(','):
+            client.send({
+                "message_format": 'text',
+                "message": message,
+                "from": "App Enlight",
+                "room_id": room.strip(),
+                "color": "red",
+                "notify": '1'
+            })
+
+    def notify_chart_alert(self, **kwargs):
+        """
+        Build and send chart alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.chart_alert_notification_vars(kwargs)
+        message = 'ALERT {}: value in "{}" chart: ' \
+                  'met alert "{}" criteria\n'.format(
+            template_vars['alert_action'],
+            template_vars['chart_name'],
+            template_vars['action_name'],
+        )
+
+        for item in template_vars['readable_values']:
+            message += '{}: {}\n'.format(item['label'], item['value'])
+
+        message += template_vars['destination_url']
+
+        api_token = self.integration.config['api_token']
+        client = HipchatIntegration.create_client(api_token)
+        for room in self.integration.config['rooms'].split(','):
+            client.send({
+                "message_format": 'text',
+                "message": message,
+                "from": "App Enlight",
+                "room_id": room.strip(),
+                "color": "red",
+                "notify": '1'
+            })
+
+    def send_digest(self, **kwargs):
+        """
+        Build and send daily digest notification
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+        f_args = (template_vars['resource_name'],
+                  template_vars['confirmed_total'],)
+        message = "Daily report digest: %s - %s reports" % f_args
+        message += '\n{}'.format(template_vars['destination_url'])
+        api_token = self.integration.config['api_token']
+        client = HipchatIntegration.create_client(api_token)
+        for room in self.integration.config['rooms'].split(','):
+            client.send({
+                "message_format": 'text',
+                "message": message,
+                "from": "App Enlight",
+                "room_id": room.strip(),
+                "color": "green",
+                "notify": '1'
+            })
+
+        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)

backend/src/appenlight/models/alert_channels/slack.py

@@ -0,0 +1,290 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.integrations.slack import SlackIntegration
+from webhelpers2.text import truncate
+
+log = logging.getLogger(__name__)
+
+
+class SlackAlertChannel(AlertChannel):
+    __mapper_args__ = {
+        'polymorphic_identity': 'slack'
+    }
+
+    def notify_reports(self, **kwargs):
+        """
+        Notify user of individual reports
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+        template_vars["title"] = template_vars['resource_name']
+
+        if template_vars['confirmed_total'] > 1:
+            template_vars['subtext'] = '%s reports' % template_vars[
+                'confirmed_total']
+        else:
+            error_title = truncate(template_vars['reports'][0][1].error or
+                                   'slow report', 90)
+            template_vars['subtext'] = error_title
+
+        log_msg = 'NOTIFY  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)
+
+        client = SlackIntegration.create_client(
+            self.integration.config['webhook_url'])
+        report_data = {
+            "username": "App Enlight",
+            "icon_emoji": ":fire:",
+            "attachments": [
+                {
+                    "mrkdwn_in": ["text", "pretext", "title", "fallback"],
+                    "fallback": "*%s* - <%s| Browse>" % (
+                        template_vars["title"],
+                        template_vars['destination_url']),
+                    "pretext": "*%s* - <%s| Browse>" % (
+                        template_vars["title"],
+                        template_vars['destination_url']),
+                    "color": "warning",
+                    "fields": [
+                        {
+                            "value": 'Info: %s' % template_vars['subtext'],
+                            "short": False
+                        }
+                    ]
+                }
+            ]
+        }
+        client.make_request(data=report_data)
+
+    def notify_report_alert(self, **kwargs):
+        """
+        Build and send report alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+
+        if kwargs['event'].unified_alert_action() == 'OPEN':
+            title = '*ALERT %s*: %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name']
+            )
+
+            template_vars['subtext'] = 'Got at least %s %s' % (
+                kwargs['event'].values['reports'],
+                template_vars['report_type']
+            )
+
+        else:
+            title = '*ALERT %s*: %s' % (
+                template_vars['alert_action'],
+                template_vars['resource_name'],
+            )
+
+            template_vars['subtext'] = ''
+
+        alert_type = template_vars['alert_type'].replace('_', ' ')
+        alert_type = alert_type.replace('alert', '').capitalize()
+
+        template_vars['type'] = "Type: %s" % alert_type
+
+        client = SlackIntegration.create_client(
+            self.integration.config['webhook_url']
+        )
+        report_data = {
+            "username": "App Enlight",
+            "icon_emoji": ":rage:",
+            "attachments": [
+                {
+                    "mrkdwn_in": ["text", "pretext", "title", "fallback"],
+                    "fallback": "%s - <%s| Browse>" % (
+                        title, template_vars['destination_url']),
+                    "pretext": "%s - <%s| Browse>" % (
+                        title, template_vars['destination_url']),
+                    "color": "danger",
+                    "fields": [
+                        {
+                            "title": template_vars['type'],
+                            "value": template_vars['subtext'],
+                            "short": False
+                        }
+                    ]
+                }
+            ]
+        }
+        client.make_request(data=report_data)
+
+    def notify_uptime_alert(self, **kwargs):
+        """
+        Build and send uptime alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.uptime_alert_notification_vars(kwargs)
+
+        title = '*ALERT %s*: %s' % (
+            template_vars['alert_action'],
+            template_vars['resource_name'],
+        )
+        client = SlackIntegration.create_client(
+            self.integration.config['webhook_url']
+        )
+        report_data = {
+            "username": "App Enlight",
+            "icon_emoji": ":rage:",
+            "attachments": [
+                {
+                    "mrkdwn_in": ["text", "pretext", "title", "fallback"],
+                    "fallback": "{} - <{}| Browse>".format(
+                        title, template_vars['destination_url']),
+                    "pretext": "{} - <{}| Browse>".format(
+                        title, template_vars['destination_url']),
+                    "color": "danger",
+                    "fields": [
+                        {
+                            "title": "Application has uptime issues",
+                            "value": template_vars['reason'],
+                            "short": False
+                        }
+                    ]
+                }
+            ]
+        }
+        client.make_request(data=report_data)
+
+    def notify_chart_alert(self, **kwargs):
+        """
+        Build and send chart alert notification
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        template_vars = self.chart_alert_notification_vars(kwargs)
+
+        title = '*ALERT {}*: value in *"{}"* chart ' \
+                'met alert *"{}"* criteria'.format(
+            template_vars['alert_action'],
+            template_vars['chart_name'],
+            template_vars['action_name'],
+        )
+
+        subtext = ''
+        for item in template_vars['readable_values']:
+            subtext += '{} - {}\n'.format(item['label'], item['value'])
+
+        client = SlackIntegration.create_client(
+            self.integration.config['webhook_url']
+        )
+        report_data = {
+            "username": "App Enlight",
+            "icon_emoji": ":rage:",
+            "attachments": [
+                {"mrkdwn_in": ["text", "pretext", "title", "fallback"],
+                 "fallback": "{} - <{}| Browse>".format(
+                     title, template_vars['destination_url']),
+                 "pretext": "{} - <{}| Browse>".format(
+                     title, template_vars['destination_url']),
+                 "color": "danger",
+                 "fields": [
+                     {
+                         "title": "Following criteria were met:",
+                         "value": subtext,
+                         "short": False
+                     }
+                 ]
+                 }
+            ]
+        }
+        client.make_request(data=report_data)
+
+    def send_digest(self, **kwargs):
+        """
+        Build and send daily digest notification
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.report_alert_notification_vars(kwargs)
+        title = "*Daily report digest*: %s" % template_vars['resource_name']
+
+        subtext = '%s reports' % template_vars['confirmed_total']
+
+        client = SlackIntegration.create_client(
+            self.integration.config['webhook_url']
+        )
+        report_data = {
+            "username": "App Enlight",
+            "attachments": [
+                {
+                    "mrkdwn_in": ["text", "pretext", "title", "fallback"],
+                    "fallback": "%s : <%s| Browse>" % (
+                        title, template_vars['destination_url']),
+                    "pretext": "%s: <%s| Browse>" % (
+                        title, template_vars['destination_url']),
+                    "color": "good",
+                    "fields": [
+                        {
+                            "title": "Got at least: %s" % subtext,
+                            "short": False
+                        }
+                    ]
+                }
+            ]
+        }
+        client.make_request(data=report_data)
+
+        log_msg = 'DIGEST  : %s via %s :: %s reports' % (
+            kwargs['user'].user_name,
+            self.channel_visible_value,
+            template_vars['confirmed_total'])
+        log.warning(log_msg)

backend/src/appenlight/models/application.py

@@ -0,0 +1,109 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import uuid
+import logging
+import sqlalchemy as sa
+from appenlight.models.resource import Resource
+from sqlalchemy.orm import aliased
+
+log = logging.getLogger(__name__)
+
+
+def generate_api_key():
+    uid = str(uuid.uuid4()).replace('-', '')
+    return uid[0:32]
+
+
+class Application(Resource):
+    """
+    Resource of application type
+    """
+
+    __tablename__ = 'applications'
+    __mapper_args__ = {'polymorphic_identity': 'application'}
+
+    # lists configurable possible permissions for this resource type
+    __possible_permissions__ = ('view', 'update_reports')
+
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('resources.resource_id',
+                                          onupdate='CASCADE',
+                                          ondelete='CASCADE', ),
+                            primary_key=True, )
+    domains = sa.Column(sa.UnicodeText(), nullable=False, default='')
+    api_key = sa.Column(sa.String(32), nullable=False, unique=True, index=True,
+                        default=generate_api_key)
+    public_key = sa.Column(sa.String(32), nullable=False, unique=True,
+                           index=True,
+                           default=generate_api_key)
+    default_grouping = sa.Column(sa.Unicode(20), nullable=False,
+                                 default='url_traceback')
+    error_report_threshold = sa.Column(sa.Integer(), default=10)
+    slow_report_threshold = sa.Column(sa.Integer(), default=10)
+    allow_permanent_storage = sa.Column(sa.Boolean(), default=False,
+                                        nullable=False)
+
+    @sa.orm.validates('default_grouping')
+    def validate_default_grouping(self, key, grouping):
+        """ validate if resouce can have specific permission """
+        assert grouping in ['url_type', 'url_traceback', 'traceback_server']
+        return grouping
+
+    report_groups = sa.orm.relationship('ReportGroup',
+                                        cascade="all, delete-orphan",
+                                        passive_deletes=True,
+                                        passive_updates=True,
+                                        lazy='dynamic',
+                                        backref=sa.orm.backref('application',
+                                                               lazy="joined"))
+
+    postprocess_conf = sa.orm.relationship('ApplicationPostprocessConf',
+                                           cascade="all, delete-orphan",
+                                           passive_deletes=True,
+                                           passive_updates=True,
+                                           backref='resource')
+
+    logs = sa.orm.relationship('Log',
+                               lazy='dynamic',
+                               backref='application',
+                               passive_deletes=True,
+                               passive_updates=True, )
+
+    integrations = sa.orm.relationship('IntegrationBase',
+                                       backref='resource',
+                                       cascade="all, delete-orphan",
+                                       passive_deletes=True,
+                                       passive_updates=True, )
+
+    def generate_api_key(self):
+        return generate_api_key()
+
+
+def after_update(mapper, connection, target):
+    from appenlight.models.services.application import ApplicationService
+    log.info('clearing out ApplicationService cache')
+    ApplicationService.by_id_cached().invalidate(target.resource_id)
+    ApplicationService.by_api_key_cached().invalidate(target.api_key)
+
+
+sa.event.listen(Application, 'after_update', after_update)
+sa.event.listen(Application, 'after_delete', after_update)

backend/src/appenlight/models/application_postprocess_conf.py

@@ -0,0 +1,50 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.base import BaseModel
+import sqlalchemy as sa
+
+from appenlight.models import Base
+from appenlight.models.report_group import ReportGroup
+
+
+class ApplicationPostprocessConf(Base, BaseModel):
+    """
+    Stores prioritizing conditions for reports
+    This is later used for rule parsing like "if 10 occurences bump priority +1"
+    """
+
+    __tablename__ = 'application_postprocess_conf'
+
+    pkey = sa.Column(sa.Integer(), nullable=False, primary_key=True)
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('resources.resource_id',
+                                          onupdate='CASCADE',
+                                          ondelete='CASCADE'))
+    do = sa.Column(sa.Unicode(25), nullable=False)
+    new_value = sa.Column(sa.UnicodeText(), nullable=False, default='')
+    rule = sa.Column(sa.dialects.postgresql.JSON,
+                     nullable=False, default={'field': 'http_status',
+                                              "op": "ge", "value": "500"})
+
+    def postprocess(self, item):
+        new_value = int(self.new_value)
+        item.priority = ReportGroup.priority + new_value

backend/src/appenlight/models/auth_token.py

@@ -0,0 +1,57 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import sqlalchemy as sa
+
+from datetime import datetime
+from appenlight.models import Base
+from ziggurat_foundations.models.base import BaseModel
+from ziggurat_foundations.models.services.user import UserService
+
+log = logging.getLogger(__name__)
+
+
+class AuthToken(Base, BaseModel):
+    """
+    Stores information about possible alerting options
+    """
+    __tablename__ = 'auth_tokens'
+
+    id = sa.Column(sa.Integer, primary_key=True, nullable=False)
+    token = sa.Column(sa.Unicode(40), nullable=False,
+                      default=lambda x: UserService.generate_random_string(40))
+    owner_id = sa.Column(sa.Unicode(30),
+                         sa.ForeignKey('users.id', onupdate='CASCADE',
+                                       ondelete='CASCADE'))
+    creation_date = sa.Column(sa.DateTime, default=lambda x: datetime.utcnow())
+    expires = sa.Column(sa.DateTime)
+    description = sa.Column(sa.Unicode, default='')
+
+    @property
+    def is_expired(self):
+        if self.expires:
+            return self.expires < datetime.utcnow()
+        else:
+            return False
+
+    def __str__(self):
+        return '<AuthToken u:%s t:%s...>' % (self.owner_id, self.token[0:10])

backend/src/appenlight/models/config.py

@@ -0,0 +1,37 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from ziggurat_foundations.models.base import BaseModel
+from sqlalchemy.dialects.postgres import JSON
+
+from . import Base
+
+
+class Config(Base, BaseModel):
+    __tablename__ = 'config'
+
+    key = sa.Column(sa.Unicode, primary_key=True)
+    section = sa.Column(sa.Unicode, primary_key=True)
+    value = sa.Column(JSON, nullable=False)
+
+    def __json__(self, request):
+        return self.get_dict()

backend/src/appenlight/models/event.py

@@ -0,0 +1,160 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+import logging
+
+from datetime import datetime
+from appenlight.models import Base, get_db_session
+from appenlight.models.services.report_stat import ReportStatService
+from appenlight.models.resource import Resource
+from appenlight.models.integrations import IntegrationException
+from pyramid.threadlocal import get_current_request
+from sqlalchemy.dialects.postgresql import JSON
+from ziggurat_foundations.models.base import BaseModel
+
+log = logging.getLogger(__name__)
+
+
+class Event(Base, BaseModel):
+    __tablename__ = 'events'
+
+    types = {'error_report_alert': 1,
+             'slow_report_alert': 3,
+             'comment': 5,
+             'assignment': 6,
+             'uptime_alert': 7,
+             'chart_alert': 9}
+
+    statuses = {'active': 1,
+                'closed': 0}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    start_date = sa.Column(sa.DateTime, default=datetime.utcnow)
+    end_date = sa.Column(sa.DateTime)
+    status = sa.Column(sa.Integer, default=1)
+    event_type = sa.Column(sa.Integer, default=1)
+    origin_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
+                               nullable=True)
+    target_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
+                               nullable=True)
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('resources.resource_id'),
+                            nullable=True)
+    target_id = sa.Column(sa.Integer)
+    target_uuid = sa.Column(sa.Unicode(40))
+    text = sa.Column(sa.UnicodeText())
+    values = sa.Column(JSON(), nullable=False, default=None)
+
+    def __repr__(self):
+        return '<Event %s, app:%s, %s>' % (self.unified_alert_name(),
+                                            self.resource_id,
+                                            self.unified_alert_action())
+
+    @property
+    def reverse_types(self):
+        return dict([(v, k) for k, v in self.types.items()])
+
+    def unified_alert_name(self):
+        return self.reverse_types[self.event_type]
+
+    def unified_alert_action(self):
+        event_name = self.reverse_types[self.event_type]
+        if self.status == Event.statuses['closed']:
+            return "CLOSE"
+        if self.status != Event.statuses['closed']:
+            return "OPEN"
+        return event_name
+
+    def send_alerts(self, request=None, resource=None, db_session=None):
+        """" Sends alerts to applicable channels """
+        db_session = get_db_session(db_session)
+        db_session.flush()
+        if not resource:
+            resource = Resource.by_resource_id(self.resource_id)
+        if not request:
+            request = get_current_request()
+        if not resource:
+            return
+        users = set([p.user for p in resource.users_for_perm('view')])
+        for user in users:
+            for channel in user.alert_channels:
+                if not channel.channel_validated or not channel.send_alerts:
+                    continue
+                else:
+                    try:
+                        channel.notify_alert(resource=resource,
+                                             event=self,
+                                             user=user,
+                                             request=request)
+                    except IntegrationException as e:
+                        log.warning('%s' % e)
+
+    def validate_or_close(self, since_when, db_session=None):
+        """ Checks if alerts should stay open or it's time to close them.
+            Generates close alert event if alerts get closed """
+        event_types = [Event.types['error_report_alert'],
+                       Event.types['slow_report_alert']]
+        app = Resource.by_resource_id(self.resource_id)
+        if self.event_type in event_types:
+            total = ReportStatService.count_by_type(
+                self.event_type, self.resource_id, since_when)
+            if Event.types['error_report_alert'] == self.event_type:
+                threshold = app.error_report_threshold
+            if Event.types['slow_report_alert'] == self.event_type:
+                threshold = app.slow_report_threshold
+
+            if total < threshold:
+                self.close()
+
+    def close(self, db_session=None):
+        """
+        Closes an event and sends notification to affected users
+        """
+        self.end_date = datetime.utcnow()
+        self.status = Event.statuses['closed']
+        log.warning('ALERT: CLOSE: %s' % self)
+        self.send_alerts()
+
+    def text_representation(self):
+        alert_type = self.unified_alert_name()
+        text = ''
+        if 'slow_report' in alert_type:
+            text += 'Slow report alert'
+        if 'error_report' in alert_type:
+            text += 'Exception report alert'
+        if 'uptime_alert' in alert_type:
+            text += 'Uptime alert'
+        if 'chart_alert' in alert_type:
+            text += 'Metrics value alert'
+
+        alert_action = self.unified_alert_action()
+        if alert_action == 'OPEN':
+            text += ' got opened.'
+        if alert_action == 'CLOSE':
+            text += ' got closed.'
+        return text
+
+    def get_dict(self, request=None):
+        dict_data = super(Event, self).get_dict()
+        dict_data['text'] = self.text_representation()
+        dict_data['resource_name'] = self.resource.resource_name
+        return dict_data

backend/src/appenlight/models/external_identity.py

@@ -0,0 +1,41 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from sqlalchemy.ext.declarative import declared_attr
+from ziggurat_foundations.models.external_identity import ExternalIdentityMixin
+
+from appenlight.models import Base
+from appenlight.lib.sqlalchemy_fields import EncryptedUnicode
+
+
+class ExternalIdentity(ExternalIdentityMixin, Base):
+    @declared_attr
+    def access_token(self):
+        return sa.Column(EncryptedUnicode(255), default='')
+
+    @declared_attr
+    def alt_token(self):
+        return sa.Column(EncryptedUnicode(255), default='')
+
+    @declared_attr
+    def token_secret(self):
+        return sa.Column(EncryptedUnicode(255), default='')

backend/src/appenlight/models/group.py

@@ -0,0 +1,50 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.group import GroupMixin
+from appenlight.models import Base
+
+
+class Group(GroupMixin, Base):
+    __possible_permissions__ = ('root_administration',
+                                'test_features',
+                                'admin_panel',
+                                'admin_users',
+                                'manage_partitions',)
+
+    def get_dict(self, exclude_keys=None, include_keys=None,
+                 include_perms=False):
+        result = super(Group, self).get_dict(exclude_keys, include_keys)
+        if include_perms:
+            result['possible_permissions'] = self.__possible_permissions__
+            result['current_permissions'] = [p.perm_name for p in
+                                             self.permissions]
+        else:
+            result['possible_permissions'] = []
+            result['current_permissions'] = []
+        exclude_keys_list = exclude_keys or []
+        include_keys_list = include_keys or []
+        d = {}
+        for k in result.keys():
+            if (k not in exclude_keys_list and
+                    (k in include_keys_list or not include_keys)):
+                d[k] = result[k]
+        return d

backend/src/appenlight/models/group_permission.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.group_permission import GroupPermissionMixin
+from appenlight.models import Base
+
+
+class GroupPermission(GroupPermissionMixin, Base):
+    pass

backend/src/appenlight/models/group_resource_permission.py

@@ -0,0 +1,28 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.group_resource_permission import \
+    GroupResourcePermissionMixin
+from appenlight.models import Base
+
+
+class GroupResourcePermission(GroupResourcePermissionMixin, Base):
+    pass

backend/src/appenlight/models/integrations/__init__.py

@@ -0,0 +1,83 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy.ext.hybrid import hybrid_property
+from ziggurat_foundations.models.base import BaseModel
+
+from appenlight.lib.encryption import decrypt_dictionary_keys
+from appenlight.lib.encryption import encrypt_dictionary_keys
+from appenlight.models import Base, get_db_session
+
+
+class IntegrationException(Exception):
+    pass
+
+
+class IntegrationBase(Base, BaseModel):
+    """
+    Model from which all integrations inherit using polymorphic approach
+    """
+    __tablename__ = 'integrations'
+
+    front_visible = False
+    as_alert_channel = False
+    supports_report_alerting = False
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    resource_id = sa.Column(sa.Integer,
+                            sa.ForeignKey('applications.resource_id'))
+    integration_name = sa.Column(sa.Unicode(64))
+    _config = sa.Column('config', JSON(), nullable=False, default='')
+    modified_date = sa.Column(sa.DateTime)
+
+    channel = sa.orm.relationship('AlertChannel',
+                                  cascade="all,delete-orphan",
+                                  passive_deletes=True,
+                                  passive_updates=True,
+                                  uselist=False,
+                                  backref='integration')
+
+    __mapper_args__ = {
+        'polymorphic_on': 'integration_name',
+        'polymorphic_identity': 'integration'
+    }
+
+    @classmethod
+    def by_app_id_and_integration_name(cls, resource_id, integration_name,
+                                       db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(cls)
+        query = query.filter(cls.integration_name == integration_name)
+        query = query.filter(cls.resource_id == resource_id)
+        return query.first()
+
+    @hybrid_property
+    def config(self):
+        return decrypt_dictionary_keys(self._config)
+
+    @config.setter
+    def config(self, value):
+        if not hasattr(value, 'items'):
+            raise Exception('IntegrationBase.config only accepts '
+                            'flat dictionaries')
+        self._config = encrypt_dictionary_keys(value)

backend/src/appenlight/models/integrations/bitbucket.py

@@ -0,0 +1,168 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import requests
+from requests_oauthlib import OAuth1
+from appenlight.models.integrations import (IntegrationBase,
+                                            IntegrationException)
+
+_ = str
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class BitbucketIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'bitbucket'
+    }
+    front_visible = True
+    as_alert_channel = False
+    supports_report_alerting = False
+    action_notification = True
+    integration_action = 'Add issue to Bitbucket'
+
+    @classmethod
+    def create_client(cls, request, user_name=None, repo_name=None):
+        """
+        Creates REST client that can authenticate to specific repo
+        uses auth tokens for current request user
+        """
+        config = request.registry.settings
+        token = None
+        secret = None
+        for identity in request.user.external_identities:
+            if identity.provider_name == 'bitbucket':
+                token = identity.access_token
+                secret = identity.token_secret
+                break
+        if not token:
+            raise IntegrationException(
+                'No valid auth token present for this service')
+        client = BitbucketClient(token, secret,
+                                 user_name,
+                                 repo_name,
+                                 config['authomatic.pr.bitbucket.key'],
+                                 config['authomatic.pr.bitbucket.secret'])
+        return client
+
+
+class BitbucketClient(object):
+    api_url = 'https://bitbucket.org/api/1.0'
+    repo_type = 'bitbucket'
+
+    def __init__(self, token, secret, owner, repo_name, bitbucket_consumer_key,
+                 bitbucket_consumer_secret):
+        self.access_token = token
+        self.token_secret = secret
+        self.owner = owner
+        self.repo_name = repo_name
+        self.bitbucket_consumer_key = bitbucket_consumer_key
+        self.bitbucket_consumer_secret = bitbucket_consumer_secret
+
+    possible_keys = {
+        'status': ['new', 'open', 'resolved', 'on hold', 'invalid',
+                   'duplicate', 'wontfix'],
+        'priority': ['trivial', 'minor', 'major', 'critical', 'blocker'],
+        'kind': ['bug', 'enhancement', 'proposal', 'task']
+    }
+
+    def get_statuses(self):
+        """Gets list of possible item statuses"""
+        return self.possible_keys['status']
+
+    def get_priorities(self):
+        """Gets list of possible item statuses"""
+        return self.possible_keys['priority']
+
+    def make_request(self, url, method='get', data=None, headers=None):
+        """
+        Performs HTTP request to bitbucket
+        """
+        auth = OAuth1(self.bitbucket_consumer_key,
+                      self.bitbucket_consumer_secret,
+                      self.access_token, self.token_secret)
+        try:
+            resp = getattr(requests, method)(url, data=data, auth=auth,
+                                             timeout=10)
+        except Exception as e:
+            raise IntegrationException(
+                _('Error communicating with Bitbucket: %s') % (e,))
+        if resp.status_code == 401:
+            raise IntegrationException(
+                _('You are not authorized to access this repo'))
+        elif resp.status_code == 404:
+            raise IntegrationException(_('User or repo name are incorrect'))
+        elif resp.status_code not in [200, 201]:
+            raise IntegrationException(
+                _('Bitbucket response_code: %s') % resp.status_code)
+        try:
+            return resp.json()
+        except Exception as e:
+            raise IntegrationException(
+                _('Error decoding response from Bitbucket: %s') % (e,))
+
+    def get_assignees(self):
+        """Gets list of possible assignees"""
+        url = '%(api_url)s/privileges/%(owner)s/%(repo_name)s' % {
+            'api_url': self.api_url,
+            'owner': self.owner,
+            'repo_name': self.repo_name}
+
+        data = self.make_request(url)
+        results = [{'user': self.owner, 'name': '(Repo owner)'}]
+        if data:
+            for entry in data:
+                results.append({"user": entry['user']['username'],
+                                "name": entry['user'].get('display_name')})
+        return results
+
+    def create_issue(self, form_data):
+        """
+        Sends creates a new issue in tracker using REST call
+        """
+        url = '%(api_url)s/repositories/%(owner)s/%(repo_name)s/issues/' % {
+            'api_url': self.api_url,
+            'owner': self.owner,
+            'repo_name': self.repo_name}
+
+        payload = {
+            "title": form_data['title'],
+            "content": form_data['content'],
+            "kind": form_data['kind'],
+            "priority": form_data['priority'],
+            "responsible": form_data['responsible']
+        }
+        data = self.make_request(url, 'post', payload)
+        f_args = {
+            "owner": self.owner,
+            "repo_name": self.repo_name,
+            "issue_id": data['local_id']
+        }
+        web_url = 'https://bitbucket.org/%(owner)s/%(repo_name)s' \
+                  '/issue/%(issue_id)s/issue-title' % f_args
+        to_return = {
+            'id': data['local_id'],
+            'resource_url': data['resource_uri'],
+            'web_url': web_url
+        }
+        return to_return

backend/src/appenlight/models/integrations/campfire.py

@@ -0,0 +1,79 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+from requests.exceptions import HTTPError, ConnectionError
+from camplight import Request, Campfire
+from camplight.exceptions import CamplightException
+
+from appenlight.models.integrations import (IntegrationBase,
+                                            IntegrationException)
+
+_ = str
+
+log = logging.getLogger(__name__)
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class CampfireIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'campfire'
+    }
+    front_visible = False
+    as_alert_channel = True
+    supports_report_alerting = True
+    action_notification = True
+    integration_action = 'Message via Campfire'
+
+    @classmethod
+    def create_client(cls, api_token, account):
+        client = CampfireClient(api_token, account)
+        return client
+
+
+class CampfireClient(object):
+    def __init__(self, api_token, account):
+        request = Request('https://%s.campfirenow.com' % account, api_token)
+        self.campfire = Campfire(request)
+
+    def get_account(self):
+        try:
+            return self.campfire.account()
+        except (HTTPError, CamplightException) as e:
+            raise IntegrationException(str(e))
+
+    def get_rooms(self):
+        try:
+            return self.campfire.rooms()
+        except (HTTPError, CamplightException) as e:
+            raise IntegrationException(str(e))
+
+    def speak_to_room(self, room, message, sound='RIMSHOT'):
+        try:
+            room = self.campfire.room(room)
+            room.join()
+            room.speak(message, type_='TextMessage')
+        except (HTTPError, CamplightException, ConnectionError) as e:
+            raise IntegrationException(str(e))

backend/src/appenlight/models/integrations/flowdock.py

@@ -0,0 +1,87 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+import requests
+from requests.auth import HTTPBasicAuth
+import simplejson as json
+
+from appenlight.models.integrations import (IntegrationBase,
+                                            IntegrationException)
+
+_ = str
+
+log = logging.getLogger(__name__)
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class FlowdockIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'flowdock'
+    }
+    front_visible = False
+    as_alert_channel = True
+    supports_report_alerting = True
+    action_notification = True
+    integration_action = 'Message via Flowdock'
+
+    @classmethod
+    def create_client(cls, api_token):
+        client = FlowdockClient(api_token)
+        return client
+
+
+class FlowdockClient(object):
+    def __init__(self, api_token):
+        self.auth = HTTPBasicAuth(api_token, '')
+        self.api_token = api_token
+        self.api_url = 'https://api.flowdock.com/v1/messages'
+
+    def make_request(self, url, method='get', data=None):
+        headers = {
+            'Content-Type': 'application/json',
+            'User-Agent': 'appenlight-flowdock',
+        }
+        try:
+            if data:
+                data = json.dumps(data)
+            resp = getattr(requests, method)(url, data=data, headers=headers,
+                                             timeout=10)
+        except Exception as e:
+            raise IntegrationException(
+                _('Error communicating with Flowdock: %s') % (e,))
+        if resp.status_code > 299:
+            raise IntegrationException(resp.text)
+        return resp
+
+    def send_to_chat(self, payload):
+        url = '%(api_url)s/chat/%(api_token)s' % {'api_url': self.api_url,
+                                                  'api_token': self.api_token}
+        return self.make_request(url, method='post', data=payload).json()
+
+    def send_to_inbox(self, payload):
+        f_args = {'api_url': self.api_url, 'api_token': self.api_token}
+        url = '%(api_url)s/team_inbox/%(api_token)s' % f_args
+        return self.make_request(url, method='post', data=payload).json()

backend/src/appenlight/models/integrations/github.py

@@ -0,0 +1,161 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import json
+import requests
+
+from . import IntegrationBase, IntegrationException
+
+_ = str
+
+
+class GithubAuthException(Exception):
+    pass
+
+
+class GithubIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'github'
+    }
+    front_visible = True
+    as_alert_channel = False
+    supports_report_alerting = False
+    action_notification = True
+    integration_action = 'Add issue to Github'
+
+    @classmethod
+    def create_client(cls, request, user_name=None, repo_name=None):
+        """
+        Creates REST client that can authenticate to specific repo
+        uses auth tokens for current request user
+        """
+        token = None
+        secret = None
+        for identity in request.user.external_identities:
+            if identity.provider_name == 'github':
+                token = identity.access_token
+                secret = identity.token_secret
+                break
+        if not token:
+            raise IntegrationException(
+                'No valid auth token present for this service')
+        client = GithubClient(token=token, owner=user_name, name=repo_name)
+        return client
+
+
+class GithubClient(object):
+    api_url = 'https://api.github.com'
+    repo_type = 'github'
+
+    def __init__(self, token, owner, name):
+        self.access_token = token
+        self.owner = owner
+        self.name = name
+
+    def make_request(self, url, method='get', data=None, headers=None):
+        req_headers = {'User-Agent': 'appenlight',
+                       'Content-Type': 'application/json',
+                       'Authorization': 'token %s' % self.access_token}
+        try:
+            if data:
+                data = json.dumps(data)
+            resp = getattr(requests, method)(url, data=data,
+                                             headers=req_headers,
+                                             timeout=10)
+        except Exception as e:
+            msg = 'Error communicating with Github: %s'
+            raise IntegrationException(_(msg) % (e,))
+
+        if resp.status_code == 404:
+            msg = 'User or repo name are incorrect'
+            raise IntegrationException(_(msg))
+        if resp.status_code == 401:
+            msg = 'You are not authorized to access this repo'
+            raise IntegrationException(_(msg))
+        elif resp.status_code not in [200, 201]:
+            msg = 'Github response_code: %s'
+            raise IntegrationException(_(msg) % resp.status_code)
+        try:
+            return resp.json()
+        except Exception as e:
+            msg = 'Error decoding response from Github: %s'
+            raise IntegrationException(_(msg) % (e,))
+
+    def get_statuses(self):
+        """Gets list of possible item statuses"""
+        url = '%(api_url)s/repos/%(owner)s/%(name)s/labels' % {
+            'api_url': self.api_url,
+            'owner': self.owner,
+            'name': self.name}
+
+        data = self.make_request(url)
+
+        statuses = []
+        for status in data:
+            statuses.append(status['name'])
+        return statuses
+
+    def get_repo(self):
+        """Gets list of possible item statuses"""
+        url = '%(api_url)s/repos/%(owner)s/%(name)s' % {
+            'api_url': self.api_url,
+            'owner': self.owner,
+            'name': self.name}
+
+        data = self.make_request(url)
+        return data
+
+    def get_assignees(self):
+        """Gets list of possible assignees"""
+        url = '%(api_url)s/repos/%(owner)s/%(name)s/collaborators' % {
+            'api_url': self.api_url,
+            'owner': self.owner,
+            'name': self.name}
+        data = self.make_request(url)
+        results = []
+        for entry in data:
+            results.append({"user": entry['login'],
+                            "name": entry.get('name')})
+        return results
+
+    def create_issue(self, form_data):
+        """
+        Make a REST call to create issue in Github's issue tracker
+        """
+        url = '%(api_url)s/repos/%(owner)s/%(name)s/issues' % {
+            'api_url': self.api_url,
+            'owner': self.owner,
+            'name': self.name}
+
+        payload = {
+            "title": form_data['title'],
+            "body": form_data['content'],
+            "labels": [],
+            "assignee": form_data['responsible']
+        }
+        payload['labels'].extend(form_data['kind'])
+        data = self.make_request(url, 'post', data=payload)
+        to_return = {
+            'id': data['number'],
+            'resource_url': data['url'],
+            'web_url': data['html_url']
+        }
+        return to_return

backend/src/appenlight/models/integrations/hipchat.py

@@ -0,0 +1,88 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+import requests
+
+from . import IntegrationBase, IntegrationException
+
+_ = str
+
+log = logging.getLogger(__name__)
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class HipchatIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'hipchat'
+    }
+    front_visible = False
+    as_alert_channel = True
+    supports_report_alerting = True
+    action_notification = True
+    integration_action = 'Message via Hipchat'
+
+    @classmethod
+    def create_client(cls, api_token):
+        client = HipchatClient(api_token)
+        return client
+
+
+class HipchatClient(object):
+    def __init__(self, api_token):
+        self.api_token = api_token
+        self.api_url = 'https://api.hipchat.com/v1'
+
+    def make_request(self, endpoint, method='get', data=None):
+        headers = {
+            'User-Agent': 'appenlight-hipchat',
+        }
+        url = '%s%s' % (self.api_url, endpoint)
+        params = {
+            'format': 'json',
+            'auth_token': self.api_token
+        }
+        try:
+            resp = getattr(requests, method)(url, data=data, headers=headers,
+                                             params=params,
+                                             timeout=3)
+        except Exception as e:
+            msg = 'Error communicating with Hipchat: %s'
+            raise IntegrationException(_(msg) % (e,))
+        if resp.status_code == 404:
+            msg = 'Error communicating with Hipchat - Room not found'
+            raise IntegrationException(msg)
+        elif resp.status_code != requests.codes.ok:
+            msg = 'Error communicating with Hipchat - status code: %s'
+            raise IntegrationException(msg % resp.status_code)
+        return resp
+
+    def get_rooms(self):
+        # not used with notification api token
+        return self.make_request('/rooms/list')
+
+    def send(self, payload):
+        return self.make_request('/rooms/message', method='post',
+                                 data=payload).json()

backend/src/appenlight/models/integrations/jira.py

@@ -0,0 +1,133 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import jira
+from appenlight.models.integrations import (IntegrationBase,
+                                            IntegrationException)
+
+_ = str
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class JiraIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'jira'
+    }
+    front_visible = True
+    as_alert_channel = False
+    supports_report_alerting = False
+    action_notification = True
+    integration_action = 'Add issue to Jira'
+
+
+class JiraClient(object):
+    def __init__(self, user_name, password, host_name, project, request=None):
+        self.user_name = user_name
+        self.password = password
+        self.host_name = host_name
+        self.project = project
+        self.request = request
+        try:
+            self.client = jira.client.JIRA(options={'server': host_name},
+                                           basic_auth=(user_name, password))
+        except jira.JIRAError as e:
+            raise IntegrationException(
+                'Communication problem: HTTP_STATUS:%s, URL:%s ' % (
+                    e.status_code, e.url))
+
+    def get_projects(self):
+        projects = self.client.projects()
+        return projects
+
+    def get_assignees(self):
+        """Gets list of possible assignees"""
+        users = self.client.search_assignable_users_for_issues(
+            None, project=self.project)
+        results = []
+        for user in users:
+            results.append({"id": user.name, "name": user.displayName})
+        return results
+
+    def get_metadata(self):
+        def cached(project_name):
+            metadata = self.client.createmeta(
+                projectKeys=project_name, expand='projects.issuetypes.fields')
+            assignees = self.get_assignees()
+            parsed_metadata = []
+            for entry in metadata['projects'][0]['issuetypes']:
+                issue = {"name": entry['name'],
+                         "id": entry['id'],
+                         "fields": []}
+                for i_id, field_i in entry['fields'].items():
+                    field = {
+                        "name": field_i['name'],
+                        "id": i_id,
+                        "required": field_i['required'],
+                        "values": [],
+                        "type": field_i['schema'].get('type')
+                    }
+                    if field_i.get('allowedValues'):
+                        field['values'] = []
+                        for i in field_i['allowedValues']:
+                            field['values'].append(
+                                {'id': i['id'],
+                                 'name': i.get('name', i.get('value', ''))
+                                 })
+                    if field['id'] == 'assignee':
+                        field['values'] = assignees
+
+                    issue['fields'].append(field)
+                parsed_metadata.append(issue)
+            return parsed_metadata
+
+        return cached(self.project)
+
+    def create_issue(self, form_data):
+        metadata = self.get_metadata()
+        payload = {
+            'project': {'key': form_data['project']},
+            'summary': form_data['title'],
+            'description': form_data['content'],
+            'issuetype': {'id': '1'},
+            "priority": {'id': form_data['priority']},
+            "assignee": {'name': form_data['responsible']},
+        }
+        for issue_type in metadata:
+            if issue_type['id'] == '1':
+                for field in issue_type['fields']:
+                    if field == 'reporter':
+                        payload["reporter"] = {'id': self.user_name},
+                    if field['required'] and field['id'] not in payload:
+                        if field['type'] == 'array':
+                            payload[field['id']] = [field['values'][0], ]
+                        elif field['type'] == 'string':
+                            payload[field['id']] = ''
+        new_issue = self.client.create_issue(fields=payload)
+        web_url = self.host_name + '/browse/' + new_issue.key
+        to_return = {
+            'id': new_issue.id,
+            'resource_url': new_issue.self,
+            'web_url': web_url
+        }
+        return to_return

backend/src/appenlight/models/integrations/slack.py

@@ -0,0 +1,79 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+import requests
+
+from appenlight.models.integrations import (IntegrationBase,
+                                            IntegrationException)
+from appenlight.lib.ext_json import json
+
+_ = str
+
+log = logging.getLogger(__name__)
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class SlackIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'slack'
+    }
+    front_visible = False
+    as_alert_channel = True
+    supports_report_alerting = True
+    action_notification = True
+    integration_action = 'Message via Slack'
+
+    @classmethod
+    def create_client(cls, api_token):
+        client = SlackClient(api_token)
+        return client
+
+
+class SlackClient(object):
+    def __init__(self, api_url):
+        self.api_url = api_url
+
+    def make_request(self, data=None):
+        headers = {
+            'User-Agent': 'appenlight-slack',
+            'Content-Type': 'application/json'
+        }
+        try:
+            resp = getattr(requests, 'post')(self.api_url,
+                                             data=json.dumps(data),
+                                             headers=headers,
+                                             timeout=3)
+        except Exception as e:
+            raise IntegrationException(
+                _('Error communicating with Slack: %s') % (e,))
+        if resp.status_code != requests.codes.ok:
+            msg = 'Error communicating with Slack - status code: %s'
+            raise IntegrationException(msg % resp.status_code)
+        return resp
+
+    def send(self, payload):
+        return self.make_request('/rooms/message', method='post',
+                                 data=payload).json()

backend/src/appenlight/models/integrations/webhooks.py

@@ -0,0 +1,143 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+import requests
+
+from appenlight.models.integrations import (IntegrationBase,
+                                            IntegrationException)
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.lib.ext_json import json
+
+_ = str
+
+log = logging.getLogger(__name__)
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class WebhooksIntegration(IntegrationBase):
+    __mapper_args__ = {
+        'polymorphic_identity': 'webhooks'
+    }
+    front_visible = False
+    as_alert_channel = True
+    supports_report_alerting = True
+    action_notification = True
+    integration_action = 'Message via Webhooks'
+
+    @classmethod
+    def create_client(cls, url):
+        client = WebhooksClient(url)
+        return client
+
+
+class WebhooksClient(object):
+    def __init__(self, url):
+        self.api_url = url
+
+    def make_request(self, url, method='get', data=None):
+        headers = {
+            'Content-Type': 'application/json',
+            'User-Agent': 'appenlight-webhooks',
+        }
+        try:
+            if data:
+                data = json.dumps(data)
+            resp = getattr(requests, method)(url, data=data, headers=headers,
+                                             timeout=3)
+        except Exception as e:
+            raise IntegrationException(
+                _('Error communicating with Webhooks: %s').format(e))
+        if resp.status_code > 299:
+            raise IntegrationException(
+                'Error communicating with Webhooks - status code: %s'.format(
+                    resp.status_code))
+        return resp
+
+    def send_to_hook(self, payload):
+        return self.make_request(self.api_url, method='post',
+                                 data=payload).json()
+
+
+class WebhooksAlertChannel(AlertChannel):
+    __mapper_args__ = {
+        'polymorphic_identity': 'webhooks'
+    }
+
+    def notify_reports(self, **kwargs):
+        """
+        Notify user of individual reports
+
+        kwargs:
+            application: application that the event applies for,
+            user: user that should be notified
+            request: request object
+            since_when: reports are newer than this time value,
+            reports: list of reports to render
+
+        """
+        template_vars = self.get_notification_basic_vars(kwargs)
+        payload = []
+        include_keys = ('id', 'http_status', 'report_type', 'resource_name',
+                        'front_url', 'resource_id', 'error', 'url_path',
+                        'tags', 'duration')
+
+        for occurences, report in kwargs['reports']:
+            r_dict = report.last_report_ref.get_dict(kwargs['request'],
+                                                     include_keys=include_keys)
+            r_dict['group']['occurences'] = occurences
+            payload.append(r_dict)
+        client = WebhooksIntegration.create_client(
+            self.integration.config['reports_webhook'])
+        client.send_to_hook(payload)
+
+    def notify_alert(self, **kwargs):
+        """
+        Notify user of report or uptime threshold events based on events alert type
+
+        Kwargs:
+            application: application that the event applies for,
+            event: event that is notified,
+            user: user that should be notified
+            request: request object
+
+        """
+        payload = {
+            'alert_action': kwargs['event'].unified_alert_action(),
+            'alert_name': kwargs['event'].unified_alert_name(),
+            'event_time': kwargs['event'].end_date or kwargs[
+                'event'].start_date,
+            'resource_name': None,
+            'resource_id': None
+        }
+        if kwargs['event'].values and kwargs['event'].values.get('reports'):
+            payload['reports'] = kwargs['event'].values.get('reports', [])
+        if 'application' in kwargs:
+            payload['resource_name'] = kwargs['application'].resource_name
+            payload['resource_id'] = kwargs['application'].resource_id
+
+        client = WebhooksIntegration.create_client(
+            self.integration.config['alerts_webhook'])
+        client.send_to_hook(payload)

backend/src/appenlight/models/log.py

@@ -0,0 +1,135 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+import logging
+import hashlib
+
+from datetime import datetime
+from appenlight.models import Base
+from appenlight.lib.utils import convert_es_type
+from appenlight.lib.enums import LogLevel
+from sqlalchemy.dialects.postgresql import JSON
+from ziggurat_foundations.models.base import BaseModel
+
+log = logging.getLogger(__name__)
+
+
+class Log(Base, BaseModel):
+    __tablename__ = 'logs'
+    __table_args__ = {'implicit_returning': False}
+
+    log_id = sa.Column(sa.BigInteger(), nullable=False, primary_key=True)
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('applications.resource_id',
+                                          onupdate='CASCADE',
+                                          ondelete='CASCADE'),
+                            nullable=False,
+                            index=True)
+    log_level = sa.Column(sa.Unicode, nullable=False, index=True,
+                          default='INFO')
+    message = sa.Column(sa.UnicodeText(), default='')
+    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                          server_default=sa.func.now())
+    request_id = sa.Column(sa.Unicode())
+    namespace = sa.Column(sa.Unicode())
+    primary_key = sa.Column(sa.Unicode())
+
+    tags = sa.Column(JSON(), default={})
+    permanent = sa.Column(sa.Boolean(), nullable=False, default=False)
+
+    def __str__(self):
+        return self.__unicode__().encode('utf8')
+
+    def __unicode__(self):
+        return '<Log id:%s, lv:%s, ns:%s >' % (
+            self.log_id, self.log_level, self.namespace)
+
+    def set_data(self, data, resource):
+        level = data.get('log_level').upper()
+        self.log_level = getattr(LogLevel, level, LogLevel.UNKNOWN)
+        self.message = data.get('message', '')
+        server_name = data.get('server', '').lower() or 'unknown'
+        self.tags = {
+            'server_name': server_name
+        }
+        if data.get('tags'):
+            for tag_tuple in data['tags']:
+                self.tags[tag_tuple[0]] = tag_tuple[1]
+        self.timestamp = data['date']
+        r_id = data.get('request_id', '')
+        if not r_id:
+            r_id = ''
+        self.request_id = r_id.replace('-', '')
+        self.resource_id = resource.resource_id
+        self.namespace = data.get('namespace') or ''
+        self.permanent = data.get('permanent')
+        self.primary_key = data.get('primary_key')
+        if self.primary_key is not None:
+            self.tags['appenlight_primary_key'] = self.primary_key
+
+    def get_dict(self):
+        instance_dict = super(Log, self).get_dict()
+        instance_dict['log_level'] = LogLevel.key_from_value(self.log_level)
+        instance_dict['resource_name'] = self.application.resource_name
+        return instance_dict
+
+    @property
+    def delete_hash(self):
+        if not self.primary_key:
+            return None
+
+        to_hash = '{}_{}_{}'.format(self.resource_id, self.primary_key,
+                                     self.namespace)
+        return hashlib.sha1(to_hash.encode('utf8')).hexdigest()
+
+    def es_doc(self):
+        tags = {}
+        tag_list = []
+        for name, value in self.tags.items():
+            # replace dot in indexed tag name
+            name = name.replace('.', '_')
+            tag_list.append(name)
+            tags[name] = {
+                "values": convert_es_type(value),
+                "numeric_values": value if (
+                    isinstance(value, (int, float)) and
+                    not isinstance(value, bool)) else None
+            }
+        return {
+            'pg_id': str(self.log_id),
+            'delete_hash': self.delete_hash,
+            'resource_id': self.resource_id,
+            'request_id': self.request_id,
+            'log_level': LogLevel.key_from_value(self.log_level),
+            'timestamp': self.timestamp,
+            'message': self.message if self.message else '',
+            'namespace': self.namespace if self.namespace else '',
+            'tags': tags,
+            'tag_list': tag_list
+        }
+
+    @property
+    def partition_id(self):
+        if self.permanent:
+            return 'rcae_l_%s' % self.timestamp.strftime('%Y_%m')
+        else:
+            return 'rcae_l_%s' % self.timestamp.strftime('%Y_%m_%d')

backend/src/appenlight/models/plugin_config.py

@@ -0,0 +1,45 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from ziggurat_foundations.models.base import BaseModel
+from sqlalchemy.dialects.postgres import JSON
+
+from . import Base
+
+
+class PluginConfig(Base, BaseModel):
+    __tablename__ = 'plugin_configs'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    plugin_name = sa.Column(sa.Unicode)
+    section = sa.Column(sa.Unicode)
+    config = sa.Column(JSON, nullable=False)
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('resources.resource_id',
+                                          onupdate='cascade',
+                                          ondelete='cascade'))
+    owner_id = sa.Column(sa.Integer(),
+                         sa.ForeignKey('users.id', onupdate='cascade',
+                                       ondelete='cascade'))
+
+    def __json__(self, request):
+        return self.get_dict()

backend/src/appenlight/models/report.py

@@ -0,0 +1,489 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from datetime import datetime
+import math
+import uuid
+import hashlib
+import copy
+import urllib.parse
+import logging
+import sqlalchemy as sa
+
+from appenlight.models import Base, Datastores
+from appenlight.lib.utils.date_utils import convert_date
+from appenlight.lib.utils import convert_es_type
+from appenlight.models.slow_call import SlowCall
+from appenlight.lib.utils import cometd_request
+from appenlight.lib.enums import ReportType, Language
+from pyramid.threadlocal import get_current_registry, get_current_request
+from sqlalchemy.dialects.postgresql import JSON
+from ziggurat_foundations.models.base import BaseModel
+
+log = logging.getLogger(__name__)
+
+REPORT_TYPE_MATRIX = {
+    'http_status': {"type": 'int',
+                    "ops": ('eq', 'ne', 'ge', 'le',)},
+    'group:priority': {"type": 'int',
+                       "ops": ('eq', 'ne', 'ge', 'le',)},
+    'duration': {"type": 'float',
+                 "ops": ('ge', 'le',)},
+    'url_domain': {"type": 'unicode',
+                   "ops": ('eq', 'ne', 'startswith', 'endswith', 'contains',)},
+    'url_path': {"type": 'unicode',
+                 "ops": ('eq', 'ne', 'startswith', 'endswith', 'contains',)},
+    'error': {"type": 'unicode',
+              "ops": ('eq', 'ne', 'startswith', 'endswith', 'contains',)},
+    'tags:server_name': {"type": 'unicode',
+                         "ops": ('eq', 'ne', 'startswith', 'endswith',
+                                 'contains',)},
+    'traceback': {"type": 'unicode',
+                  "ops": ('contains',)},
+    'group:occurences': {"type": 'int',
+                         "ops": ('eq', 'ne', 'ge', 'le',)}
+}
+
+
+class Report(Base, BaseModel):
+    __tablename__ = 'reports'
+    __table_args__ = {'implicit_returning': False}
+
+    id = sa.Column(sa.Integer, nullable=False, primary_key=True)
+    group_id = sa.Column(sa.BigInteger,
+                         sa.ForeignKey('reports_groups.id', ondelete='cascade',
+                                       onupdate='cascade'))
+    resource_id = sa.Column(sa.Integer(), nullable=False, index=True)
+    report_type = sa.Column(sa.Integer(), nullable=False, index=True)
+    error = sa.Column(sa.UnicodeText(), index=True)
+    extra = sa.Column(JSON(), default={})
+    request = sa.Column(JSON(), nullable=False, default={})
+    ip = sa.Column(sa.String(39), index=True, default='')
+    username = sa.Column(sa.Unicode(255), default='')
+    user_agent = sa.Column(sa.Unicode(255), default='')
+    url = sa.Column(sa.UnicodeText(), index=True)
+    request_id = sa.Column(sa.Text())
+    request_stats = sa.Column(JSON(), nullable=False, default={})
+    traceback = sa.Column(JSON(), nullable=False, default=None)
+    traceback_hash = sa.Column(sa.Text())
+    start_time = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                           server_default=sa.func.now())
+    end_time = sa.Column(sa.DateTime())
+    duration = sa.Column(sa.Float, default=0)
+    http_status = sa.Column(sa.Integer, index=True)
+    url_domain = sa.Column(sa.Unicode(100), index=True)
+    url_path = sa.Column(sa.Unicode(255), index=True)
+    tags = sa.Column(JSON(), nullable=False, default={})
+    language = sa.Column(sa.Integer(), default=0)
+    # this is used to determine partition for the report
+    report_group_time = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                                  server_default=sa.func.now())
+
+    logs = sa.orm.relationship(
+        'Log',
+        lazy='dynamic',
+        passive_deletes=True,
+        passive_updates=True,
+        primaryjoin="and_(Report.request_id==Log.request_id, "
+                    "Log.request_id != None, Log.request_id != '')",
+        foreign_keys='[Log.request_id]')
+
+    slow_calls = sa.orm.relationship('SlowCall',
+                                     backref='detail',
+                                     cascade="all, delete-orphan",
+                                     passive_deletes=True,
+                                     passive_updates=True,
+                                     order_by='SlowCall.timestamp')
+
+    def set_data(self, data, resource, protocol_version=None):
+        self.http_status = data['http_status']
+        self.priority = data['priority']
+        self.error = data['error']
+        report_language = data.get('language', '').lower()
+        self.language = getattr(Language, report_language, Language.unknown)
+        # we need temp holder here to decide later
+        # if we want to to commit the tags if report is marked for creation
+        self.tags = {
+            'server_name': data['server'],
+            'view_name': data['view_name']
+        }
+        if data.get('tags'):
+            for tag_tuple in data['tags']:
+                self.tags[tag_tuple[0]] = tag_tuple[1]
+        self.traceback = data['traceback']
+        stripped_traceback = self.stripped_traceback()
+        tb_repr = repr(stripped_traceback).encode('utf8')
+        self.traceback_hash = hashlib.sha1(tb_repr).hexdigest()
+        url_info = urllib.parse.urlsplit(
+            data.get('url', ''), allow_fragments=False)
+        self.url_domain = url_info.netloc[:128]
+        self.url_path = url_info.path[:2048]
+        self.occurences = data['occurences']
+        if self.error:
+            self.report_type = ReportType.error
+        else:
+            self.report_type = ReportType.slow
+
+        # but if its status 404 its 404 type
+        if self.http_status in [404, '404'] or self.error == '404 Not Found':
+            self.report_type = ReportType.not_found
+            self.error = ''
+
+        self.generate_grouping_hash(data.get('appenlight.group_string',
+                                             data.get('group_string')),
+                                    resource.default_grouping,
+                                    protocol_version)
+
+        # details
+        if data['http_status'] in [404, '404']:
+            data = {"username": data["username"],
+                    "ip": data["ip"],
+                    "url": data["url"],
+                    "user_agent": data["user_agent"]}
+            if data.get('HTTP_REFERER') or data.get('http_referer'):
+                data['HTTP_REFERER'] = data.get(
+                    'HTTP_REFERER', '') or data.get('http_referer', '')
+
+        self.resource_id = resource.resource_id
+        self.username = data['username']
+        self.user_agent = data['user_agent']
+        self.ip = data['ip']
+        self.extra = {}
+        if data.get('extra'):
+            for extra_tuple in data['extra']:
+                self.extra[extra_tuple[0]] = extra_tuple[1]
+
+        self.url = data['url']
+        self.request_id = data.get('request_id', '').replace('-', '') or str(
+            uuid.uuid4())
+        request_data = data.get('request', {})
+
+        self.request = request_data
+        self.request_stats = data.get('request_stats', {})
+        traceback = data.get('traceback')
+        if not traceback:
+            traceback = data.get('frameinfo')
+        self.traceback = traceback
+        start_date = convert_date(data.get('start_time'))
+        if not self.start_time or self.start_time < start_date:
+            self.start_time = start_date
+
+        self.end_time = convert_date(data.get('end_time'), False)
+        self.duration = 0
+
+        if self.start_time and self.end_time:
+            d = self.end_time - self.start_time
+            self.duration = d.total_seconds()
+
+        # update tags with other vars
+        if self.username:
+            self.tags['user_name'] = self.username
+        self.tags['report_language'] = Language.key_from_value(self.language)
+
+    def add_slow_calls(self, data, report_group):
+        slow_calls = []
+        for call in data.get('slow_calls', []):
+            sc_inst = SlowCall()
+            sc_inst.set_data(call, resource_id=self.resource_id,
+                             report_group=report_group)
+            slow_calls.append(sc_inst)
+            self.slow_calls.extend(slow_calls)
+        return slow_calls
+
+    def get_dict(self, request, details=False, exclude_keys=None,
+                 include_keys=None):
+        from appenlight.models.services.report_group import ReportGroupService
+        instance_dict = super(Report, self).get_dict()
+        instance_dict['req_stats'] = self.req_stats()
+        instance_dict['group'] = {}
+        instance_dict['group']['id'] = self.report_group.id
+        instance_dict['group'][
+            'total_reports'] = self.report_group.total_reports
+        instance_dict['group']['last_report'] = self.report_group.last_report
+        instance_dict['group']['priority'] = self.report_group.priority
+        instance_dict['group']['occurences'] = self.report_group.occurences
+        instance_dict['group'][
+            'last_timestamp'] = self.report_group.last_timestamp
+        instance_dict['group'][
+            'first_timestamp'] = self.report_group.first_timestamp
+        instance_dict['group']['public'] = self.report_group.public
+        instance_dict['group']['fixed'] = self.report_group.fixed
+        instance_dict['group']['read'] = self.report_group.read
+        instance_dict['group'][
+            'average_duration'] = self.report_group.average_duration
+
+        instance_dict[
+            'resource_name'] = self.report_group.application.resource_name
+        instance_dict['report_type'] = self.report_type
+
+        if instance_dict['http_status'] == 404 and not instance_dict['error']:
+            instance_dict['error'] = '404 Not Found'
+
+        if details:
+            instance_dict['affected_users_count'] = \
+                ReportGroupService.affected_users_count(self.report_group)
+            instance_dict['top_affected_users'] = [
+                {'username': u.username, 'count': u.count} for u in
+                ReportGroupService.top_affected_users(self.report_group)]
+            instance_dict['application'] = {'integrations': []}
+            for integration in self.report_group.application.integrations:
+                if integration.front_visible:
+                    instance_dict['application']['integrations'].append(
+                        {'name': integration.integration_name,
+                         'action': integration.integration_action})
+            instance_dict['comments'] = [c.get_dict() for c in
+                                         self.report_group.comments]
+
+            instance_dict['group']['next_report'] = None
+            instance_dict['group']['previous_report'] = None
+            next_in_group = self.get_next_in_group()
+            previous_in_group = self.get_previous_in_group()
+            if next_in_group:
+                instance_dict['group']['next_report'] = next_in_group.id
+            if previous_in_group:
+                instance_dict['group']['previous_report'] = \
+                    previous_in_group.id
+
+            # slow call ordering
+            def find_parent(row, data):
+                for r in reversed(data):
+                    try:
+                        if (row['timestamp'] > r['timestamp'] and
+                                    row['end_time'] < r['end_time']):
+                            return r
+                    except TypeError as e:
+                        log.warning('reports_view.find_parent: %s' % e)
+                return None
+
+            new_calls = []
+            calls = [c.get_dict() for c in self.slow_calls]
+            while calls:
+                # start from end
+                for x in range(len(calls) - 1, -1, -1):
+                    parent = find_parent(calls[x], calls)
+                    if parent:
+                        parent['children'].append(calls[x])
+                    else:
+                        # no parent at all? append to new calls anyways
+                        new_calls.append(calls[x])
+                        # print 'append', calls[x]
+                    del calls[x]
+                    break
+            instance_dict['slow_calls'] = new_calls
+
+        instance_dict['front_url'] = self.get_public_url(request)
+
+        exclude_keys_list = exclude_keys or []
+        include_keys_list = include_keys or []
+        for k in list(instance_dict.keys()):
+            if k == 'group':
+                continue
+            if (k in exclude_keys_list or
+                    (k not in include_keys_list and include_keys)):
+                del instance_dict[k]
+        return instance_dict
+
+    def get_previous_in_group(self):
+        start_day = self.report_group_time.date().replace(day=1)
+        end_day = start_day.replace(month=start_day.month+1)
+        query = self.report_group.reports.filter(Report.id < self.id)
+        query = query.filter(Report.report_group_time.between(
+            start_day, end_day))
+        return query.order_by(sa.desc(Report.id)).first()
+
+    def get_next_in_group(self):
+        start_day = self.report_group_time.date().replace(day=1)
+        end_day = start_day.replace(month=start_day.month+1)
+        query = self.report_group.reports.filter(Report.id > self.id)
+        query = query.filter(Report.report_group_time.between(
+            start_day, end_day))
+        return query.order_by(sa.asc(Report.id)).first()
+
+    def get_public_url(self, request=None, report_group=None, _app_url=None):
+        """
+        Returns url that user can use to visit specific report
+        """
+        if not request:
+            request = get_current_request()
+        url = request.route_url('/', _app_url=_app_url)
+        if report_group:
+            return (url + 'ui/report/%s/%s') % (report_group.id, self.id)
+        return (url + 'ui/report/%s/%s') % (self.group_id, self.id)
+
+    def req_stats(self):
+        stats = self.request_stats.copy()
+        stats['percentages'] = {}
+        stats['percentages']['main'] = 100.0
+        main = stats.get('main', 0.0)
+        if not main:
+            return None
+        for name, call_time in stats.items():
+            if ('calls' not in name and 'main' not in name and
+                        'percentages' not in name):
+                stats['main'] -= call_time
+                stats['percentages'][name] = math.floor(
+                    (call_time / main * 100.0))
+                stats['percentages']['main'] -= stats['percentages'][name]
+        if stats['percentages']['main'] < 0.0:
+            stats['percentages']['main'] = 0.0
+            stats['main'] = 0.0
+        return stats
+
+    def generate_grouping_hash(self, hash_string=None, default_grouping=None,
+                               protocol_version=None):
+        """
+        Generates SHA1 hash that will be used to group reports together
+        """
+        if not hash_string:
+            location = self.tags.get('view_name') or self.url_path;
+            server_name = self.tags.get('server_name') or ''
+            if default_grouping == 'url_traceback':
+                hash_string = '%s_%s_%s' % (self.traceback_hash, location,
+                                             self.error)
+                if self.language == Language.javascript:
+                    hash_string = '%s_%s' % (self.traceback_hash, self.error)
+
+            elif default_grouping == 'traceback_server':
+                hash_string = '%s_%s' % (self.traceback_hash, server_name)
+                if self.language == Language.javascript:
+                    hash_string = '%s_%s' % (self.traceback_hash, server_name)
+            else:
+                hash_string = '%s_%s' % (self.error, location)
+        binary_string = hash_string.encode('utf8')
+        self.grouping_hash = hashlib.sha1(binary_string).hexdigest()
+        return self.grouping_hash
+
+    def stripped_traceback(self):
+        """
+        Traceback without local vars
+        """
+        stripped_traceback = copy.deepcopy(self.traceback)
+
+        if isinstance(stripped_traceback, list):
+            for row in stripped_traceback:
+                row.pop('vars', None)
+        return stripped_traceback
+
+    def notify_channel(self, report_group):
+        """
+        Sends notification to websocket channel
+        """
+        settings = get_current_registry().settings
+        log.info('notify cometd')
+        if self.report_type != ReportType.error:
+            return
+        payload = {
+            'type': 'message',
+            "user": '__system__',
+            "channel": 'app_%s' % self.resource_id,
+            'message': {
+                'type': 'report',
+                'report': {
+                    'group': {
+                        'priority': report_group.priority,
+                        'first_timestamp': report_group.first_timestamp,
+                        'last_timestamp': report_group.last_timestamp,
+                        'average_duration': report_group.average_duration,
+                        'occurences': report_group.occurences
+                    },
+                    'report_id': self.id,
+                    'group_id': self.group_id,
+                    'resource_id': self.resource_id,
+                    'http_status': self.http_status,
+                    'url_domain': self.url_domain,
+                    'url_path': self.url_path,
+                    'error': self.error or '',
+                    'server': self.tags.get('server_name'),
+                    'view_name': self.tags.get('view_name'),
+                    'front_url': self.get_public_url(),
+                }
+            }
+
+        }
+
+        cometd_request(settings['cometd.secret'], '/message', [payload],
+                       servers=[settings['cometd_servers']])
+
+    def es_doc(self):
+        tags = {}
+        tag_list = []
+        for name, value in self.tags.items():
+            name = name.replace('.', '_')
+            tag_list.append(name)
+            tags[name] = {
+                "values": convert_es_type(value),
+                "numeric_values": value if (
+                    isinstance(value, (int, float)) and
+                    not isinstance(value, bool)) else None}
+
+        if 'user_name' not in self.tags and self.username:
+            tags["user_name"] = {"value": [self.username],
+                                 "numeric_value": None}
+        return {
+            '_id': str(self.id),
+            'pg_id': str(self.id),
+            'resource_id': self.resource_id,
+            'http_status': self.http_status or '',
+            'start_time': self.start_time,
+            'end_time': self.end_time,
+            'url_domain': self.url_domain if self.url_domain else '',
+            'url_path': self.url_path if self.url_path else '',
+            'duration': self.duration,
+            'error': self.error if self.error else '',
+            'report_type': self.report_type,
+            'request_id': self.request_id,
+            'ip': self.ip,
+            'group_id': str(self.group_id),
+            '_parent': str(self.group_id),
+            'tags': tags,
+            'tag_list': tag_list
+        }
+
+    @property
+    def partition_id(self):
+        return 'rcae_r_%s' % self.report_group_time.strftime('%Y_%m')
+
+
+def after_insert(mapper, connection, target):
+    if not hasattr(target, '_skip_ft_index'):
+        data = target.es_doc()
+        data.pop('_id', None)
+        Datastores.es.index(target.partition_id, 'report', data,
+                            parent=target.group_id, id=target.id)
+
+
+def after_update(mapper, connection, target):
+    if not hasattr(target, '_skip_ft_index'):
+        data = target.es_doc()
+        data.pop('_id', None)
+        Datastores.es.index(target.partition_id, 'report', data,
+                            parent=target.group_id, id=target.id)
+
+
+def after_delete(mapper, connection, target):
+    if not hasattr(target, '_skip_ft_index'):
+        query = {'term': {'pg_id': target.id}}
+        Datastores.es.delete_by_query(target.partition_id, 'report', query)
+
+
+sa.event.listen(Report, 'after_insert', after_insert)
+sa.event.listen(Report, 'after_update', after_update)
+sa.event.listen(Report, 'after_delete', after_delete)

backend/src/appenlight/models/report_assignment.py

@@ -0,0 +1,37 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.base import BaseModel
+from appenlight.models import Base
+import sqlalchemy as sa
+
+
+class ReportAssignment(Base, BaseModel):
+    __tablename__ = 'reports_assignments'
+
+    group_id = sa.Column(sa.BigInteger,
+                         sa.ForeignKey('reports_groups.id', ondelete='cascade',
+                                       onupdate='cascade'),
+                         primary_key=True)
+    owner_id = sa.Column(sa.Integer,
+                          sa.ForeignKey('users.id', onupdate='CASCADE',
+                                        ondelete='CASCADE'), primary_key=True)
+    report_time = sa.Column(sa.DateTime(), nullable=False)

backend/src/appenlight/models/report_comment.py

@@ -0,0 +1,55 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+
+from datetime import datetime
+from appenlight.models import Base
+from ziggurat_foundations.models.base import BaseModel
+
+
+class ReportComment(Base, BaseModel):
+    __tablename__ = 'reports_comments'
+
+    comment_id = sa.Column(sa.Integer, nullable=False, primary_key=True)
+    group_id = sa.Column(sa.BigInteger,
+                         sa.ForeignKey('reports_groups.id', ondelete='cascade',
+                                       onupdate='cascade'))
+    body = sa.Column(sa.UnicodeText(), default='')
+    owner_id = sa.Column(sa.Integer,
+                         sa.ForeignKey('users.id', onupdate='CASCADE',
+                                       ondelete='CASCADE'))
+    created_timestamp = sa.Column(sa.DateTime(),
+                                  default=datetime.utcnow,
+                                  server_default=sa.func.now())
+    report_time = sa.Column(sa.DateTime(), nullable=False)
+
+    owner = sa.orm.relationship('User',
+                                lazy='joined')
+
+    @property
+    def processed_body(self):
+        return self.body
+
+    def get_dict(self):
+        instance_dict = super(ReportComment, self).get_dict()
+        instance_dict['user_name'] = self.owner.user_name
+        return instance_dict

backend/src/appenlight/models/report_group.py

@@ -0,0 +1,251 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import sqlalchemy as sa
+
+from datetime import datetime
+
+from pyramid.threadlocal import get_current_request
+from sqlalchemy.dialects.postgresql import JSON
+from ziggurat_foundations.models.base import BaseModel
+
+from appenlight.models import Base, get_db_session, Datastores
+from appenlight.lib.enums import ReportType
+from appenlight.lib.rule import Rule
+from appenlight.lib.redis_keys import REDIS_KEYS
+from appenlight.models.report import REPORT_TYPE_MATRIX
+
+log = logging.getLogger(__name__)
+
+
+class ReportGroup(Base, BaseModel):
+    __tablename__ = 'reports_groups'
+    __table_args__ = {'implicit_returning': False}
+
+    id = sa.Column(sa.BigInteger(), nullable=False, primary_key=True)
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('applications.resource_id',
+                                          onupdate='CASCADE',
+                                          ondelete='CASCADE'),
+                            nullable=False,
+                            index=True)
+    priority = sa.Column(sa.Integer, nullable=False, index=True, default=5,
+                         server_default='5')
+    first_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                                server_default=sa.func.now())
+    last_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                               server_default=sa.func.now())
+    error = sa.Column(sa.UnicodeText(), index=True)
+    grouping_hash = sa.Column(sa.String(40), default='')
+    triggered_postprocesses_ids = sa.Column(JSON(), nullable=False,
+                                            default=list)
+    report_type = sa.Column(sa.Integer, default=1)
+    total_reports = sa.Column(sa.Integer, default=1)
+    last_report = sa.Column(sa.Integer)
+    occurences = sa.Column(sa.Integer, default=1)
+    average_duration = sa.Column(sa.Float, default=0)
+    summed_duration = sa.Column(sa.Float, default=0)
+    read = sa.Column(sa.Boolean(), index=True, default=False)
+    fixed = sa.Column(sa.Boolean(), index=True, default=False)
+    notified = sa.Column(sa.Boolean(), index=True, default=False)
+    public = sa.Column(sa.Boolean(), index=True, default=False)
+
+    reports = sa.orm.relationship('Report',
+                                  lazy='dynamic',
+                                  backref='report_group',
+                                  cascade="all, delete-orphan",
+                                  passive_deletes=True,
+                                  passive_updates=True, )
+
+    comments = sa.orm.relationship('ReportComment',
+                                   lazy='dynamic',
+                                   backref='report',
+                                   cascade="all, delete-orphan",
+                                   passive_deletes=True,
+                                   passive_updates=True,
+                                   order_by="ReportComment.comment_id")
+
+    assigned_users = sa.orm.relationship('User',
+                                         backref=sa.orm.backref(
+                                             'assigned_reports_relation',
+                                             lazy='dynamic',
+                                             order_by=sa.desc(
+                                                 "reports_groups.id")
+                                         ),
+                                         passive_deletes=True,
+                                         passive_updates=True,
+                                         secondary='reports_assignments',
+                                         order_by="User.user_name")
+
+    stats = sa.orm.relationship('ReportStat',
+                                lazy='dynamic',
+                                backref='report',
+                                passive_deletes=True,
+                                passive_updates=True, )
+
+    last_report_ref = sa.orm.relationship('Report',
+                                          uselist=False,
+                                          primaryjoin="ReportGroup.last_report "
+                                                      "== Report.id",
+                                          foreign_keys="Report.id",
+                                          cascade="all, delete-orphan",
+                                          passive_deletes=True,
+                                          passive_updates=True, )
+
+    def __repr__(self):
+        return '<ReportGroup id:{}>'.format(self.id)
+
+    def get_report(self, report_id=None, public=False):
+        """
+        Gets report with specific id or latest report if id was not specified
+        """
+        from .report import Report
+
+        if not report_id:
+            return self.last_report_ref
+        else:
+            return self.reports.filter(Report.id == report_id).first()
+
+    def get_public_url(self, request, _app_url=None):
+        url = request.route_url('/', _app_url=_app_url)
+        return (url + 'ui/report/%s') % self.id
+
+    def run_postprocessing(self, report):
+        """
+        Alters report group priority based on postprocessing configuration
+        """
+        request = get_current_request()
+        get_db_session(None, self).flush()
+        for action in self.application.postprocess_conf:
+            get_db_session(None, self).flush()
+            rule_obj = Rule(action.rule, REPORT_TYPE_MATRIX)
+            report_dict = report.get_dict(request)
+            # if was not processed yet
+            if (rule_obj.match(report_dict) and
+                    action.pkey not in self.triggered_postprocesses_ids):
+                action.postprocess(self)
+                # this way sqla can track mutation of list
+                self.triggered_postprocesses_ids = \
+                    self.triggered_postprocesses_ids + [action.pkey]
+
+        get_db_session(None, self).flush()
+        # do not go out of bounds
+        if self.priority < 1:
+            self.priority = 1
+        if self.priority > 10:
+            self.priority = 10
+
+    def get_dict(self, request):
+        instance_dict = super(ReportGroup, self).get_dict()
+        instance_dict['server_name'] = self.get_report().tags.get(
+            'server_name')
+        instance_dict['view_name'] = self.get_report().tags.get('view_name')
+        instance_dict['resource_name'] = self.application.resource_name
+        instance_dict['report_type'] = self.get_report().report_type
+        instance_dict['url_path'] = self.get_report().url_path
+        instance_dict['front_url'] = self.get_report().get_public_url(request)
+        del instance_dict['triggered_postprocesses_ids']
+        return instance_dict
+
+    def es_doc(self):
+        return {
+            '_id': str(self.id),
+            'pg_id': str(self.id),
+            'resource_id': self.resource_id,
+            'error': self.error,
+            'fixed': self.fixed,
+            'public': self.public,
+            'read': self.read,
+            'priority': self.priority,
+            'occurences': self.occurences,
+            'average_duration': self.average_duration,
+            'summed_duration': self.summed_duration,
+            'first_timestamp': self.first_timestamp,
+            'last_timestamp': self.last_timestamp
+        }
+
+    def set_notification_info(self, notify_10=False, notify_100=False):
+        """
+        Update redis notification maps for notification job
+        """
+        current_time = datetime.utcnow().replace(second=0, microsecond=0)
+        # global app counter
+        key = REDIS_KEYS['counters']['reports_per_type'].format(
+            self.report_type, current_time)
+        Datastores.redis.incr(key)
+        Datastores.redis.expire(key, 3600 * 24)
+        # detailed app notification
+        Datastores.redis.sadd(REDIS_KEYS['apps_that_had_reports'],
+                              self.resource_id)
+        # only notify for exceptions here
+        if self.report_type == ReportType.error:
+            Datastores.redis.sadd(REDIS_KEYS['apps_that_had_reports'],
+                                  self.resource_id)
+        key = REDIS_KEYS['counters']['report_group_occurences'].format(self.id)
+        Datastores.redis.incr(key)
+        Datastores.redis.expire(key, 3600 * 24)
+
+        if notify_10:
+            key = REDIS_KEYS['counters'][
+                'report_group_occurences_10th'].format(self.id)
+            Datastores.redis.setex(key, 3600 * 24, 1)
+        if notify_100:
+            key = REDIS_KEYS['counters'][
+                'report_group_occurences_100th'].format(self.id)
+            Datastores.redis.setex(key, 3600 * 24, 1)
+
+        key = REDIS_KEYS['reports_to_notify_per_type_per_app'].format(
+            self.report_type, self.resource_id)
+        Datastores.redis.sadd(key, self.id)
+        Datastores.redis.expire(key, 3600 * 24)
+
+    @property
+    def partition_id(self):
+        return 'rcae_r_%s' % self.first_timestamp.strftime('%Y_%m')
+
+
+def after_insert(mapper, connection, target):
+    if not hasattr(target, '_skip_ft_index'):
+        data = target.es_doc()
+        data.pop('_id', None)
+        Datastores.es.index(target.partition_id, 'report_group',
+                            data, id=target.id)
+
+
+def after_update(mapper, connection, target):
+    if not hasattr(target, '_skip_ft_index'):
+        data = target.es_doc()
+        data.pop('_id', None)
+        Datastores.es.index(target.partition_id, 'report_group',
+                            data, id=target.id)
+
+
+def after_delete(mapper, connection, target):
+    query = {'term': {'group_id': target.id}}
+    Datastores.es.delete_by_query(target.partition_id, 'report', query)
+    query = {'term': {'pg_id': target.id}}
+    Datastores.es.delete_by_query(target.partition_id, 'report_group', query)
+
+
+sa.event.listen(ReportGroup, 'after_insert', after_insert)
+sa.event.listen(ReportGroup, 'after_update', after_update)
+sa.event.listen(ReportGroup, 'after_delete', after_delete)

backend/src/appenlight/models/report_stat.py

@@ -0,0 +1,79 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+
+from appenlight.lib.enums import ReportType
+from appenlight.models import Base
+from ziggurat_foundations.models.base import BaseModel
+
+
+class ReportStat(Base, BaseModel):
+    __tablename__ = 'reports_stats'
+    __table_args__ = {'implicit_returning': False}
+
+    group_id = sa.Column(sa.BigInteger(),
+                         sa.ForeignKey('reports_groups.id'),
+                         nullable=False)
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('applications.resource_id'),
+                            nullable=False)
+    start_interval = sa.Column(sa.DateTime(), nullable=False)
+    occurences = sa.Column(sa.Integer, nullable=True, default=0)
+    owner_user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'),
+                              nullable=True)
+    type = sa.Column(sa.Integer, nullable=True, default=0)
+    duration = sa.Column(sa.Float, nullable=True, default=0)
+    id = sa.Column(sa.BigInteger, nullable=False, primary_key=True)
+    server_name = sa.Column(sa.Unicode(128), nullable=False, default='')
+    view_name = sa.Column(sa.Unicode(128), nullable=False, default='')
+
+    @property
+    def partition_id(self):
+        return 'rcae_r_%s' % self.start_interval.strftime('%Y_%m')
+
+    def es_doc(self):
+        return {
+            'resource_id': self.resource_id,
+            'timestamp': self.start_interval,
+            'pg_id': str(self.id),
+            'permanent': True,
+            'request_id': None,
+            'log_level': 'ERROR',
+            'message': None,
+            'namespace': 'appenlight.error',
+            'tags': {
+                'duration': {'values': self.duration,
+                             'numeric_values': self.duration},
+                'occurences': {'values': self.occurences,
+                               'numeric_values': self.occurences},
+                'group_id': {'values': self.group_id,
+                             'numeric_values': self.group_id},
+                'type': {'values': ReportType.key_from_value(self.type),
+                         'numeric_values': self.type},
+                'server_name': {'values': self.server_name,
+                                'numeric_values': None},
+                'view_name': {'values': self.view_name,
+                              'numeric_values': None},
+            },
+            'tag_list': ['duration', 'occurences', 'group_id', 'type',
+                         'server_name', 'view_name']
+        }

backend/src/appenlight/models/request_metric.py

@@ -0,0 +1,69 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from datetime import datetime
+
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSON
+
+from ziggurat_foundations.models.base import BaseModel
+from appenlight.lib.utils import convert_es_type
+from appenlight.models import Base
+
+
+class Metric(Base, BaseModel):
+    __tablename__ = 'metrics'
+    __table_args__ = {'implicit_returning': False}
+
+    pkey = sa.Column(sa.BigInteger(), primary_key=True)
+    resource_id = sa.Column(sa.Integer(),
+                            sa.ForeignKey('applications.resource_id'),
+                            nullable=False, primary_key=True)
+    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                          server_default=sa.func.now())
+    tags = sa.Column(JSON(), default={})
+    namespace = sa.Column(sa.Unicode(255))
+
+    @property
+    def partition_id(self):
+        return 'rcae_m_%s' % self.timestamp.strftime('%Y_%m_%d')
+
+    def es_doc(self):
+        tags = {}
+        tag_list = []
+        for name, value in self.tags.items():
+            # replace dot in indexed tag name
+            name = name.replace('.', '_')
+            tag_list.append(name)
+            tags[name] = {
+                "values": convert_es_type(value),
+                "numeric_values": value if (
+                    isinstance(value, (int, float)) and
+                    not isinstance(value, bool)) else None
+            }
+
+        return {
+            'resource_id': self.resource_id,
+            'timestamp': self.timestamp,
+            'namespace': self.namespace,
+            'tags': tags,
+            'tag_list': tag_list
+        }

backend/src/appenlight/models/resource.py

@@ -0,0 +1,88 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from appenlight.models import Base
+from appenlight.lib.utils import permission_tuple_to_dict
+from pyramid.security import Allow, ALL_PERMISSIONS
+from ziggurat_foundations.models.resource import ResourceMixin
+
+
+class Resource(ResourceMixin, Base):
+    events = sa.orm.relationship('Event',
+                                 lazy='dynamic',
+                                 backref='resource',
+                                 passive_deletes=True,
+                                 passive_updates=True)
+
+    @property
+    def owner_user_name(self):
+        if self.owner:
+            return self.owner.user_name
+
+    @property
+    def owner_group_name(self):
+        if self.owner_group:
+            return self.owner_group.group_name
+
+    def get_dict(self, exclude_keys=None, include_keys=None,
+                 include_perms=False, include_processing_rules=False):
+        result = super(Resource, self).get_dict(exclude_keys, include_keys)
+        result['possible_permissions'] = self.__possible_permissions__
+        if include_perms:
+            result['current_permissions'] = self.user_permissions_list
+        else:
+            result['current_permissions'] = []
+        if include_processing_rules:
+            result["postprocessing_rules"] = [rule.get_dict() for rule
+                                              in self.postprocess_conf]
+        else:
+            result["postprocessing_rules"] = []
+        exclude_keys_list = exclude_keys or []
+        include_keys_list = include_keys or []
+        d = {}
+        for k in result.keys():
+            if (k not in exclude_keys_list and
+                    (k in include_keys_list or not include_keys)):
+                d[k] = result[k]
+        for k in ['owner_user_name', 'owner_group_name']:
+            if (k not in exclude_keys_list and
+                    (k in include_keys_list or not include_keys)):
+                d[k] = getattr(self, k)
+        return d
+
+    @property
+    def user_permissions_list(self):
+        return [permission_tuple_to_dict(perm) for perm in
+                self.users_for_perm('__any_permission__',
+                                    limit_group_permissions=True)]
+
+    @property
+    def __acl__(self):
+        acls = []
+
+        if self.owner_user_id:
+            acls.extend([(Allow, self.owner_user_id, ALL_PERMISSIONS,), ])
+
+        if self.owner_group_id:
+            acls.extend([(Allow, "group:%s" % self.owner_group_id,
+                          ALL_PERMISSIONS,), ])
+        return acls

backend/src/appenlight/models/services/__init__.py

@@ -0,0 +1,21 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+

backend/src/appenlight/models/services/alert_channel.py

@@ -0,0 +1,40 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from appenlight.models import get_db_session
+from appenlight.models.alert_channel import AlertChannel
+from appenlight.models.services.base import BaseService
+
+
+class AlertChannelService(BaseService):
+    @classmethod
+    def by_owner_id_and_pkey(cls, owner_id, pkey, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannel)
+        query = query.filter(AlertChannel.owner_id == owner_id)
+        return query.filter(AlertChannel.pkey == pkey).first()
+
+    @classmethod
+    def by_integration_id(cls, integration_id, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannel)
+        query = query.filter(AlertChannel.integration_id == integration_id)
+        return query.first()

backend/src/appenlight/models/services/alert_channel_action.py

@@ -0,0 +1,64 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from appenlight.models import get_db_session
+from appenlight.models.alert_channel_action import AlertChannelAction
+from appenlight.models.services.base import BaseService
+
+
+class AlertChannelActionService(BaseService):
+    @classmethod
+    def by_owner_id_and_pkey(cls, owner_id, pkey, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannelAction)
+        query = query.filter(AlertChannelAction.owner_id == owner_id)
+        return query.filter(AlertChannelAction.pkey == pkey).first()
+
+    @classmethod
+    def by_pkey(cls, pkey, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannelAction)
+        return query.filter(AlertChannelAction.pkey == pkey).first()
+
+    @classmethod
+    def by_owner_id_and_type(cls, owner_id, alert_type, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannelAction)
+        query = query.filter(AlertChannelAction.owner_id == owner_id)
+        return query.filter(AlertChannelAction.type == alert_type).first()
+
+    @classmethod
+    def by_type(cls, alert_type, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannelAction)
+        return query.filter(AlertChannelAction.type == alert_type)
+
+    @classmethod
+    def by_other_id(cls, other_id, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannelAction)
+        return query.filter(AlertChannelAction.other_id == other_id)
+
+    @classmethod
+    def by_resource_id(cls, resource_id, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AlertChannelAction)
+        return query.filter(AlertChannelAction.resource_id == resource_id)

backend/src/appenlight/models/services/application.py

@@ -0,0 +1,193 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import urllib.parse
+
+import sqlalchemy as sa
+from pyramid.threadlocal import get_current_request
+
+from appenlight.lib.cache_regions import get_region
+from appenlight.lib.enums import ReportType
+from appenlight.models import get_db_session
+from appenlight.models.report_group import ReportGroup
+from appenlight.models.application import Application
+from appenlight.models.event import Event
+from appenlight.models.services.base import BaseService
+from appenlight.models.services.event import EventService
+
+log = logging.getLogger(__name__)
+
+
+# cache_memory_min_1 = get_region('memory_min_1')
+
+class ApplicationService(BaseService):
+    @classmethod
+    def by_api_key(cls, api_key, db_session=None):
+        db_session = get_db_session(db_session)
+        q = db_session.query(Application)
+        q = q.filter(Application.api_key == api_key)
+        q = q.options(sa.orm.eagerload(Application.owner))
+        return q.first()
+
+    @classmethod
+    def by_api_key_cached(cls, db_session=None):
+        db_session = get_db_session(db_session)
+        cache_region = get_region('redis_min_1')
+
+        @cache_region.cache_on_arguments('ApplicationService.by_api_key')
+        def cached(*args, **kwargs):
+            app = cls.by_api_key(*args, db_session=db_session, **kwargs)
+            if app:
+                db_session.expunge(app)
+            return app
+
+        return cached
+
+    @classmethod
+    def by_public_api_key(cls, api_key, db_session=None, from_cache=False,
+                          request=None):
+        db_session = get_db_session(db_session)
+        cache_region = get_region('redis_min_1')
+
+        def uncached(api_key):
+            q = db_session.query(Application)
+            q = q.filter(Application.public_key == api_key)
+            q = q.options(sa.orm.eagerload(Application.owner))
+            return q.first()
+
+        if from_cache:
+            @cache_region.cache_on_arguments(
+                'ApplicationService.by_public_api_key')
+            def cached(api_key):
+                app = uncached(api_key)
+                if app:
+                    db_session.expunge(app)
+                return app
+
+            app = cached(api_key)
+        else:
+            app = uncached(api_key)
+        return app
+
+    @classmethod
+    def by_id(cls, db_id, db_session=None):
+        db_session = get_db_session(db_session)
+        q = db_session.query(Application)
+        q = q.filter(Application.resource_id == db_id)
+        return q.first()
+
+    @classmethod
+    def by_id_cached(cls, db_session=None):
+        db_session = get_db_session(db_session)
+        cache_region = get_region('redis_min_1')
+
+        @cache_region.cache_on_arguments('ApplicationService.by_id')
+        def cached(*args, **kwargs):
+            app = cls.by_id(*args, db_session=db_session, **kwargs)
+            if app:
+                db_session.expunge(app)
+            return app
+
+        return cached
+
+    @classmethod
+    def by_ids(cls, db_ids, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(Application)
+        query = query.filter(Application.resource_id.in_(db_ids))
+        return query
+
+    @classmethod
+    def by_http_referer(cls, referer_string, db_session=None):
+        db_session = get_db_session(db_session)
+        domain = urllib.parse.urlsplit(
+            referer_string, allow_fragments=False).netloc
+        if domain:
+            if domain.startswith('www.'):
+                domain = domain[4:]
+        q = db_session.query(Application).filter(Application.domain == domain)
+        return q.first()
+
+    @classmethod
+    def last_updated(cls, since_when, exclude_status=None, db_session=None):
+        db_session = get_db_session(db_session)
+        q = db_session.query(Application)
+        q2 = ReportGroup.last_updated(
+            since_when, exclude_status=exclude_status, db_session=db_session)
+        q2 = q2.from_self(ReportGroup.resource_id)
+        q2 = q2.group_by(ReportGroup.resource_id)
+        q = q.filter(Application.resource_id.in_(q2))
+        return q
+
+    @classmethod
+    def check_for_groups_alert(cls, resource, event_type, *args, **kwargs):
+        """ Check for open alerts depending on group type.
+        Create new one if nothing is found and send alerts """
+        db_session = get_db_session(kwargs.get('db_session'))
+        request = get_current_request()
+        report_groups = kwargs['report_groups']
+        occurence_dict = kwargs['occurence_dict']
+
+        error_reports = 0
+        slow_reports = 0
+        for group in report_groups:
+            occurences = occurence_dict.get(group.id, 1)
+            if group.get_report().report_type == ReportType.error:
+                error_reports += occurences
+            elif group.get_report().report_type == ReportType.slow:
+                slow_reports += occurences
+
+        log_msg = 'LIMIT INFO: %s : %s error reports. %s slow_reports' % (
+            resource,
+            error_reports,
+            slow_reports)
+        logging.warning(log_msg)
+        threshold = 10
+        for event_type in ['error_report_alert', 'slow_report_alert']:
+            if (error_reports < resource.error_report_threshold and
+                        event_type == 'error_report_alert'):
+                continue
+            elif (slow_reports <= resource.slow_report_threshold and
+                          event_type == 'slow_report_alert'):
+                continue
+            if event_type == 'error_report_alert':
+                amount = error_reports
+                threshold = resource.error_report_threshold
+            elif event_type == 'slow_report_alert':
+                amount = slow_reports
+                threshold = resource.slow_report_threshold
+
+            event = EventService.for_resource([resource.resource_id],
+                                              event_type=Event.types[
+                                                  event_type],
+                                              status=Event.statuses['active'])
+            if event.first():
+                log.info('ALERT: PROGRESS: %s %s' % (event_type, resource))
+            else:
+                log.warning('ALERT: OPEN: %s %s' % (event_type, resource))
+                new_event = Event(resource_id=resource.resource_id,
+                                  event_type=Event.types[event_type],
+                                  status=Event.statuses['active'],
+                                  values={'reports': amount,
+                                          'threshold': threshold})
+                db_session.add(new_event)
+                new_event.send_alerts(request=request, resource=resource)

backend/src/appenlight/models/services/application_postprocess_conf.py

@@ -0,0 +1,41 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+
+from appenlight.models import get_db_session
+from appenlight.models.application_postprocess_conf import ApplicationPostprocessConf
+from appenlight.models.services.base import BaseService
+
+
+class ApplicationPostprocessConfService(BaseService):
+
+    @classmethod
+    def by_pkey(cls, pkey, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(ApplicationPostprocessConf)
+        return query.filter(ApplicationPostprocessConf.pkey == pkey).first()
+
+    @classmethod
+    def by_pkey_and_resource_id(cls, pkey, resource_id, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(ApplicationPostprocessConf)
+        query = query.filter(ApplicationPostprocessConf.resource_id == resource_id)
+        return query.filter(ApplicationPostprocessConf.pkey == pkey).first()

backend/src/appenlight/models/services/auth_token.py

@@ -0,0 +1,36 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+from appenlight.models.auth_token import AuthToken
+from appenlight.models.services.base import BaseService
+from ziggurat_foundations.models.base import get_db_session
+
+log = logging.getLogger(__name__)
+
+
+class AuthTokenService(BaseService):
+    @classmethod
+    def by_token(cls, token, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(AuthToken).filter(AuthToken.token == token)
+        return query.first()

backend/src/appenlight/models/services/base.py

@@ -0,0 +1,24 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+
+class BaseService(object):
+    pass

backend/src/appenlight/models/services/config.py

@@ -0,0 +1,105 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+import sqlalchemy as sa
+from pyramid.threadlocal import get_current_registry
+
+from appenlight.models.config import Config
+from appenlight.models.services.base import BaseService
+from appenlight.models import get_db_session
+
+log = logging.getLogger(__name__)
+
+
+class ConfigService(BaseService):
+    @classmethod
+    def all(cls, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(Config)
+        return query
+
+    @classmethod
+    def filtered_key_and_section(cls, pairs=None, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(Config)
+        if pairs:
+            conditions = []
+            for pair in pairs:
+                conditions.append(sa.and_(
+                    Config.key == pair['key'],
+                    Config.section == pair['section'])
+                )
+
+            query = query.filter(sa.or_(*conditions))
+        return query
+
+    @classmethod
+    def create_config(cls, key, section, value=None, db_session=None):
+        config = Config(key=key, section=section, value=value)
+        db_session = get_db_session(db_session)
+        db_session.add(config)
+        db_session.flush()
+        return config
+
+    @classmethod
+    def by_key_and_section(cls, key, section, auto_create=False,
+                           default_value=None, db_session=None):
+        db_session = get_db_session(db_session)
+        registry = get_current_registry()
+
+        @registry.cache_regions.memory_min_1.cache_on_arguments(
+            namespace='ConfigService.by_key_and_section')
+        def cached(key, section):
+            query = db_session.query(Config).filter(Config.key == key)
+            query = query.filter(Config.section == section)
+            config = query.first()
+            if config:
+                db_session.expunge(config)
+            return config
+
+        config = cached(key, section)
+        if config:
+            config = db_session.merge(config, load=False)
+        if config is None and auto_create:
+            config = ConfigService.create_config(key, section,
+                                                 value=default_value)
+            cached.invalidate(key, section)
+        return config
+
+    @classmethod
+    def setup_default_values(self):
+        """
+        Will add fresh default config values to database if no keys are found
+        :return:
+        """
+        log.info('Checking/setting default values')
+        self.by_key_and_section('template_footer_html', 'global',
+                                default_value='', auto_create=True)
+        self.by_key_and_section('list_groups_to_non_admins', 'global',
+                                default_value=True, auto_create=True)
+        self.by_key_and_section('per_application_reports_rate_limit', 'global',
+                                default_value=2000, auto_create=True)
+        self.by_key_and_section('per_application_logs_rate_limit', 'global',
+                                default_value=100000, auto_create=True)
+        self.by_key_and_section('per_application_metrics_rate_limit', 'global',
+                                default_value=100000, auto_create=True)

backend/src/appenlight/models/services/event.py

@@ -0,0 +1,114 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from pyramid.threadlocal import get_current_registry
+from paginate_sqlalchemy import SqlalchemyOrmPage
+from appenlight.models import get_db_session
+from appenlight.models.event import Event
+from appenlight.models.services.base import BaseService
+
+
+class EventService(BaseService):
+    @classmethod
+    def for_resource(cls, resource_ids, event_type=None, status=None,
+                     since_when=None, limit=20, event_id=None,
+                     target_uuid=None, order_by=None, or_target_user_id=None,
+                     db_session=None):
+        """
+        Fetches events including based on passed params OR if target_user_id
+        is present include events that just target this user
+        """
+        db_session = get_db_session(db_session)
+        query = db_session.query(Event)
+        query = query.options(sa.orm.joinedload(Event.resource))
+        and_cond = [Event.resource_id.in_(resource_ids)]
+        if not resource_ids:
+            and_cond = [Event.resource_id == -999]
+
+        if event_type:
+            and_cond.append(Event.event_type == event_type)
+        if status:
+            and_cond.append(Event.status == status)
+        if since_when:
+            and_cond.append(Event.start_date >= since_when)
+        if event_id:
+            and_cond.append(Event.id == event_id)
+        if target_uuid:
+            and_cond.append(Event.target_uuid == target_uuid)
+
+        or_cond = []
+
+        if or_target_user_id:
+            or_cond.append(sa.or_(Event.target_user_id == or_target_user_id))
+
+        query = query.filter(sa.or_(sa.and_(*and_cond),
+                                    *or_cond))
+        if not order_by:
+            query = query.order_by(sa.desc(Event.start_date))
+        if limit:
+            query = query.limit(limit)
+
+        return query
+
+    @classmethod
+    def by_type_and_status(cls, event_types, status_types, since_when=None,
+                           older_than=None, db_session=None, app_ids=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(Event)
+        query = query.filter(Event.event_type.in_(event_types))
+        query = query.filter(Event.status.in_(status_types))
+        if since_when:
+            query = query.filter(Event.start_date >= since_when)
+        if older_than:
+            query = query.filter(Event.start_date <= older_than)
+        if app_ids:
+            query = query.filter(Event.resource_id.in_(app_ids))
+        return query
+
+    @classmethod
+    def latest_for_user(cls, user, db_session=None):
+        registry = get_current_registry()
+        resources = user.resources_with_perms(
+            ['view'], resource_types=registry.resource_types)
+        resource_ids = [r.resource_id for r in resources]
+        db_session = get_db_session(db_session)
+        return EventService.for_resource(
+            resource_ids, or_target_user_id=user.id, limit=10,
+            db_session=db_session)
+
+    @classmethod
+    def get_paginator(cls, user, page=1, item_count=None, items_per_page=50,
+                      order_by=None, filter_settings=None, db_session=None):
+        if not filter_settings:
+            filter_settings = {}
+        registry = get_current_registry()
+        resources = user.resources_with_perms(
+            ['view'], resource_types=registry.resource_types)
+        resource_ids = [r.resource_id for r in resources]
+        query = EventService.for_resource(
+            resource_ids, or_target_user_id=user.id, limit=100,
+            db_session=db_session)
+
+        paginator = SqlalchemyOrmPage(query, page=page,
+                                      items_per_page=items_per_page,
+                                      **filter_settings)
+        return paginator

backend/src/appenlight/models/services/group.py

@@ -0,0 +1,32 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from appenlight.models import get_db_session
+from appenlight.models.group import Group
+from appenlight.models.services.base import BaseService
+
+
+class GroupService(BaseService):
+    @classmethod
+    def by_id(cls, group_id, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(Group).filter(Group.id == group_id)
+        return query.first()

backend/src/appenlight/models/services/group_resource_permission.py

@@ -0,0 +1,38 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from appenlight.models.group_resource_permission import GroupResourcePermission
+from appenlight.models import get_db_session
+from appenlight.models.services.base import BaseService
+
+
+class GroupResourcePermissionService(BaseService):
+    @classmethod
+    def by_resource_group_and_perm(cls, group_id, perm_name, resource_id,
+                                   db_session=None):
+        """ return all instances by user name, perm name and resource id """
+        db_session = get_db_session(db_session)
+        query = db_session.query(GroupResourcePermission)
+        query = query.filter(GroupResourcePermission.group_id == group_id)
+        query = query.filter(
+            GroupResourcePermission.resource_id == resource_id)
+        query = query.filter(GroupResourcePermission.perm_name == perm_name)
+        return query.first()

backend/src/appenlight/models/services/log.py

@@ -0,0 +1,207 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import paginate
+import logging
+import sqlalchemy as sa
+
+from appenlight.models.log import Log
+from appenlight.models import get_db_session, Datastores
+from appenlight.models.services.base import BaseService
+from appenlight.lib.utils import es_index_name_limiter
+
+log = logging.getLogger(__name__)
+
+
+class LogService(BaseService):
+    @classmethod
+    def get_logs(cls, resource_ids=None, filter_settings=None,
+                 db_session=None):
+        # ensure we always have id's passed
+        if not resource_ids:
+            # raise Exception('No App ID passed')
+            return []
+        db_session = get_db_session(db_session)
+        q = db_session.query(Log)
+        q = q.filter(Log.resource_id.in_(resource_ids))
+        if filter_settings.get('start_date'):
+            q = q.filter(Log.timestamp >= filter_settings.get('start_date'))
+        if filter_settings.get('end_date'):
+            q = q.filter(Log.timestamp <= filter_settings.get('end_date'))
+        if filter_settings.get('log_level'):
+            q = q.filter(
+                Log.log_level == filter_settings.get('log_level').upper())
+        if filter_settings.get('request_id'):
+            request_id = filter_settings.get('request_id', '')
+            q = q.filter(Log.request_id == request_id.replace('-', ''))
+        if filter_settings.get('namespace'):
+            q = q.filter(Log.namespace == filter_settings.get('namespace'))
+        q = q.order_by(sa.desc(Log.timestamp))
+        return q
+
+    @classmethod
+    def es_query_builder(cls, app_ids, filter_settings):
+        if not filter_settings:
+            filter_settings = {}
+
+        query = {
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": [{"terms": {"resource_id": list(app_ids)}}]
+                    }
+                }
+            }
+        }
+
+        start_date = filter_settings.get('start_date')
+        end_date = filter_settings.get('end_date')
+        filter_part = query['query']['filtered']['filter']['and']
+
+        for tag in filter_settings.get('tags', []):
+            tag_values = [v.lower() for v in tag['value']]
+            key = "tags.%s.values" % tag['name'].replace('.', '_')
+            filter_part.append({"terms": {key: tag_values}})
+
+        date_range = {"range": {"timestamp": {}}}
+        if start_date:
+            date_range["range"]["timestamp"]["gte"] = start_date
+        if end_date:
+            date_range["range"]["timestamp"]["lte"] = end_date
+        if start_date or end_date:
+            filter_part.append(date_range)
+
+        levels = filter_settings.get('level')
+        if levels:
+            filter_part.append({"terms": {'log_level': levels}})
+        namespaces = filter_settings.get('namespace')
+        if namespaces:
+            filter_part.append({"terms": {'namespace': namespaces}})
+
+        request_ids = filter_settings.get('request_id')
+        if request_ids:
+            filter_part.append({"terms": {'request_id': request_ids}})
+
+        messages = filter_settings.get('message')
+        if messages:
+            query['query']['filtered']['query'] = {
+                'match': {"message": ' '.join(messages)}}
+        return query
+
+    @classmethod
+    def get_time_series_aggregate(cls, app_ids=None, filter_settings=None):
+        if not app_ids:
+            return {}
+        es_query = cls.es_query_builder(app_ids, filter_settings)
+        es_query["aggs"] = {
+            "events_over_time": {
+                "date_histogram": {
+                    "field": "timestamp",
+                    "interval": "1h",
+                    "min_doc_count": 0
+                }
+            }
+        }
+        log.debug(es_query)
+        index_names = es_index_name_limiter(filter_settings.get('start_date'),
+                                            filter_settings.get('end_date'),
+                                            ixtypes=['logs'])
+        if index_names:
+            results = Datastores.es.search(
+                es_query, index=index_names, doc_type='log', size=0)
+        else:
+            results = []
+        return results
+
+    @classmethod
+    def get_search_iterator(cls, app_ids=None, page=1, items_per_page=50,
+                            order_by=None, filter_settings=None, limit=None):
+        if not app_ids:
+            return {}, 0
+
+        es_query = cls.es_query_builder(app_ids, filter_settings)
+        sort_query = {
+            "sort": [
+                {"timestamp": {"order": "desc"}}
+            ]
+        }
+        es_query.update(sort_query)
+        log.debug(es_query)
+        es_from = (page - 1) * items_per_page
+        index_names = es_index_name_limiter(filter_settings.get('start_date'),
+                                            filter_settings.get('end_date'),
+                                            ixtypes=['logs'])
+        if not index_names:
+            return {}, 0
+
+        results = Datastores.es.search(es_query, index=index_names,
+                                       doc_type='log', size=items_per_page,
+                                       es_from=es_from)
+        if results['hits']['total'] > 5000:
+            count = 5000
+        else:
+            count = results['hits']['total']
+        return results['hits'], count
+
+    @classmethod
+    def get_paginator_by_app_ids(cls, app_ids=None, page=1, item_count=None,
+                                 items_per_page=50, order_by=None,
+                                 filter_settings=None,
+                                 exclude_columns=None, db_session=None):
+        if not filter_settings:
+            filter_settings = {}
+        results, item_count = cls.get_search_iterator(app_ids, page,
+                                                      items_per_page, order_by,
+                                                      filter_settings)
+        paginator = paginate.Page([],
+                                  item_count=item_count,
+                                  items_per_page=items_per_page,
+                                  **filter_settings)
+        ordered_ids = tuple(item['_source']['pg_id']
+                            for item in results.get('hits', []))
+
+        sorted_instance_list = []
+        if ordered_ids:
+            db_session = get_db_session(db_session)
+            query = db_session.query(Log)
+            query = query.filter(Log.log_id.in_(ordered_ids))
+            query = query.order_by(sa.desc('timestamp'))
+            sa_items = query.all()
+            # resort by score
+            for i_id in ordered_ids:
+                for item in sa_items:
+                    if str(item.log_id) == str(i_id):
+                        sorted_instance_list.append(item)
+        paginator.sa_items = sorted_instance_list
+        return paginator
+
+    @classmethod
+    def query_by_primary_key_and_namespace(cls, list_of_pairs,
+                                           db_session=None):
+        db_session = get_db_session(db_session)
+        list_of_conditions = []
+        query = db_session.query(Log)
+        for pair in list_of_pairs:
+            list_of_conditions.append(sa.and_(
+                Log.primary_key == pair['pk'], Log.namespace == pair['ns']))
+        query = query.filter(sa.or_(*list_of_conditions))
+        query = query.order_by(sa.asc(Log.timestamp), sa.asc(Log.log_id))
+        return query

backend/src/appenlight/models/services/plugin_config.py

@@ -0,0 +1,57 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+
+from appenlight.models.plugin_config import PluginConfig
+from appenlight.models.services.base import BaseService
+from appenlight.models import get_db_session
+
+log = logging.getLogger(__name__)
+
+
+class PluginConfigService(BaseService):
+    @classmethod
+    def all(cls, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(PluginConfig)
+        return query
+
+    @classmethod
+    def by_id(cls, plugin_id, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(PluginConfig)
+        query = query.filter(PluginConfig.id == plugin_id)
+        return query.first()
+
+    @classmethod
+    def by_query(cls, resource_id=None, plugin_name=None,
+                 section=None, db_session=None):
+        db_session = get_db_session(db_session)
+
+        query = db_session.query(PluginConfig)
+        if resource_id:
+            query = query.filter(PluginConfig.resource_id == resource_id)
+        if plugin_name:
+            query = query.filter(PluginConfig.plugin_name == plugin_name)
+        if section:
+            query = query.filter(PluginConfig.section == section)
+        return query

backend/src/appenlight/models/services/report.py

@@ -0,0 +1,62 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import sqlalchemy as sa
+
+from appenlight.models import get_db_session
+from appenlight.models.report import Report
+from appenlight.models.report_stat import ReportStat
+from appenlight.models.services.base import BaseService
+
+log = logging.getLogger(__name__)
+
+
+class ReportService(BaseService):
+    @classmethod
+    def by_app_ids(cls, app_ids=None, order_by=True, db_session=None):
+        db_session = get_db_session(db_session)
+        q = db_session.query(Report)
+        if app_ids:
+            q = q.filter(Report.resource_id.in_(app_ids))
+        if order_by:
+            q = q.order_by(sa.desc(Report.id))
+        return q
+
+    @classmethod
+    def generate_stat_rows(cls, report, resource, report_group, occurences=1,
+                           db_session=None):
+        """
+        Generates timeseries for this report's group
+        """
+        db_session = get_db_session(db_session)
+        stats = ReportStat(resource_id=report.resource_id,
+                           group_id=report_group.id,
+                           start_interval=report.start_time,
+                           owner_user_id=resource.owner_user_id,
+                           server_name=report.tags.get('server_name'),
+                           view_name=report.tags.get('view_name'),
+                           type=report.report_type,
+                           occurences=occurences,
+                           duration=report.duration)
+        db_session.add(stats)
+        db_session.flush()
+        return stats

backend/src/appenlight/models/services/report_assignment.py

@@ -0,0 +1,33 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from appenlight.models import get_db_session
+from appenlight.models.services.base import BaseService
+
+
+class ReportAssignmentService(BaseService):
+    @classmethod
+    def by_group_id_and_user(cls, group_id, user_name, db_session=None):
+        db_session = get_db_session(db_session)
+
+        query = db_session.query(cls).filter(cls.group_id == group_id)
+        query = query.filter(cls.user_name == user_name)
+        return query.first()

backend/src/appenlight/models/services/report_group.py

@@ -0,0 +1,454 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import paginate
+import sqlalchemy as sa
+import appenlight.lib.helpers as h
+
+from datetime import datetime
+
+from appenlight.models import get_db_session, Datastores
+from appenlight.models.report import Report
+from appenlight.models.report_group import ReportGroup
+from appenlight.models.report_comment import ReportComment
+from appenlight.models.user import User
+from appenlight.models.services.base import BaseService
+from appenlight.lib.enums import ReportType
+from appenlight.lib.utils import es_index_name_limiter
+
+log = logging.getLogger(__name__)
+
+
+class ReportGroupService(BaseService):
+    @classmethod
+    def get_trending(cls, request, filter_settings, limit=15,
+                     db_session=None):
+        """
+        Returns report groups trending for specific time interval
+        """
+        db_session = get_db_session(db_session)
+
+        tags = []
+        if filter_settings.get('tags'):
+            for tag in filter_settings['tags']:
+                tags.append(
+                    {'terms': {
+                        'tags.{}.values'.format(tag['name']): tag['value']}})
+
+        index_names = es_index_name_limiter(
+            start_date=filter_settings['start_date'],
+            end_date=filter_settings['end_date'],
+            ixtypes=['reports'])
+
+        if not index_names or not filter_settings['resource']:
+            return []
+
+        es_query = {
+            'aggs': {'parent_agg': {'aggs': {'groups': {'aggs': {
+                'sub_agg': {
+                    'value_count': {'field': 'tags.group_id.values'}}},
+                'filter': {'exists': {'field': 'tags.group_id.values'}}}},
+                'terms': {'field': 'tags.group_id.values', 'size': limit}}},
+            'query': {'filtered': {
+                'filter': {'and': [
+                    {'terms': {
+                        'resource_id': [filter_settings['resource'][0]]}
+                    },
+                    {'range': {'timestamp': {
+                        'gte': filter_settings['start_date'],
+                        'lte': filter_settings['end_date']}}}]
+                }
+            }}
+        }
+        if tags:
+            es_query['query']['filtered']['filter']['and'].extend(tags)
+
+        result = Datastores.es.search(
+            es_query, index=index_names, doc_type='log', size=0)
+        series = []
+        for bucket in result['aggregations']['parent_agg']['buckets']:
+            series.append({
+                'key': bucket['key'],
+                'groups': bucket['groups']['sub_agg']['value']
+            })
+
+        report_groups_d = {}
+        for g in series:
+            report_groups_d[int(g['key'])] = g['groups'] or 0
+
+        query = db_session.query(ReportGroup)
+        query = query.filter(ReportGroup.id.in_(list(report_groups_d.keys())))
+        query = query.options(
+            sa.orm.joinedload(ReportGroup.last_report_ref))
+        results = [(report_groups_d[group.id], group,) for group in query]
+        return sorted(results, reverse=True, key=lambda x:x[0])
+
+    @classmethod
+    def get_search_iterator(cls, app_ids=None, page=1, items_per_page=50,
+                            order_by=None, filter_settings=None, limit=None):
+        if not app_ids:
+            return {}
+        if not filter_settings:
+            filter_settings = {}
+
+        query = {
+            "size": 0,
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": [{"terms": {"resource_id": list(app_ids)}}]
+                    }
+                }
+            },
+
+            "aggs": {
+                "top_groups": {
+                    "terms": {
+                        "size": 5000,
+                        "field": "_parent",
+                        "order": {
+                            "newest": "desc"
+                        }
+                    },
+                    "aggs": {
+                        "top_reports_hits": {
+                            "top_hits": {"size": 1,
+                                         "sort": {"start_time": "desc"}
+                                         }
+                        },
+                        "newest": {
+                            "max": {"field": "start_time"}
+                        }
+                    }
+                }
+            }
+        }
+
+        start_date = filter_settings.get('start_date')
+        end_date = filter_settings.get('end_date')
+        filter_part = query['query']['filtered']['filter']['and']
+        date_range = {"range": {"start_time": {}}}
+        if start_date:
+            date_range["range"]["start_time"]["gte"] = start_date
+        if end_date:
+            date_range["range"]["start_time"]["lte"] = end_date
+        if start_date or end_date:
+            filter_part.append(date_range)
+
+        priorities = filter_settings.get('priority')
+
+        for tag in filter_settings.get('tags', []):
+            tag_values = [v.lower() for v in tag['value']]
+            key = "tags.%s.values" % tag['name'].replace('.', '_')
+            filter_part.append({"terms": {key: tag_values}})
+
+        if priorities:
+            filter_part.append({"has_parent": {
+                "parent_type": "report_group",
+                "query": {
+                    "terms": {'priority': priorities}
+                }}})
+
+        min_occurences = filter_settings.get('min_occurences')
+        if min_occurences:
+            filter_part.append({"has_parent": {
+                "parent_type": "report_group",
+                "query": {
+                    "range": {'occurences': {"gte": min_occurences[0]}}
+                }}})
+
+        min_duration = filter_settings.get('min_duration')
+        max_duration = filter_settings.get('max_duration')
+
+        request_ids = filter_settings.get('request_id')
+        if request_ids:
+            filter_part.append({"terms": {'request_id': request_ids}})
+
+        duration_range = {"range": {"average_duration": {}}}
+        if min_duration:
+            duration_range["range"]["average_duration"]["gte"] = \
+                min_duration[0]
+        if max_duration:
+            duration_range["range"]["average_duration"]["lte"] = \
+                max_duration[0]
+        if min_duration or max_duration:
+            filter_part.append({"has_parent": {
+                "parent_type": "report_group",
+                "query": duration_range}})
+
+        http_status = filter_settings.get('http_status')
+        report_type = filter_settings.get('report_type', [ReportType.error])
+        # set error report type if http status is not found
+        # and we are dealing with slow reports
+        if not http_status or ReportType.slow in report_type:
+            filter_part.append({"terms": {'report_type': report_type}})
+        if http_status:
+            filter_part.append({"terms": {'http_status': http_status}})
+
+        messages = filter_settings.get('message')
+        if messages:
+            condition = {'match': {"message": ' '.join(messages)}}
+            query['query']['filtered']['query'] = condition
+        errors = filter_settings.get('error')
+        if errors:
+            condition = {'match': {"error": ' '.join(errors)}}
+            query['query']['filtered']['query'] = condition
+        url_domains = filter_settings.get('url_domain')
+        if url_domains:
+            condition = {'terms': {"url_domain": url_domains}}
+            query['query']['filtered']['query'] = condition
+        url_paths = filter_settings.get('url_path')
+        if url_paths:
+            condition = {'terms': {"url_path": url_paths}}
+            query['query']['filtered']['query'] = condition
+
+        if filter_settings.get('report_status'):
+            for status in filter_settings.get('report_status'):
+                if status == 'never_reviewed':
+                    filter_part.append({"has_parent": {
+                        "parent_type": "report_group",
+                        "query": {
+                            "term": {"read": False}
+                        }}})
+                elif status == 'reviewed':
+                    filter_part.append({"has_parent": {
+                        "parent_type": "report_group",
+                        "query": {
+                            "term": {"read": True}
+                        }}})
+                elif status == 'public':
+                    filter_part.append({"has_parent": {
+                        "parent_type": "report_group",
+                        "query": {
+                            "term": {"public": True}
+                        }}})
+                elif status == 'fixed':
+                    filter_part.append({"has_parent": {
+                        "parent_type": "report_group",
+                        "query": {
+                            "term": {"fixed": True}
+                        }}})
+
+        # logging.getLogger('pyelasticsearch').setLevel(logging.DEBUG)
+        index_names = es_index_name_limiter(filter_settings.get('start_date'),
+                                            filter_settings.get('end_date'),
+                                            ixtypes=['reports'])
+        if index_names:
+            results = Datastores.es.search(
+                query, index=index_names, doc_type=["report", "report_group"],
+                size=0)
+        else:
+            return []
+        return results['aggregations']
+
+    @classmethod
+    def get_paginator_by_app_ids(cls, app_ids=None, page=1, item_count=None,
+                                 items_per_page=50, order_by=None,
+                                 filter_settings=None,
+                                 exclude_columns=None, db_session=None):
+        if not filter_settings:
+            filter_settings = {}
+        results = cls.get_search_iterator(app_ids, page, items_per_page,
+                                          order_by, filter_settings)
+
+        ordered_ids = []
+        if results:
+            for item in results['top_groups']['buckets']:
+                pg_id = item['top_reports_hits']['hits']['hits'][0]['_source'][
+                    'pg_id']
+                ordered_ids.append(pg_id)
+        log.info(filter_settings)
+        paginator = paginate.Page(ordered_ids, items_per_page=items_per_page,
+                                  **filter_settings)
+        sa_items = ()
+        if paginator.items:
+            db_session = get_db_session(db_session)
+            # latest report detail
+            query = db_session.query(Report)
+            query = query.options(sa.orm.joinedload(Report.report_group))
+            query = query.filter(Report.id.in_(paginator.items))
+            if filter_settings.get('order_col'):
+                order_col = filter_settings.get('order_col')
+                if filter_settings.get('order_dir') == 'dsc':
+                    sort_on = 'desc'
+                else:
+                    sort_on = 'asc'
+                if order_col == 'when':
+                    order_col = 'last_timestamp'
+                query = query.order_by(getattr(sa, sort_on)(
+                    getattr(ReportGroup, order_col)))
+            sa_items = query.all()
+        sorted_instance_list = []
+        for i_id in ordered_ids:
+            for report in sa_items:
+                if (str(report.id) == i_id and
+                            report not in sorted_instance_list):
+                    sorted_instance_list.append(report)
+        paginator.sa_items = sorted_instance_list
+        return paginator
+
+    @classmethod
+    def by_app_ids(cls, app_ids=None, order_by=True, db_session=None):
+        db_session = get_db_session(db_session)
+        q = db_session.query(ReportGroup)
+        if app_ids:
+            q = q.filter(ReportGroup.resource_id.in_(app_ids))
+        if order_by:
+            q = q.order_by(sa.desc(ReportGroup.id))
+        return q
+
+    @classmethod
+    def by_id(cls, group_id, app_ids=None, db_session=None):
+        db_session = get_db_session(db_session)
+        q = db_session.query(ReportGroup).filter(
+            ReportGroup.id == int(group_id))
+        if app_ids:
+            q = q.filter(ReportGroup.resource_id.in_(app_ids))
+        return q.first()
+
+    @classmethod
+    def by_ids(cls, group_ids=None, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(ReportGroup)
+        query = query.filter(ReportGroup.id.in_(group_ids))
+        return query
+
+    @classmethod
+    def by_hash_and_resource(self, resource_id,
+                             grouping_hash, db_session=None):
+        db_session = get_db_session(db_session)
+        q = db_session.query(ReportGroup)
+        q = q.filter(ReportGroup.resource_id == resource_id)
+        q = q.filter(ReportGroup.grouping_hash == grouping_hash)
+        q = q.filter(ReportGroup.fixed == False)
+        return q.first()
+
+    @classmethod
+    def users_commenting(cls, report_group, exclude_user_id=None,
+                         db_session=None):
+        db_session = get_db_session(None, report_group)
+        query = db_session.query(User).distinct()
+        query = query.filter(User.id == ReportComment.owner_id)
+        query = query.filter(ReportComment.group_id == report_group.id)
+        if exclude_user_id:
+            query = query.filter(ReportComment.owner_id != exclude_user_id)
+        return query
+
+    @classmethod
+    def affected_users_count(cls, report_group, db_session=None):
+        db_session = get_db_session(db_session)
+        query = db_session.query(sa.func.count(Report.username))
+        query = query.filter(Report.group_id == report_group.id)
+        query = query.filter(Report.username != '')
+        query = query.filter(Report.username != None)
+        query = query.group_by(Report.username)
+        return query.count()
+
+    @classmethod
+    def top_affected_users(cls, report_group, db_session=None):
+        db_session = get_db_session(db_session)
+        count_label = sa.func.count(Report.username).label('count')
+        query = db_session.query(Report.username, count_label)
+        query = query.filter(Report.group_id == report_group.id)
+        query = query.filter(Report.username != None)
+        query = query.filter(Report.username != '')
+        query = query.group_by(Report.username)
+        query = query.order_by(sa.desc(count_label))
+        query = query.limit(50)
+        return query
+
+    @classmethod
+    def get_report_stats(cls, request, filter_settings):
+        """
+        Gets report dashboard graphs
+        Returns information for BAR charts with occurences/interval information
+        detailed means version that returns time intervals - non detailed
+        returns total sum
+        """
+        delta = filter_settings['end_date'] - filter_settings['start_date']
+        if delta < h.time_deltas.get('12h')['delta']:
+            interval = '1m'
+        elif delta <= h.time_deltas.get('3d')['delta']:
+            interval = '5m'
+        elif delta >= h.time_deltas.get('2w')['delta']:
+            interval = '24h'
+        else:
+            interval = '1h'
+
+        group_id = filter_settings.get('group_id')
+
+        es_query = {
+            'aggs': {'parent_agg': {'aggs': {'types': {
+                'aggs': {'sub_agg': {'terms': {'field': 'tags.type.values'}}},
+                'filter': {
+                    'and': [{'exists': {'field': 'tags.type.values'}}]}
+            }},
+                'date_histogram': {'extended_bounds': {
+                    'max': filter_settings['end_date'],
+                    'min': filter_settings['start_date']},
+                    'field': 'timestamp',
+                    'interval': interval,
+                    'min_doc_count': 0}}},
+            'query': {'filtered': {
+                'filter': {'and': [
+                    {'terms': {
+                        'resource_id': [filter_settings['resource'][0]]}},
+                    {'range': {'timestamp': {
+                        'gte': filter_settings['start_date'],
+                        'lte': filter_settings['end_date']}}}]
+                }
+            }}
+        }
+        if group_id:
+            parent_agg = es_query['aggs']['parent_agg']
+            filters = parent_agg['aggs']['types']['filter']['and']
+            filters.append({'terms': {'tags.group_id.values': [group_id]}})
+
+        index_names = es_index_name_limiter(
+            start_date=filter_settings['start_date'],
+            end_date=filter_settings['end_date'],
+            ixtypes=['reports'])
+
+        if not index_names:
+            return []
+
+        result = Datastores.es.search(es_query,
+                                      index=index_names,
+                                      doc_type='log',
+                                      size=0)
+        series = []
+        for bucket in result['aggregations']['parent_agg']['buckets']:
+            point = {
+                'x': datetime.utcfromtimestamp(int(bucket['key']) / 1000),
+                'report': 0,
+                'not_found': 0,
+                'slow_report': 0
+            }
+            for subbucket in bucket['types']['sub_agg']['buckets']:
+                if subbucket['key'] == 'slow':
+                    point['slow_report'] = subbucket['doc_count']
+                elif subbucket['key'] == 'error':
+                    point['report'] = subbucket['doc_count']
+                elif subbucket['key'] == 'not_found':
+                    point['not_found'] = subbucket['doc_count']
+            series.append(point)
+        return series

backend/src/appenlight/models/services/report_stat.py

@@ -0,0 +1,55 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from appenlight.models import Datastores
+from appenlight.models.services.base import BaseService
+from appenlight.lib.enums import ReportType
+from appenlight.lib.utils import es_index_name_limiter
+
+
+class ReportStatService(BaseService):
+    @classmethod
+    def count_by_type(cls, report_type, resource_id, since_when):
+        report_type = ReportType.key_from_value(report_type)
+
+        index_names = es_index_name_limiter(start_date=since_when,
+                              ixtypes=['reports'])
+
+        es_query = {
+            'aggs': {'reports': {'aggs': {
+                'sub_agg': {'value_count': {'field': 'tags.group_id.values'}}},
+                'filter': {'and': [{'terms': {'resource_id': [resource_id]}},
+                                   {'exists': {
+                                       'field': 'tags.group_id.values'}}]}}},
+            'query': {'filtered': {'filter': {
+                'and': [{'terms': {'resource_id': [resource_id]}},
+                        {'terms': {'tags.type.values': [report_type]}},
+                        {'range': {'timestamp': {
+                            'gte': since_when}}}]}}}}
+
+        if index_names:
+            result = Datastores.es.search(es_query,
+                                      index=index_names,
+                                      doc_type='log',
+                                      size=0)
+            return result['aggregations']['reports']['sub_agg']['value']
+        else:
+            return 0

backend/src/appenlight/models/services/request_metric.py

@@ -0,0 +1,448 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from datetime import datetime
+
+import appenlight.lib.helpers as h
+from appenlight.models import get_db_session, Datastores
+from appenlight.models.services.base import BaseService
+from appenlight.lib.enums import ReportType
+from appenlight.lib.utils import es_index_name_limiter
+
+try:
+    from ae_uptime_ce.models.services.uptime_metric import \
+        UptimeMetricService
+except ImportError:
+    UptimeMetricService = None
+
+
+def check_key(key, stats, uptime, total_seconds):
+    if key not in stats:
+        stats[key] = {'name': key,
+                      'requests': 0,
+                      'errors': 0,
+                      'tolerated_requests': 0,
+                      'frustrating_requests': 0,
+                      'satisfying_requests': 0,
+                      'total_minutes': total_seconds / 60.0,
+                      'uptime': uptime,
+                      'apdex': 0,
+                      'rpm': 0,
+                      'response_time': 0,
+                      'avg_response_time': 0}
+
+
+class RequestMetricService(BaseService):
+    @classmethod
+    def get_metrics_stats(cls, request, filter_settings, db_session=None):
+        delta = filter_settings['end_date'] - filter_settings['start_date']
+        if delta < h.time_deltas.get('12h')['delta']:
+            interval = '1m'
+        elif delta <= h.time_deltas.get('3d')['delta']:
+            interval = '5m'
+        elif delta >= h.time_deltas.get('2w')['delta']:
+            interval = '24h'
+        else:
+            interval = '1h'
+
+        filter_settings['namespace'] = ['appenlight.request_metric']
+
+        es_query = {
+            'aggs': {
+                'parent_agg': {
+                    'aggs': {'custom': {'aggs': {'sub_agg': {
+                        'sum': {'field': 'tags.custom.numeric_values'}}},
+                        'filter': {'exists': {
+                            'field': 'tags.custom.numeric_values'}}},
+                        'main': {'aggs': {'sub_agg': {'sum': {
+                            'field': 'tags.main.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.main.numeric_values'}}},
+                        'nosql': {'aggs': {'sub_agg': {'sum': {
+                            'field': 'tags.nosql.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.nosql.numeric_values'}}},
+                        'remote': {'aggs': {'sub_agg': {'sum': {
+                            'field': 'tags.remote.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.remote.numeric_values'}}},
+                        'requests': {'aggs': {'sub_agg': {'sum': {
+                            'field': 'tags.requests.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.requests.numeric_values'}}},
+                        'sql': {'aggs': {'sub_agg': {
+                            'sum': {'field': 'tags.sql.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.sql.numeric_values'}}},
+                        'tmpl': {'aggs': {'sub_agg': {'sum': {
+                            'field': 'tags.tmpl.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.tmpl.numeric_values'}}}},
+                    'date_histogram': {'extended_bounds': {
+                        'max': filter_settings['end_date'],
+                        'min': filter_settings['start_date']},
+                        'field': 'timestamp',
+                        'interval': interval,
+                        'min_doc_count': 0}}},
+            'query': {'filtered': {
+                'filter': {'and': [{'terms': {
+                    'resource_id': [filter_settings['resource'][0]]}},
+                    {'range': {'timestamp': {
+                        'gte': filter_settings['start_date'],
+                        'lte': filter_settings['end_date']}}},
+                    {'terms': {'namespace': [
+                        'appenlight.request_metric']}}]}}}}
+
+        index_names = es_index_name_limiter(
+            start_date=filter_settings['start_date'],
+            end_date=filter_settings['end_date'],
+            ixtypes=['metrics'])
+        if not index_names:
+            return []
+
+        result = Datastores.es.search(es_query,
+                                      index=index_names,
+                                      doc_type='log',
+                                      size=0)
+
+        plot_data = []
+        for item in result['aggregations']['parent_agg']['buckets']:
+            x_time = datetime.utcfromtimestamp(int(item['key']) / 1000)
+            point = {"x": x_time}
+            for key in ['custom', 'main', 'nosql', 'remote',
+                        'requests', 'sql', 'tmpl']:
+                value = item[key]['sub_agg']['value']
+                point[key] = round(value, 3) if value else 0
+            plot_data.append(point)
+
+        return plot_data
+
+    @classmethod
+    def get_requests_breakdown(cls, request, filter_settings,
+                               db_session=None):
+        db_session = get_db_session(db_session)
+
+        # fetch total time of all requests in this time range
+        index_names = es_index_name_limiter(
+            start_date=filter_settings['start_date'],
+            end_date=filter_settings['end_date'],
+            ixtypes=['metrics'])
+
+        if index_names and filter_settings['resource']:
+            es_query = {
+                'aggs': {'main': {'aggs': {
+                    'sub_agg': {'sum': {'field': 'tags.main.numeric_values'}}},
+                    'filter': {'exists': {
+                        'field': 'tags.main.numeric_values'}}}},
+                'query': {'filtered': {
+                    'filter': {'and': [
+                        {'terms': {
+                            'resource_id': [filter_settings['resource'][0]]}},
+                        {'range': {'timestamp': {
+                            'gte': filter_settings['start_date'],
+                            'lte': filter_settings['end_date']}}},
+                        {'terms': {'namespace': [
+                            'appenlight.request_metric']}}]}}}}
+            result = Datastores.es.search(es_query,
+                                          index=index_names,
+                                          doc_type='log',
+                                          size=0)
+            total_time_spent = result['aggregations']['main']['sub_agg'][
+                'value']
+        else:
+            total_time_spent = 0
+        script_text = "doc['tags.main.numeric_values'].value / {}".format(
+            total_time_spent)
+
+        if index_names and filter_settings['resource']:
+            es_query = {
+                'aggs': {
+                    'parent_agg': {
+                        'aggs': {'main': {'aggs': {
+                            'sub_agg': {
+                                'sum': {'field': 'tags.main.numeric_values'}}},
+                            'filter': {
+                                'exists': {
+                                    'field': 'tags.main.numeric_values'}}},
+                            'percentage': {
+                                'aggs': {'sub_agg': {
+                                    'sum': {
+                                        'lang': 'expression',
+                                        'script': script_text}}},
+                                'filter': {
+                                    'exists': {
+                                        'field': 'tags.main.numeric_values'}}},
+                            'requests': {'aggs': {'sub_agg': {
+                                'sum': {
+                                    'field': 'tags.requests.numeric_values'}}},
+                                'filter': {'exists': {
+                                    'field': 'tags.requests.numeric_values'}}}},
+                        'terms': {'field': 'tags.view_name.values',
+                                  'order': {'percentage>sub_agg': 'desc'},
+                                  'size': 15}}},
+                'query': {'filtered': {'filter': {'and': [
+                    {'terms': {
+                        'resource_id': [filter_settings['resource'][0]]}},
+                    {'range': {
+                        'timestamp': {'gte': filter_settings['start_date'],
+                                      'lte': filter_settings['end_date']
+                                      }
+                    }
+                    }
+                ]}
+                }}
+            }
+            result = Datastores.es.search(es_query,
+                                          index=index_names,
+                                          doc_type='log',
+                                          size=0)
+            series = result['aggregations']['parent_agg']['buckets']
+        else:
+            series = []
+
+        and_part = [
+            {"term": {"resource_id": filter_settings['resource'][0]}},
+            {"terms": {"tags.view_name.values": [row['key'] for
+                                                 row in series]}},
+            {"term": {"report_type": str(ReportType.slow)}}
+        ]
+        query = {
+            "aggs": {
+                "top_reports": {
+                    "terms": {
+                        "field": "tags.view_name.values",
+                        "size": len(series)
+                    },
+                    "aggs": {
+                        "top_calls_hits": {
+                            "top_hits": {
+                                "sort": {"start_time": "desc"},
+                                "size": 5
+                            }
+                        }
+                    }
+                }
+            },
+
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": and_part
+                    }
+                }
+            }
+        }
+        details = {}
+        index_names = es_index_name_limiter(ixtypes=['reports'])
+        if index_names and series:
+            result = Datastores.es.search(
+                query, doc_type='report', size=0, index=index_names)
+            for bucket in result['aggregations']['top_reports']['buckets']:
+                details[bucket['key']] = []
+
+                for hit in bucket['top_calls_hits']['hits']['hits']:
+                    details[bucket['key']].append(
+                        {'report_id': hit['_source']['pg_id'],
+                         'group_id': hit['_source']['group_id']}
+                    )
+
+        results = []
+        for row in series:
+            result = {
+                'key': row['key'],
+                'main': row['main']['sub_agg']['value'],
+                'requests': row['requests']['sub_agg']['value']
+            }
+            # es can return 'infinity'
+            try:
+                result['percentage'] = float(
+                    row['percentage']['sub_agg']['value'])
+            except ValueError:
+                result['percentage'] = 0
+
+            result['latest_details'] = details.get(row['key']) or []
+            results.append(result)
+
+        return results
+
+    @classmethod
+    def get_apdex_stats(cls, request, filter_settings,
+                        threshold=1, db_session=None):
+        """
+        Returns information and calculates APDEX score per server for dashboard
+        server information (upper right stats boxes)
+        """
+        # Apdex t = (Satisfied Count + Tolerated Count / 2) / Total Samples
+        db_session = get_db_session(db_session)
+        index_names = es_index_name_limiter(
+            start_date=filter_settings['start_date'],
+            end_date=filter_settings['end_date'], ixtypes=['metrics'])
+
+        requests_series = []
+
+        if index_names and filter_settings['resource']:
+            es_query = {
+                'aggs': {
+                    'parent_agg': {'aggs': {
+                        'frustrating': {'aggs': {'sub_agg': {
+                            'sum': {'field': 'tags.requests.numeric_values'}}},
+                            'filter': {'and': [
+                                {'range': {
+                                    'tags.main.numeric_values': {'gte': '4'}}},
+                                {'exists': {
+                                    'field': 'tags.requests.numeric_values'}}]
+                            }
+                        },
+                        'main': {'aggs': {'sub_agg': {'sum': {
+                            'field': 'tags.main.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.main.numeric_values'}}},
+                        'requests': {'aggs': {'sub_agg': {
+                            'sum': {
+                                'field': 'tags.requests.numeric_values'}}},
+                            'filter': {'exists': {
+                                'field': 'tags.requests.numeric_values'}}},
+                        'tolerated': {'aggs': {'sub_agg': {
+                            'sum': {
+                                'field': 'tags.requests.numeric_values'}}},
+                            'filter': {'and': [
+                                {'range': {
+                                    'tags.main.numeric_values': {'gte': '1'}}},
+                                {'range': {
+                                    'tags.main.numeric_values': {'lt': '4'}}},
+                                {'exists': {
+                                    'field': 'tags.requests.numeric_values'}}]}
+                        }
+                    },
+                        'terms': {'field': 'tags.server_name.values',
+                                  'size': 999999}}},
+                'query': {
+                    'filtered': {
+                        'filter': {'and': [{'terms': {
+                            'resource_id': [
+                                filter_settings['resource'][0]]}},
+                            {'range': {'timestamp': {
+                                'gte': filter_settings['start_date'],
+                                'lte': filter_settings['end_date']}}},
+                            {'terms': {'namespace': [
+                                'appenlight.request_metric']}}]}}}}
+
+            result = Datastores.es.search(es_query,
+                                          index=index_names,
+                                          doc_type='log',
+                                          size=0)
+            for bucket in result['aggregations']['parent_agg']['buckets']:
+                requests_series.append({
+                    'frustrating': bucket['frustrating']['sub_agg']['value'],
+                    'main': bucket['main']['sub_agg']['value'],
+                    'requests': bucket['requests']['sub_agg']['value'],
+                    'tolerated': bucket['tolerated']['sub_agg']['value'],
+                    'key': bucket['key']
+                })
+
+        since_when = filter_settings['start_date']
+        until = filter_settings['end_date']
+
+        # total errors
+
+        index_names = es_index_name_limiter(
+            start_date=filter_settings['start_date'],
+            end_date=filter_settings['end_date'], ixtypes=['reports'])
+
+        report_series = []
+        if index_names and filter_settings['resource']:
+            report_type = ReportType.key_from_value(ReportType.error)
+            es_query = {
+                'aggs': {
+                    'parent_agg': {'aggs': {'errors': {'aggs': {'sub_agg': {
+                        'sum': {
+                            'field': 'tags.occurences.numeric_values'}}},
+                        'filter': {'and': [
+                            {'terms': {
+                                'tags.type.values': [report_type]}},
+                            {'exists': {
+                                'field': 'tags.occurences.numeric_values'}}]
+                        }
+                    }},
+                        'terms': {'field': 'tags.server_name.values',
+                                  'size': 999999}}},
+                'query': {'filtered': {
+                    'filter': {'and': [
+                        {'terms': {
+                            'resource_id': [filter_settings['resource'][0]]}},
+                        {'range': {
+                            'timestamp': {'gte': filter_settings['start_date'],
+                                          'lte': filter_settings['end_date']}}
+                        },
+                        {'terms': {'namespace': ['appenlight.error']}}]
+                    }
+                }}
+            }
+            result = Datastores.es.search(es_query,
+                                          index=index_names,
+                                          doc_type='log',
+                                          size=0)
+            for bucket in result['aggregations']['parent_agg']['buckets']:
+                report_series.append(
+                    {'key': bucket['key'],
+                     'errors': bucket['errors']['sub_agg']['value']
+                    }
+                )
+
+        stats = {}
+        if UptimeMetricService is not None:
+            uptime = UptimeMetricService.get_uptime_by_app(
+                filter_settings['resource'][0],
+                since_when=since_when, until=until)
+        else:
+            uptime = 0
+
+        total_seconds = (until - since_when).total_seconds()
+
+        for stat in requests_series:
+            check_key(stat['key'], stats, uptime, total_seconds)
+            stats[stat['key']]['requests'] = int(stat['requests'])
+            stats[stat['key']]['response_time'] = stat['main']
+            stats[stat['key']]['tolerated_requests'] = stat['tolerated']
+            stats[stat['key']]['frustrating_requests'] = stat['frustrating']
+        for server in report_series:
+            check_key(server['key'], stats, uptime, total_seconds)
+            stats[server['key']]['errors'] = server['errors']
+
+        server_stats = list(stats.values())
+        for stat in server_stats:
+            stat['satisfying_requests'] = stat['requests'] - stat['errors'] \
+                                          - stat['frustrating_requests'] - \
+                                          stat['tolerated_requests']
+            if stat['satisfying_requests'] < 0:
+                stat['satisfying_requests'] = 0
+
+            if stat['requests']:
+                stat['avg_response_time'] = round(stat['response_time'] /
+                                                  stat['requests'], 3)
+                qual_requests = stat['satisfying_requests'] + \
+                                stat['tolerated_requests'] / 2.0
+                stat['apdex'] = round((qual_requests / stat['requests']) * 100,
+                                      2)
+                stat['rpm'] = round(stat['requests'] / stat['total_minutes'],
+                                    2)
+
+        return sorted(server_stats, key=lambda x: x['name'])

backend/src/appenlight/models/services/slow_call.py

@@ -0,0 +1,174 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from appenlight.models import get_db_session, Datastores
+from appenlight.models.report import Report
+from appenlight.models.services.base import BaseService
+from appenlight.lib.utils import es_index_name_limiter
+
+
+class SlowCallService(BaseService):
+    @classmethod
+    def get_time_consuming_calls(cls, request, filter_settings,
+                                 db_session=None):
+        db_session = get_db_session(db_session)
+        # get slow calls from older partitions too
+        index_names = es_index_name_limiter(
+            start_date=filter_settings['start_date'],
+            end_date=filter_settings['end_date'],
+            ixtypes=['slow_calls'])
+        if index_names and filter_settings['resource']:
+            # get longest time taking hashes
+            es_query = {
+                'aggs': {
+                    'parent_agg': {
+                        'aggs': {
+                            'duration': {
+                                'aggs': {'sub_agg': {
+                                    'sum': {
+                                        'field': 'tags.duration.numeric_values'}
+                                }},
+                                'filter': {'exists': {
+                                    'field': 'tags.duration.numeric_values'}}},
+                            'total': {
+                                'aggs': {'sub_agg': {'value_count': {
+                                    'field': 'tags.statement_hash.values'}}},
+                                'filter': {'exists': {
+                                    'field': 'tags.statement_hash.values'}}}},
+                        'terms': {'field': 'tags.statement_hash.values',
+                                  'order': {'duration>sub_agg': 'desc'},
+                                  'size': 15}}},
+                'query': {'filtered': {
+                    'filter': {'and': [
+                        {'terms': {
+                            'resource_id': [filter_settings['resource'][0]]
+                        }},
+                        {'range': {'timestamp': {
+                            'gte': filter_settings['start_date'],
+                            'lte': filter_settings['end_date']}
+                        }}]
+                    }
+                }
+                }
+            }
+            result = Datastores.es.search(
+                es_query, index=index_names, doc_type='log', size=0)
+            results = result['aggregations']['parent_agg']['buckets']
+        else:
+            return []
+        hashes = [i['key'] for i in results]
+
+        # get queries associated with hashes
+        calls_query = {
+            "aggs": {
+                "top_calls": {
+                    "terms": {
+                        "field": "tags.statement_hash.values",
+                        "size": 15
+                    },
+                    "aggs": {
+                        "top_calls_hits": {
+                            "top_hits": {
+                                "sort": {"timestamp": "desc"},
+                                "size": 5
+                            }
+                        }
+                    }
+                }
+            },
+            "query": {
+                "filtered": {
+                    "filter": {
+                        "and": [
+                            {
+                                "terms": {
+                                    "resource_id": [
+                                        filter_settings['resource'][0]
+                                    ]
+                                }
+                            },
+                            {
+                                "terms": {
+                                    "tags.statement_hash.values": hashes
+                                }
+                            },
+                            {
+                                "range": {
+                                    "timestamp": {
+                                        "gte": filter_settings['start_date'],
+                                        "lte": filter_settings['end_date']
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                }
+            }
+        }
+        calls = Datastores.es.search(calls_query,
+                                     index=index_names,
+                                     doc_type='log',
+                                     size=0)
+        call_results = {}
+        report_ids = []
+        for call in calls['aggregations']['top_calls']['buckets']:
+            hits = call['top_calls_hits']['hits']['hits']
+            call_results[call['key']] = [i['_source'] for i in hits]
+            report_ids.extend([i['_source']['tags']['report_id']['values']
+                               for i in hits])
+        if report_ids:
+            r_query = db_session.query(Report.group_id, Report.id)
+            r_query = r_query.filter(Report.id.in_(report_ids))
+            r_query = r_query.filter(
+                Report.start_time >= filter_settings['start_date'])
+        else:
+            r_query = []
+        reports_reversed = {}
+        for report in r_query:
+            reports_reversed[report.id] = report.group_id
+
+        final_results = []
+        for item in results:
+            if item['key'] not in call_results:
+                continue
+            call = call_results[item['key']][0]
+            row = {'occurences': item['total']['sub_agg']['value'],
+                   'total_duration': round(
+                       item['duration']['sub_agg']['value']),
+                   'statement': call['message'],
+                   'statement_type': call['tags']['type']['values'],
+                   'statement_subtype': call['tags']['subtype']['values'],
+                   'statement_hash': item['key'],
+                   'latest_details': []}
+            if row['statement_type'] in ['tmpl', ' remote']:
+                params = call['tags']['parameters']['values'] \
+                    if 'parameters' in call['tags'] else ''
+                row['statement'] = '{} ({})'.format(call['message'], params)
+            for call in call_results[item['key']]:
+                report_id = call['tags']['report_id']['values']
+                group_id = reports_reversed.get(report_id)
+                if group_id:
+                    row['latest_details'].append(
+                        {'group_id': group_id, 'report_id': report_id})
+
+            final_results.append(row)
+
+        return final_results

backend/src/appenlight/models/services/tag.py

@@ -0,0 +1,102 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from appenlight.lib.ext_json import json
+from pyramid.threadlocal import get_current_registry
+from appenlight.models.tag import Tag
+from appenlight.models.services.base import BaseService
+from appenlight.models import get_db_session
+
+
+class TagService(BaseService):
+    @classmethod
+    def cut_name(cls, tag_name):
+        return tag_name[:32]
+
+    @classmethod
+    def cut_value(cls, value):
+        if isinstance(value, str):
+            value = value[:128]
+        return value
+
+    @classmethod
+    def by_resource_id_and_value(cls, resource_id, tag_name, value,
+                                 db_session=None, create_missing=True):
+        """
+        Fetches tag and creates a new one if missing
+        """
+        db_session = get_db_session(db_session)
+        registry = get_current_registry()
+
+        @registry.cache_regions.redis_min_10.cache_on_arguments(
+            namespace='TagService.by_resource_id_and_value')
+        def cached(resource_id, tag_name, value):
+            reduced_name = cls.cut_name(tag_name.decode('utf8'))
+            reduced_value = cls.cut_value(value.decode('utf8'))
+
+            query = db_session.query(Tag)
+            query = query.filter(Tag.resource_id == resource_id)
+            query = query.filter(Tag.name == reduced_name)
+            query = query.filter(sa.cast(Tag.value, sa.types.TEXT) ==
+                                 sa.cast(json.dumps(reduced_value),
+                                         sa.types.TEXT))
+            tag = query.first()
+            if tag:
+                db_session.expunge(tag)
+            return tag
+
+        view = cached(resource_id, tag_name.encode('utf8'),
+                      value.encode('utf8'))
+        if not view and create_missing:
+            view = cls.create_tag(resource_id,
+                                  cls.cut_name(tag_name),
+                                  cls.cut_value(value),
+                                  db_session)
+            cached.invalidate(resource_id, tag_name.encode('utf8'),
+                              value.encode('utf8'))
+        return view
+
+    @classmethod
+    def create_tag(cls, resource_id, tag_name, value, db_session=None):
+
+        tag = Tag(resource_id=resource_id,
+                  name=cls.cut_name(tag_name),
+                  value=cls.cut_value(value))
+        db_session = get_db_session(db_session)
+        db_session.add(tag)
+        db_session.flush()
+        return tag
+
+    @classmethod
+    def by_tag_id(cls, tag_id, db_session=None):
+        db_session = get_db_session(db_session)
+        registry = get_current_registry()
+
+        @registry.cache_regions.redis_min_10.cache_on_arguments(
+            namespace='TagService.by_tag_id')
+        def cached(tag_id):
+            tag = db_session.query(Tag).filter(Tag.id == tag_id).first()
+            if tag:
+                db_session.expunge(tag)
+            return tag
+
+        return cached(tag_id)

backend/src/appenlight/models/services/user.py

@@ -0,0 +1,158 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import pyramid_mailer
+import pyramid.renderers
+import sqlalchemy as sa
+
+from collections import namedtuple
+from datetime import datetime
+
+from appenlight.lib.rule import Rule
+from appenlight.models import get_db_session
+from appenlight.models.integrations import IntegrationException
+from appenlight.models.report import REPORT_TYPE_MATRIX
+from appenlight.models.user import User
+from appenlight.models.services.base import BaseService
+from paginate_sqlalchemy import SqlalchemyOrmPage
+from pyramid.threadlocal import get_current_registry
+
+log = logging.getLogger(__name__)
+
+GroupOccurence = namedtuple('GroupOccurence', ['occurences', 'group'])
+
+
+class UserService(BaseService):
+    @classmethod
+    def all(cls, db_session=None):
+        return get_db_session(db_session).query(User).order_by(User.user_name)
+
+    @classmethod
+    def send_email(cls, request, recipients, variables, template,
+                   immediately=False, silent=False):
+        html = pyramid.renderers.render(template, variables, request)
+        title = variables.get('email_title',
+                              variables.get('title', "No Title"))
+        title = title.replace('\r', '').replace('\n', '')
+        sender = "{} <{}>".format(
+            request.registry.settings['mailing.from_name'],
+            request.registry.settings['mailing.from_email'])
+        message = pyramid_mailer.message.Message(
+            subject=title, sender=sender, recipients=recipients, html=html)
+        if immediately:
+            try:
+                request.registry.mailer.send_immediately(message)
+            except Exception as e:
+                log.warning('Exception %s' % e)
+                if not silent:
+                    raise
+        else:
+            request.registry.mailer.send(message)
+
+    @classmethod
+    def get_paginator(cls, page=1, item_count=None, items_per_page=50,
+                      order_by=None, filter_settings=None,
+                      exclude_columns=None, db_session=None):
+        registry = get_current_registry()
+        if not exclude_columns:
+            exclude_columns = []
+        if not filter_settings:
+            filter_settings = {}
+        db_session = get_db_session(db_session)
+        q = db_session.query(User)
+        if filter_settings.get('order_col'):
+            order_col = filter_settings.get('order_col')
+            if filter_settings.get('order_dir') == 'dsc':
+                sort_on = 'desc'
+            else:
+                sort_on = 'asc'
+            q = q.order_by(getattr(sa, sort_on)(getattr(User, order_col)))
+        else:
+            q = q.order_by(sa.desc(User.registered_date))
+            # remove urlgen or it never caches count
+        cache_params = dict(filter_settings)
+        cache_params.pop('url', None)
+        cache_params.pop('url_maker', None)
+
+        @registry.cache_regions.redis_min_5.cache_on_arguments()
+        def estimate_users(cache_key):
+            o_q = q.order_by(False)
+            return o_q.count()
+
+        item_count = estimate_users(cache_params)
+        # if the number of pages is low we may want to invalidate the count to
+        # provide 'real time' update - use case -
+        # errors just started to flow in
+        if item_count < 1000:
+            item_count = estimate_users.refresh(cache_params)
+        paginator = SqlalchemyOrmPage(q, page=page,
+                                      item_count=item_count,
+                                      items_per_page=items_per_page,
+                                      **filter_settings)
+        return paginator
+
+    @classmethod
+    def get_valid_channels(cls, user):
+        return [channel for channel in user.alert_channels
+                if channel.channel_validated]
+
+    @classmethod
+    def report_notify(cls, user, request, application, report_groups,
+                      occurence_dict, since_when=None, db_session=None):
+        db_session = get_db_session(db_session)
+        if not report_groups:
+            return True
+
+        if not since_when:
+            since_when = datetime.utcnow()
+        for channel in cls.get_valid_channels(user):
+            confirmed_groups = []
+
+            for group in report_groups:
+                occurences = occurence_dict.get(group.id, 1)
+                for action in channel.channel_actions:
+                    not_matched = (
+                        action.resource_id and action.resource_id !=
+                        application.resource_id)
+                    if action.type != 'report' or not_matched:
+                        continue
+                    should_notify = (action.action == 'always' or
+                                     not group.notified)
+                    rule_obj = Rule(action.rule, REPORT_TYPE_MATRIX)
+                    report_dict = group.get_report().get_dict(request)
+                    if rule_obj.match(report_dict) and should_notify:
+                        item = GroupOccurence(occurences, group)
+                        if item not in confirmed_groups:
+                            confirmed_groups.append(item)
+
+            # send individual reports
+            total_confirmed = len(confirmed_groups)
+            if not total_confirmed:
+                continue
+            try:
+                channel.notify_reports(resource=application,
+                                       user=user,
+                                       request=request,
+                                       since_when=since_when,
+                                       reports=confirmed_groups)
+            except IntegrationException as e:
+                log.warning('%s' % e)

backend/src/appenlight/models/slow_call.py

@@ -0,0 +1,120 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+import hashlib
+
+from datetime import datetime, timedelta
+from appenlight.models import Base
+from sqlalchemy.dialects.postgresql import JSON
+from ziggurat_foundations.models.base import BaseModel
+
+
+class SlowCall(Base, BaseModel):
+    __tablename__ = 'slow_calls'
+    __table_args__ = {'implicit_returning': False}
+
+    resource_id = sa.Column(sa.Integer(), nullable=False, index=True)
+    id = sa.Column(sa.Integer, nullable=False, primary_key=True)
+    report_id = sa.Column(sa.BigInteger,
+                          sa.ForeignKey('reports.id',
+                                        ondelete='cascade',
+                                        onupdate='cascade'),
+                          primary_key=True)
+    duration = sa.Column(sa.Float(), default=0)
+    statement = sa.Column(sa.UnicodeText(), default='')
+    statement_hash = sa.Column(sa.Unicode(60), default='')
+    parameters = sa.Column(JSON(), nullable=False, default=dict)
+    type = sa.Column(sa.Unicode(16), default='')
+    subtype = sa.Column(sa.Unicode(16), default=None)
+    location = sa.Column(sa.Unicode(255), default='')
+    timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                          server_default=sa.func.now())
+    report_group_time = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                                  server_default=sa.func.now())
+
+    def set_data(self, data, protocol_version=None, resource_id=None,
+                 report_group=None):
+        self.resource_id = resource_id
+        if data.get('start') and data.get('end'):
+            self.timestamp = data.get('start')
+            d = data.get('end') - data.get('start')
+            self.duration = d.total_seconds()
+        self.statement = data.get('statement', '')
+        self.type = data.get('type', 'unknown')[:16]
+        self.parameters = data.get('parameters', {})
+        self.location = data.get('location', '')[:255]
+        self.report_group_time = report_group.first_timestamp
+        if 'subtype' in data:
+            self.subtype = data.get('subtype', 'unknown')[:16]
+        if self.type == 'tmpl':
+            self.set_hash('{} {}'.format(self.statement, self.parameters))
+        else:
+            self.set_hash()
+
+    def set_hash(self, custom_statement=None):
+        statement = custom_statement or self.statement
+        self.statement_hash = hashlib.sha1(
+            statement.encode('utf8')).hexdigest()
+
+    @property
+    def end_time(self):
+        if self.duration and self.timestamp:
+            return self.timestamp + timedelta(seconds=self.duration)
+        return None
+
+    def get_dict(self):
+        instance_dict = super(SlowCall, self).get_dict()
+        instance_dict['children'] = []
+        instance_dict['end_time'] = self.end_time
+        return instance_dict
+
+    def es_doc(self):
+        doc = {
+            'resource_id': self.resource_id,
+            'timestamp': self.timestamp,
+            'pg_id': str(self.id),
+            'permanent': False,
+            'request_id': None,
+            'log_level': 'UNKNOWN',
+            'message': self.statement,
+            'namespace': 'appenlight.slow_call',
+            'tags': {
+                'report_id': {'values': self.report_id,
+                              'numeric_values': self.report_id},
+                'duration': {'values': None, 'numeric_values': self.duration},
+                'statement_hash': {'values': self.statement_hash,
+                                   'numeric_values': None},
+                'type': {'values': self.type, 'numeric_values': None},
+                'subtype': {'values': self.subtype, 'numeric_values': None},
+                'location': {'values': self.location, 'numeric_values': None},
+                'parameters': {'values': None, 'numeric_values': None}
+            },
+            'tag_list': ['report_id', 'duration', 'statement_hash', 'type',
+                         'subtype', 'location']
+        }
+        if isinstance(self.parameters, str):
+            doc['tags']['parameters']['values'] = self.parameters[:255]
+        return doc
+
+    @property
+    def partition_id(self):
+        return 'rcae_sc_%s' % self.report_group_time.strftime('%Y_%m')

backend/src/appenlight/models/tag.py

@@ -0,0 +1,42 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import sqlalchemy as sa
+from datetime import datetime
+from ziggurat_foundations.models.base import BaseModel
+from sqlalchemy.dialects.postgres import JSON
+
+from . import Base
+
+
+class Tag(Base, BaseModel):
+    __tablename__ = 'tags'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    resource_id = sa.Column(sa.Integer,
+                            sa.ForeignKey('resources.resource_id'))
+    name = sa.Column(sa.Unicode(512), nullable=False)
+    value = sa.Column(JSON, nullable=False)
+    first_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                                server_default=sa.func.now())
+    last_timestamp = sa.Column(sa.DateTime(), default=datetime.utcnow,
+                               server_default=sa.func.now())
+    times_seen = sa.Column(sa.Integer, nullable=False, default=0)

backend/src/appenlight/models/user.py

@@ -0,0 +1,138 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import sqlalchemy as sa
+from datetime import datetime
+from appenlight.models import Base, get_db_session
+from appenlight.models.services.event import EventService
+from appenlight.models.integrations import IntegrationException
+from pyramid.threadlocal import get_current_request
+from ziggurat_foundations.models.user import UserMixin
+
+log = logging.getLogger(__name__)
+
+
+class User(UserMixin, Base):
+    __possible_permissions__ = []
+
+    first_name = sa.Column(sa.Unicode(25))
+    last_name = sa.Column(sa.Unicode(25))
+    company_name = sa.Column(sa.Unicode(255), default='')
+    company_address = sa.Column(sa.Unicode(255), default='')
+    zip_code = sa.Column(sa.Unicode(25), default='')
+    city = sa.Column(sa.Unicode(50), default='')
+    default_report_sort = sa.Column(sa.Unicode(25), default='newest')
+    notes = sa.Column(sa.UnicodeText, default='')
+    notifications = sa.Column(sa.Boolean(), default=True)
+    registration_ip = sa.Column(sa.UnicodeText(), default='')
+    alert_channels = sa.orm.relationship('AlertChannel',
+                                         cascade="all,delete-orphan",
+                                         passive_deletes=True,
+                                         passive_updates=True,
+                                         backref='owner',
+                                         order_by='AlertChannel.channel_name, '
+                                                  'AlertChannel.channel_value')
+
+    alert_actions = sa.orm.relationship('AlertChannelAction',
+                                        cascade="all,delete-orphan",
+                                        passive_deletes=True,
+                                        passive_updates=True,
+                                        backref='owner',
+                                        order_by='AlertChannelAction.pkey')
+
+    auth_tokens = sa.orm.relationship('AuthToken',
+                                      cascade="all,delete-orphan",
+                                      passive_deletes=True,
+                                      passive_updates=True,
+                                      backref='owner',
+                                      order_by='AuthToken.creation_date')
+
+    def get_dict(self, exclude_keys=None, include_keys=None,
+                 extended_info=False):
+        result = super(User, self).get_dict(exclude_keys, include_keys)
+        if extended_info:
+            result['groups'] = [g.group_name for g in self.groups]
+            result['permissions'] = [p.perm_name for p in self.permissions]
+            request = get_current_request()
+            apps = self.resources_with_perms(
+                ['view'], resource_types=['application'])
+            result['applications'] = sorted(
+                [{'resource_id': a.resource_id,
+                  'resource_name': a.resource_name}
+                 for a in apps.all()],
+                key=lambda x: x['resource_name'].lower())
+            result['assigned_reports'] = [r.get_dict(request) for r
+                                          in self.assigned_report_groups]
+            result['latest_events'] = [ev.get_dict(request) for ev
+                                       in self.latest_events()]
+
+        exclude_keys_list = exclude_keys or []
+        include_keys_list = include_keys or []
+        d = {}
+        for k in result.keys():
+            if (k not in exclude_keys_list and
+                    (k in include_keys_list or not include_keys)):
+                d[k] = result[k]
+        return d
+
+    def __repr__(self):
+        return '<User: %s, id: %s>' % (self.user_name, self.id)
+
+    @property
+    def assigned_report_groups(self):
+        from appenlight.models.report_group import ReportGroup
+
+        resources = self.resources_with_perms(
+            ['view'], resource_types=['application'])
+        query = self.assigned_reports_relation
+        rid_list = [r.resource_id for r in resources]
+        query = query.filter(ReportGroup.resource_id.in_(rid_list))
+        query = query.limit(50)
+        return query
+
+    def feed_report(self, report):
+        """ """
+        if not hasattr(self, 'current_reports'):
+            self.current_reports = []
+        self.current_reports.append(report)
+
+    def send_digest(self, request, application, reports, since_when=None,
+                    db_session=None):
+        db_session = get_db_session(db_session)
+        if not reports:
+            return True
+        if not since_when:
+            since_when = datetime.utcnow()
+        for channel in self.alert_channels:
+            if not channel.channel_validated or not channel.daily_digest:
+                continue
+            try:
+                channel.send_digest(resource=application,
+                                    user=self,
+                                    request=request,
+                                    since_when=since_when,
+                                    reports=reports)
+            except IntegrationException as e:
+                log.warning('%s' % e)
+
+    def latest_events(self):
+        return EventService.latest_for_user(self)

backend/src/appenlight/models/user_group.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.user_group import UserGroupMixin
+from appenlight.models import Base
+
+
+class UserGroup(UserGroupMixin, Base):
+    pass

backend/src/appenlight/models/user_permission.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.user_permission import UserPermissionMixin
+from appenlight.models import Base
+
+
+class UserPermission(UserPermissionMixin, Base):
+    pass

backend/src/appenlight/models/user_resource_permission.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+from ziggurat_foundations.models.user_resource_permission import UserResourcePermissionMixin
+from appenlight.models import Base
+
+
+class UserResourcePermission(UserResourcePermissionMixin, Base):
+    pass

backend/src/appenlight/predicates.py

@@ -0,0 +1,148 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+from appenlight.forms import CSRFException
+
+log = logging.getLogger(__name__)
+
+from pyramid.interfaces import IDefaultCSRFOptions
+from pyramid.session import (
+    check_csrf_origin,
+    check_csrf_token,
+)
+
+# taken directly from pyramid 1.7
+# pyramid/viewderivers.py
+# the difference is this deriver will ignore csrf_check when auth token
+# policy is in effect
+
+def csrf_view(view, info):
+    explicit_val = info.options.get('require_csrf')
+    defaults = info.registry.queryUtility(IDefaultCSRFOptions)
+    if defaults is None:
+        default_val = False
+        token = 'csrf_token'
+        header = 'X-CSRF-Token'
+        safe_methods = frozenset(["GET", "HEAD", "OPTIONS", "TRACE"])
+    else:
+        default_val = defaults.require_csrf
+        token = defaults.token
+        header = defaults.header
+        safe_methods = defaults.safe_methods
+    enabled = (
+        explicit_val is True or
+        (explicit_val is not False and default_val)
+    )
+    # disable if both header and token are disabled
+    enabled = enabled and (token or header)
+    wrapped_view = view
+    if enabled:
+        def csrf_view(context, request):
+            is_from_auth_token = 'auth:auth_token' in \
+                                 request.effective_principals
+            if is_from_auth_token:
+                log.debug('ignoring CSRF check, auth token used')
+            elif (
+                request.method not in safe_methods and
+                (
+                # skip exception views unless value is explicitly defined
+                        getattr(request, 'exception', None) is None or
+                        explicit_val is not None
+                )
+            ):
+                check_csrf_origin(request, raises=True)
+                check_csrf_token(request, token, header, raises=True)
+            return view(context, request)
+
+        wrapped_view = csrf_view
+    return wrapped_view
+
+csrf_view.options = ('require_csrf',)
+
+
+class PublicReportGroup(object):
+    def __init__(self, val, config):
+        self.val = val
+
+    def text(self):
+        return 'public_report_group = %s' % (self.val,)
+
+    phash = text
+
+    def __call__(self, context, request):
+        report_group = getattr(context, 'report_group', None)
+        if report_group:
+            return context.report_group.public == self.val
+
+
+class contextTypeClass(object):
+    def __init__(self, context_property, config):
+        self.context_property = context_property[0]
+        self.cls = context_property[1]
+
+    def text(self):
+        return 'context_type_class = %s, %s' % (
+            self.context_property, self.cls)
+
+    phash = text
+
+    def __call__(self, context, request):
+        to_check = getattr(context, self.context_property, None)
+        return isinstance(to_check, self.cls)
+
+
+def unauthed_report_predicate(context, request):
+    """
+    This allows the user to access the view if context object public
+    flag is True
+    """
+    if context.public:
+        return True
+
+
+def unauthed_report_predicate_inv(context, request):
+    """
+    This allows the user to access the view if context object public
+    flag is NOT True
+    """
+    if context.public:
+        return False
+    return True
+
+
+def unauthed_report_predicate(context, request):
+    """
+    This allows the user to access the view if context object public
+    flag is True
+    """
+    if context.public:
+        return True
+
+
+def unauthed_report_predicate_inv(context, request):
+    """
+    This allows the user to access the view if context object public
+    flag is NOT True
+    """
+    if context.public:
+        return False
+    return True

backend/src/appenlight/scripts/__init__.py

@@ -0,0 +1,21 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+

backend/src/appenlight/scripts/cleanup.py

@@ -0,0 +1,68 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import logging
+import argparse
+from pyramid.paster import setup_logging
+from pyramid.paster import bootstrap
+from appenlight.celery.tasks import logs_cleanup
+
+log = logging.getLogger(__name__)
+
+
+def main():
+    choices = ['logs']
+
+    parser = argparse.ArgumentParser(description='Cleanup App Enlight logs')
+    parser.add_argument('-c', '--config', required=True,
+                        help='Configuration ini file of application')
+    parser.add_argument('-t', '--types', choices=choices,
+                        default='logs',
+                        help='Which parts of database should get cleared')
+    parser.add_argument('-r', '--resource', required=True, help='Resource id')
+    parser.add_argument('-n', '--namespace', help='Limit to Namespace')
+    args = parser.parse_args()
+
+    config_uri = args.config
+    setup_logging(config_uri)
+    log.setLevel(logging.INFO)
+    env = bootstrap(config_uri)
+
+    config = {
+        'types': args.types,
+        'namespace': args.namespace,
+        'resource': int(args.resource),
+    }
+
+    action_cleanup_logs(config)
+
+
+def action_cleanup_logs(config):
+    filter_settings = {
+        'namespace': []
+    }
+    if config['namespace']:
+        filter_settings['namespace'].append(config['namespace'])
+    logs_cleanup(config['resource'], filter_settings)
+
+
+if __name__ == '__main__':
+    main()

backend/src/appenlight/scripts/initialize_db.py

@@ -0,0 +1,158 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import argparse
+import getpass
+import logging
+
+from pyramid.paster import setup_logging, bootstrap
+from pyramid.threadlocal import get_current_request
+
+from appenlight.forms import UserRegisterForm
+from appenlight.lib.ext_json import json
+from appenlight.models import (
+    DBSession,
+    Group,
+    GroupPermission,
+    User,
+    AuthToken
+)
+from appenlight.models.services.group import GroupService
+
+log = logging.getLogger(__name__)
+
+_ = str
+
+
+def is_yes(input_data):
+    return input_data in ['y', 'yes']
+
+
+def is_no(input_data):
+    return input_data in ['n', 'no']
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Populate App Enlight database',
+        add_help=False)
+    parser.add_argument('-c', '--config', required=True,
+                        help='Configuration ini file of application')
+    parser.add_argument('--username', default=None,
+                        help='User  to create')
+    parser.add_argument('--password', default=None,
+                        help='Password for created user')
+    parser.add_argument('--email', default=None,
+                        help='Email for created user')
+    parser.add_argument('--auth-token', default=None,
+                        help='Auth token for created user')
+    args = parser.parse_args()
+    config_uri = args.config
+
+    setup_logging(config_uri)
+    env = bootstrap(config_uri)
+    request = env['request']
+    with get_current_request().tm:
+        group = GroupService.by_id(1)
+        if not group:
+            group = Group(id=1, group_name='Administrators',
+                           description="Top level permission owners")
+            DBSession.add(group)
+            permission = GroupPermission(perm_name='root_administration')
+            group.permissions.append(permission)
+
+    create_user = True if args.username else None
+    while create_user is None:
+        response = input(
+            'Do you want to create a new admin? (n)\n').lower()
+
+        if is_yes(response or 'n'):
+            create_user = True
+        elif is_no(response or 'n'):
+            create_user = False
+
+    if create_user:
+        csrf_token = request.session.get_csrf_token()
+        user_name = args.username
+        print('*********************************************************')
+        while user_name is None:
+            response = input('What is the username of new admin?\n')
+            form = UserRegisterForm(
+                user_name=response, csrf_token=csrf_token,
+                csrf_context=request)
+            form.validate()
+            if form.user_name.errors:
+                print(form.user_name.errors[0])
+            else:
+                user_name = response
+                print('The admin username is "{}"\n'.format(user_name))
+        print('*********************************************************')
+        email = args.email
+        while email is None:
+            response = input('What is the email of admin account?\n')
+            form = UserRegisterForm(
+                email=response, csrf_token=csrf_token,
+                csrf_context=request)
+            form.validate()
+            if form.email.errors:
+                print(form.email.errors[0])
+            else:
+                email = response
+                print('The admin email is "{}"\n'.format(email))
+        print('*********************************************************')
+        user_password = args.password
+        confirmed_password = args.password
+        while user_password is None or confirmed_password is None:
+            response = getpass.getpass(
+                'What is the password for admin account?\n')
+            form = UserRegisterForm(
+                user_password=response, csrf_token=csrf_token,
+                csrf_context=request)
+            form.validate()
+            if form.user_password.errors:
+                print(form.user_password.errors[0])
+            else:
+                user_password = response
+
+            response = getpass.getpass('Please confirm the password.\n')
+            if user_password == response:
+                confirmed_password = response
+            else:
+                print('Passwords do not match. Please try again')
+        print('*********************************************************')
+
+    with get_current_request().tm:
+        if create_user:
+            group = GroupService.by_id(1)
+            user = User(user_name=user_name, email=email, status=1)
+            user.regenerate_security_code()
+            user.set_password(user_password)
+            DBSession.add(user)
+            token = AuthToken(description="Uptime monitoring token")
+            if args.auth_token:
+                token.token = args.auth_token
+            user.auth_tokens.append(token)
+            group.users.append(user)
+            print('USER CREATED')
+            print(json.dumps(user.get_dict()))
+            print('*********************************************************')
+            print('AUTH TOKEN')
+            print(json.dumps(user.auth_tokens[0].get_dict()))

backend/src/appenlight/scripts/make_config.py

@@ -0,0 +1,62 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import argparse
+import logging
+import os
+import pkg_resources
+import jinja2
+from cryptography.fernet import Fernet
+
+
+log = logging.getLogger(__name__)
+
+
+def gen_secret():
+    return Fernet.generate_key().decode('utf8')
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Generate App Enlight static resources',
+        add_help=False)
+    parser.add_argument('config', help='Name of generated file')
+    parser.add_argument(
+        '--domain',
+        default='appenlight-rhodecode.local',
+        help='Domain which will be used to serve the application')
+    parser.add_argument(
+        '--dbstring',
+        default='postgresql://appenlight:test@127.0.0.1:5432/appenlight',
+        help='Domain which will be used to serve the application')
+    args = parser.parse_args()
+    ini_path = os.path.join('templates', 'ini', 'production.ini.jinja2')
+    template_str = pkg_resources.resource_string('appenlight', ini_path)
+    template = jinja2.Template(template_str.decode('utf8'))
+    template_vars = {'appenlight_encryption_secret': gen_secret(),
+                     'appenlight_authtkt_secret': gen_secret(),
+                     'appenlight_redis_session_secret': gen_secret(),
+                     'appenlight_domain': args.domain,
+                     'appenlight_dbstring': args.dbstring,
+                     }
+    compiled = template.render(**template_vars)
+    with open(args.config, 'w') as f:
+        f.write(compiled)

backend/src/appenlight/scripts/migratedb.py

@@ -0,0 +1,75 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# App Enlight Enterprise Edition, including its added features, Support
+# services, and proprietary license terms, please see
+# https://rhodecode.com/licenses/
+
+import argparse
+import logging
+import sys
+
+from alembic.config import Config
+from alembic import command
+from pyramid.paster import setup_logging, bootstrap
+from pyramid.threadlocal import get_current_registry, get_current_request
+
+from appenlight.lib import get_callable
+from appenlight.models.services.config import ConfigService
+
+log = logging.getLogger(__name__)
+
+
+def main(argv=sys.argv):
+    parser = argparse.ArgumentParser(
+        description='Migrate App Enlight database to latest version',
+        add_help=False)
+    parser.add_argument('-c', '--config', required=True,
+                        help='Configuration ini file of application')
+    args = parser.parse_args()
+    config_uri = args.config
+
+    setup_logging(config_uri)
+    bootstrap(config_uri)
+    registry = get_current_registry()
+    alembic_cfg = Config()
+    alembic_cfg.set_main_option("script_location",
+                                "ziggurat_foundations:migrations")
+    alembic_cfg.set_main_option("sqlalchemy.url",
+                                registry.settings["sqlalchemy.url"])
+    command.upgrade(alembic_cfg, "head")
+    alembic_cfg = Config()
+    alembic_cfg.set_main_option("script_location", "appenlight:migrations")
+    alembic_cfg.set_main_option("sqlalchemy.url",
+                                registry.settings["sqlalchemy.url"])
+    command.upgrade(alembic_cfg, "head")
+
+    for plugin_name, config in registry.appenlight_plugins.items():
+        if config['sqlalchemy_migrations']:
+            alembic_cfg = Config()
+            alembic_cfg.set_main_option("script_location",
+                                        config['sqlalchemy_migrations'])
+            alembic_cfg.set_main_option("sqlalchemy.url",
+                                        registry.settings["sqlalchemy.url"])
+            command.upgrade(alembic_cfg, "head")
+
+    with get_current_request().tm:
+        ConfigService.setup_default_values()
+
+        for plugin_name, config in registry.appenlight_plugins.items():
+            if config['default_values_setter']:
+                get_callable(config['default_values_setter'])()