configs: refactor and updated for python3 migrations

rhodecode/config/licenses.json

@@ -1,1943 +1,1921 b''
 [
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             },
             {
                 "fullName": "Zope Public License 2.0",
                 "shortName": "zpl20",
                 "spdxId": "ZPL-2.0",
                 "url": "http://spdx.org/licenses/ZPL-2.0.html"
             }
         ],
         "name": "python2.7-setuptools-38.4.0"
     },
     {
         "license": {
             "fullName": "Python Software Foundation License version 2",
             "shortName": "psfl",
             "spdxId": "Python-2.0",
             "url": "http://spdx.org/licenses/Python-2.0.html"
         },
         "name": "python-2.7.15"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-beautifulsoup4-4.6.3"
     },
     {
         "license": "UNKNOWN",
         "name": "python2.7-bootstrapped-pip-9.0.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-webtest-2.0.33"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-waitress-1.3.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-webob-1.8.5"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-six-1.11.0"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-coverage-4.5.4"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-cov-core-1.15.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "OSI Approved :: BSD License"
             }
         ],
         "name": "python2.7-mock-3.0.5"
     },
     {
         "license": [
             {
                 "fullName": "ASL"
             },
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-funcsigs-1.0.2"
     },
     {
         "license": [
             {
                 "fullName": "GNU Lesser General Public License v3 or later (LGPLv3+)"
             },
             {
                 "fullName": "LGPL"
             }
         ],
         "name": "python2.7-gprof2dot-2017.9.19"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             },
             {
                 "fullName": "DFSG approved"
             }
         ],
         "name": "python2.7-pytest-timeout-1.3.3"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pytest-4.6.5"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-more-itertools-5.0.0"
     },
     {
         "license": [
             {
-                "fullName": "MIT License",
-                "shortName": "mit",
-                "spdxId": "MIT",
-                "url": "http://spdx.org/licenses/MIT.html"
-            }
-        ],
-        "name": "python2.7-pathlib2-2.3.5"
-    },
-    {
-        "license": [
-            {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "New BSD License"
             }
         ],
         "name": "python2.7-scandir-1.10.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-wcwidth-0.1.7"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-importlib-metadata-0.23"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-configparser-4.0.2"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-contextlib2-0.6.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-zipp-0.6.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pluggy-0.13.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-atomicwrites-1.3.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-attrs-19.1.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "BSD or Apache License, Version 2.0"
             },
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-packaging-19.2"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pyparsing-2.4.2"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-py-1.8.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pytest-profiling-1.7.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pytest-runner-5.1"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-pytest-sugar-0.9.2"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-termcolor-1.1.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pytest-cov-2.7.1"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-appenlight-client-0.6.26"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-requests-2.9.1"
     },
     {
         "license": [
             {
                 "fullName": "Apache 2.0 and Proprietary"
             }
         ],
         "name": "python2.7-rhodecode-tools-1.3.0"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-elasticsearch1-dsl-0.0.12"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-elasticsearch1-1.10.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-urllib3-1.25.2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             },
             {
                 "fullName": "Dual License"
             }
         ],
         "name": "python2.7-python-dateutil-2.8.1"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-elasticsearch2-2.5.1"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-elasticsearch-dsl-6.3.1"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-ipaddress-1.0.23"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-elasticsearch-6.3.1"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "BSD 2-clause \"Simplified\" License",
                 "shortName": "bsd2",
                 "spdxId": "BSD-2-Clause",
                 "url": "http://spdx.org/licenses/BSD-2-Clause.html"
             }
         ],
         "name": "python2.7-whoosh-2.7.4"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "BSD 3-clause \"New\" or \"Revised\" License",
                 "shortName": "bsd3",
                 "spdxId": "BSD-3-Clause",
                 "url": "http://spdx.org/licenses/BSD-3-Clause.html"
             }
         ],
         "name": "python2.7-markupsafe-1.1.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-mako-1.1.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-future-0.14.3"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-click-7.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-ipython-5.1.0"
     },
     {
         "license": [
             {
                 "fullName": "GNU General Public License v3 (GPLv3)"
             },
             {
                 "fullName": "GNU General Public License v1.0 only",
                 "shortName": "gpl1",
                 "spdxId": "GPL-1.0",
                 "url": "http://spdx.org/licenses/GPL-1.0.html"
             }
         ],
         "name": "python2.7-gnureadline-6.3.8"
     },
     {
         "license": [
             {
                 "fullName": "ISC License",
                 "shortName": "isc",
                 "spdxId": "ISC",
                 "url": "http://spdx.org/licenses/ISC.html"
             },
             {
                 "fullName": "ISC License (ISCL)"
             }
         ],
         "name": "python2.7-pexpect-4.7.0"
     },
     {
         "license": [],
         "name": "python2.7-ptyprocess-0.6.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-backports.shutil-get-terminal-size-1.0.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-pygments-2.4.2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-prompt-toolkit-1.0.18"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-traitlets-4.3.3"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-enum34-1.1.6"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "new BSD License"
             }
         ],
         "name": "python2.7-decorator-4.1.2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-ipython-genutils-0.2.0"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-simplegeneric-0.8.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pickleshare-0.7.5"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-ipdb-0.12"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-gunicorn-19.9.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-futures-3.0.2"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-greenlet-0.4.15"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-gevent-1.4.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-psutil-5.6.5"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-bumpversion-0.5.3"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-invoke-0.13.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-alembic-1.3.1"
     },
     {
         "license": {
             "fullName": "Apache License 2.0",
             "shortName": "asl20",
             "spdxId": "Apache-2.0",
             "url": "http://spdx.org/licenses/Apache-2.0.html"
         },
         "name": "python2.7-python-editor-1.0.4"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-sqlalchemy-1.3.11"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-jupyter-core-4.5.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-jupyter-client-5.0.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "LGPL+BSD"
             },
             {
                 "fullName": "GNU Library or Lesser General Public License (LGPL)"
             }
         ],
         "name": "python2.7-pyzmq-14.6.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-nbformat-4.4.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-jsonschema-2.6.0"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-functools32-3.2.3.post2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-nbconvert-5.3.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-testpath-0.4.4"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-pandocfilters-1.4.2"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-bleach-3.1.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-webencodings-0.5.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-entrypoints-0.2.2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-jinja2-2.9.6"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-mistune-0.8.4"
     },
     {
         "license": {
             "fullName": "GNU Lesser General Public License v3.0 or later",
             "shortName": "lgpl3Plus",
             "spdxId": "LGPL-3.0+",
             "url": "http://spdx.org/licenses/LGPL-3.0+.html"
         },
         "name": "python2.7-psycopg2-2.8.4"
     },
     {
         "license": {
             "fullName": "PostgreSQL License",
             "shortName": "postgresql",
             "spdxId": "PostgreSQL",
             "url": "http://spdx.org/licenses/PostgreSQL.html"
         },
         "name": "postgresql-9.6.10"
     },
     {
         "license": [
             {
                 "fullName": "zlib License",
                 "shortName": "zlib",
                 "spdxId": "Zlib",
                 "url": "http://spdx.org/licenses/Zlib.html"
             },
             {
                 "fullName": "libpng License",
                 "shortName": "libpng",
                 "spdxId": "Libpng",
                 "url": "http://spdx.org/licenses/Libpng.html"
             }
         ],
         "name": "python2.7-pysqlite-2.8.3"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pymysql-0.8.1"
     },
     {
         "license": [
             {
                 "fullName": "GNU General Public License v1.0 only",
                 "shortName": "gpl1",
                 "spdxId": "GPL-1.0",
                 "url": "http://spdx.org/licenses/GPL-1.0.html"
             }
         ],
         "name": "python2.7-mysql-python-1.2.5"
     },
     {
         "license": {
             "fullName": "GNU Library General Public License v2.1 only",
             "shortName": "lgpl21",
             "spdxId": "LGPL-2.1",
             "url": "http://spdx.org/licenses/LGPL-2.1.html"
         },
         "name": "mariadb-connector-c-2.3.4"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-zope.interface-4.6.0"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-zope.event-4.4"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-zope.deprecation-4.4.0"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-zope.cachedescriptors-4.3.1"
     },
     {
         "license": [
             {
                 "fullName": "PSF or ZPL"
             }
         ],
         "name": "python2.7-wsgiref-0.1.2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-webhelpers-1.3"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-webhelpers2-2.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-weberror-0.10.3"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-paste-3.2.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-tempita-0.5.2"
     },
     {
         "license": {
             "fullName": "Repoze License",
             "url": "http://www.repoze.org/LICENSE.txt"
         },
         "name": "python2.7-venusian-1.2.0"
     },
     {
         "license": {
             "fullName": "The Unlicense",
             "spdxId": "Unlicense",
             "url": "http://unlicense.org/"
         },
         "name": "python2.7-urlobject-2.4.3"
     },
     {
         "license": {
             "fullName": "Repoze License",
             "url": "http://www.repoze.org/LICENSE.txt"
         },
         "name": "python2.7-translationstring-1.3"
     },
     {
         "license": [
             {
                 "fullName": "BSD-derived (http://www.repoze.org/LICENSE.txt)"
             }
         ],
         "name": "python2.7-supervisor-4.1.0"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-subprocess32-3.5.4"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-sshpubkeys-3.1.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-ecdsa-0.13.2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "BSD or Apache License, Version 2.0"
             },
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-cryptography-2.6.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-cffi-1.12.3"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-pycparser-2.19"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-asn1crypto-0.24.0"
     },
     {
         "license": [
             {
                 "fullName": "Academic Free License (AFL)"
             },
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-simplejson-3.16.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-routes-2.4.1"
     },
     {
         "license": {
             "fullName": "Repoze License",
             "url": "http://www.repoze.org/LICENSE.txt"
         },
         "name": "python2.7-repoze.lru-0.7"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-redis-3.3.11"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-py-gfm-0.1.4"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-markdown-2.6.11"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-tzlocal-1.5.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pytz-2019.2"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-python-saml-2.4.2"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-defusedxml-0.6.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-isodate-0.6.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-dm.xmlsec.binding-1.3.7"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-lxml-4.2.5"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-wheel-0.30.0"
     },
     {
         "license": [
             {
                 "fullName": "License :: OSI Approved :: MIT License"
             },
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-python-pam-1.8.4"
     },
     {
         "license": "UNKNOWN",
         "name": "linux-pam-1.3.0"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-python-memcached-1.59"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-python-ldap-3.1.0"
     },
     {
         "license": {
             "fullName": "MIT License",
             "shortName": "mit",
             "spdxId": "MIT",
             "url": "http://spdx.org/licenses/MIT.html"
         },
         "name": "libkrb5-1.15.2"
     },
     {
         "license": "UNKNOWN",
         "name": "openldap-2.4.45"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "BSD 2-clause \"Simplified\" License",
                 "shortName": "bsd2",
                 "spdxId": "BSD-2-Clause",
                 "url": "http://spdx.org/licenses/BSD-2-Clause.html"
             }
         ],
         "name": "python2.7-pyasn1-modules-0.2.6"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-pyasn1-0.4.7"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-pyramid-mailer-0.15.1"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-transaction-2.4.0"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-repoze.sendmail-4.4.1"
     },
     {
         "license": {
             "fullName": "Repoze License",
             "url": "http://www.repoze.org/LICENSE.txt"
         },
         "name": "python2.7-pyramid-1.10.4"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-plaster-pastedeploy-0.7"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-plaster-1.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pastedeploy-2.0.1"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-hupper-1.9.1"
     },
     {
         "license": {
             "fullName": "Repoze License",
             "url": "http://www.repoze.org/LICENSE.txt"
         },
         "name": "python2.7-pyramid-mako-1.1.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             },
             {
                 "fullName": "Repoze License",
                 "url": "http://www.repoze.org/LICENSE.txt"
             }
         ],
         "name": "python2.7-pyramid-debugtoolbar-4.5.1"
     },
     {
         "license": [
             {
                 "fullName": "Public Domain",
                 "shortName": "publicDomain"
             }
         ],
         "name": "python2.7-pycrypto-2.6.1"
     },
     {
         "license": {
             "fullName": "MIT License",
             "shortName": "mit",
             "spdxId": "MIT",
             "url": "http://spdx.org/licenses/MIT.html"
         },
         "name": "python2.7-pycurl-7.43.0.3"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-py-bcrypt-0.4"
     },
     {
         "license": [
             {
                 "fullName": "BSD-derived (http://www.repoze.org/LICENSE.txt)"
             }
         ],
         "name": "python2.7-peppercorn-0.6"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pastescript-3.2.0"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-pyotp-2.3.0"
     },
     {
         "license": [
             {
                 "fullName": "Apache License 2.0",
                 "shortName": "asl20",
                 "spdxId": "Apache-2.0",
                 "url": "http://spdx.org/licenses/Apache-2.0.html"
             }
         ],
         "name": "python2.7-msgpack-python-0.5.6"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-kombu-4.6.6"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-amqp-2.5.2"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-vine-1.3.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-itsdangerous-0.24"
     },
     {
         "license": [
             {
                 "fullName": "MIT License",
                 "shortName": "mit",
                 "spdxId": "MIT",
                 "url": "http://spdx.org/licenses/MIT.html"
             }
         ],
         "name": "python2.7-iso8601-0.1.12"
     },
     {
         "license": [
             {
                 "fullName": "Zope Public License 2.1",
                 "shortName": "zpl21",
                 "spdxId": "ZPL-2.1",
                 "url": "http://spdx.org/licenses/ZPL-2.1.html"
             }
         ],
         "name": "python2.7-infrae.cache-1.0.1"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-beaker-1.9.1"
     },
     {
         "license": [
             {
                 "fullName": "Python Software Foundation License version 2",
                 "shortName": "psfl",
                 "spdxId": "Python-2.0",
                 "url": "http://spdx.org/licenses/Python-2.0.html"
             }
         ],
         "name": "python2.7-formencode-1.2.4"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-dogpile.cache-0.9.0"
     },
     {
         "license": {
             "fullName": "BSD 2-clause \"Simplified\" License",
             "shortName": "bsd2",
             "spdxId": "BSD-2-Clause",
             "url": "http://spdx.org/licenses/BSD-2-Clause.html"
         },
         "name": "python2.7-docutils-0.14"
     },
     {
         "license": [
             {
                 "fullName": "Repoze Public License"
             },
             {
                 "fullName": "BSD-derived (http://www.repoze.org/LICENSE.txt)"
             }
         ],
         "name": "python2.7-deform-2.0.8"
     },
     {
         "license": {
             "fullName": "Repoze License",
             "url": "http://www.repoze.org/LICENSE.txt"
         },
         "name": "python2.7-colander-1.7.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD-like (http://repoze.org/license.html)"
             }
         ],
         "name": "python2.7-chameleon-2.24"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-cssselect-1.0.3"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
-        "name": "python2.7-configobj-5.0.6"
-    },
-    {
-        "license": [
-            {
-                "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
-                "shortName": "bsdOriginal",
-                "spdxId": "BSD-4-Clause",
-                "url": "http://spdx.org/licenses/BSD-4-Clause.html"
-            }
-        ],
         "name": "python2.7-channelstream-0.6.14"
     },
     {
         "license": [
             {
                 "fullName": "Repoze Public License"
             },
             {
                 "fullName": "BSD-derived (http://www.repoze.org/LICENSE.txt)"
             }
         ],
         "name": "python2.7-pyramid-jinja2-2.7"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-ws4py-0.5.1"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-celery-4.3.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-billiard-3.6.1.0"
     },
     {
         "license": [
             {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
         "name": "python2.7-babel-1.3"
     },
     {
         "license": "UNKNOWN",
         "name": "python2.7-rhodecode-testdata-0.10.0"
     }
 ]

rhodecode/config/middleware.py

@@ -1,616 +1,630 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import collections
 import tempfile
 import time
 import logging.config
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
 from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config import utils as config_utils
 from rhodecode.config.settings_maker import SettingsMaker
 from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
     write_metadata_if_needed, write_usage_data)
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 def is_http_error(response):
     # error which should have traceback
     return response.status_code > 499
 
 
 def should_load_all():
     """
     Returns if all application components should be loaded. In some cases it's
     desired to skip apps loading for faster shell script execution
     """
     ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
     if ssh_cmd:
         return False
 
     return True
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid.
 
     Specials:
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     start_time = time.time()
     log.info('Pyramid app config starting')
 
     sanitize_settings_and_apply_defaults(global_config, settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
     config = Configurator(settings=settings)
     # Init our statsd at very start
     config.registry.statsd = StatsdClient.statsd
 
     # Apply compatibility patches
     patches.inspect_getargspec()
 
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first
     includeme_first(config)
 
     includeme(config)
 
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
     celery_settings = get_celery_config(settings)
     config.configure_celery(celery_settings)
 
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 
     total_time = time.time() - start_time
-    log.info('Pyramid app `%s` created and configured in %.2fs',
-             getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
+    log.info('Pyramid app created and configured in %.2fs', total_time)
     return pyramid_app
 
 
 def get_celery_config(settings):
     """
     Converts basic ini configuration into celery 4.X options
     """
 
     def key_converter(key_name):
         pref = 'celery.'
         if key_name.startswith(pref):
             return key_name[len(pref):].replace('.', '_').lower()
 
     def type_converter(parsed_key, value):
         # cast to int
         if value.isdigit():
             return int(value)
 
         # cast to bool
         if value.lower() in ['true', 'false', 'True', 'False']:
             return value.lower() == 'true'
         return value
 
     celery_config = {}
     for k, v in settings.items():
         pref = 'celery.'
         if k.startswith(pref):
             celery_config[key_converter(k)] = type_converter(key_converter(k), v)
 
     # TODO:rethink if we want to support celerybeat based file config, probably NOT
     # beat_config = {}
     # for section in parser.sections():
     #     if section.startswith('celerybeat:'):
     #         name = section.split(':', 1)[1]
     #         beat_config[name] = get_beat_config(parser, section)
 
     # final compose of settings
     celery_settings = {}
 
     if celery_config:
         celery_settings.update(celery_config)
     # if beat_config:
     #     celery_settings.update({'beat_schedule': beat_config})
 
     return celery_settings
 
 
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
     pyramid.
     """
 
     if not getattr(request, 'vcs_call', None):
         # handle like regular case with our error_handler
         return error_handler(HTTPNotFound(), request)
 
     # handle not found view as a vcs call
     settings = request.registry.settings
     ae_client = getattr(request, 'ae_client', None)
     vcs_app = VCSMiddleware(
         HTTPNotFound(), request.registry, settings,
         appenlight_client=ae_client)
 
     return wsgiapp(vcs_app)(None, request)
 
 
 def error_handler(exception, request):
     import rhodecode
     from rhodecode.lib import helpers
     from rhodecode.lib.utils2 import str2bool
 
     rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPException):
         base_response = exception
     elif isinstance(exception, VCSCommunicationError):
         base_response = VCSServerUnavailable()
 
     if is_http_error(base_response):
         log.exception(
             'error occurred handling this request for path: %s', request.path)
 
     error_explanation = base_response.explanation or str(base_response)
     if base_response.status_code == 404:
         error_explanation += " Optionally you don't have permission to access this page."
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = error_explanation
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rhodecode_title
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     c.causes = []
     if is_http_error(base_response):
         c.causes.append('Server is overloaded.')
         c.causes.append('Server database connection is lost.')
         c.causes.append('Server expected unhandled error.')
 
     if hasattr(base_response, 'causes'):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
     exc_info = sys.exc_info()
     c.exception_id = id(exc_info)
     c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                           or base_response.status_code > 499
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
+    debug_mode = rhodecode.ConfigGet().get_bool('debug')
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
-    c.exception_debug = str2bool(rhodecode.CONFIG.get('debug'))
+    c.exception_debug = debug_mode
     c.exception_config_ini = rhodecode.CONFIG.get('__file__')
 
+    if debug_mode:
+        try:
+            from rich.traceback import install
+            install(show_locals=True)
+            log.debug('Installing rich tracebacks...')
+        except ImportError:
+            pass
+
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
 
     statsd = request.registry.statsd
     if statsd and base_response.status_code > 499:
         exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:web",
                           "http_code:{}".format(base_response.status_code),
                           "type:{}".format(exc_type)])
 
     return response
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     def robots_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/robots.txt'))
 
     config.add_view(robots_redirect, route_name='robots')
     config.add_route('robots', '/robots.txt')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
+ce_auth_resources = [
+    'rhodecode.authentication.plugins.auth_crowd',
+    'rhodecode.authentication.plugins.auth_headers',
+    'rhodecode.authentication.plugins.auth_jasig_cas',
+    'rhodecode.authentication.plugins.auth_ldap',
+    'rhodecode.authentication.plugins.auth_pam',
+    'rhodecode.authentication.plugins.auth_rhodecode',
+    'rhodecode.authentication.plugins.auth_token',
+]
+
+
 def includeme(config, auth_resources=None):
     from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)
 
     # plugin information
     config.registry.rhodecode_plugins = collections.OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     config.add_directive('configure_celery', configure_celery)
 
     if settings.get('appenlight', False):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
+    config.include('rhodecode.lib.rc_cache.archive_cache')
+
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
-        ce_auth_resources = [
-            'rhodecode.authentication.plugins.auth_crowd',
-            'rhodecode.authentication.plugins.auth_headers',
-            'rhodecode.authentication.plugins.auth_jasig_cas',
-            'rhodecode.authentication.plugins.auth_ldap',
-            'rhodecode.authentication.plugins.auth_pam',
-            'rhodecode.authentication.plugins.auth_rhodecode',
-            'rhodecode.authentication.plugins.auth_token',
-        ]
 
         # load CE authentication plugins
 
         if auth_resources:
             ce_auth_resources.extend(auth_resources)
 
         for resource in ce_auth_resources:
             config.include(resource)
 
         # Auto discover authentication plugins and include their configuration.
         if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
             from rhodecode.authentication import discover_legacy_plugins
             discover_legacy_plugins(config)
 
     # apps
     if load_all:
         log.debug('Starting config.include() calls')
         config.include('rhodecode.api.includeme')
         config.include('rhodecode.apps._base.includeme')
         config.include('rhodecode.apps._base.navigation.includeme')
         config.include('rhodecode.apps._base.subscribers.includeme')
         config.include('rhodecode.apps.hovercards.includeme')
         config.include('rhodecode.apps.ops.includeme')
         config.include('rhodecode.apps.channelstream.includeme')
         config.include('rhodecode.apps.file_store.includeme')
         config.include('rhodecode.apps.admin.includeme')
         config.include('rhodecode.apps.login.includeme')
         config.include('rhodecode.apps.home.includeme')
         config.include('rhodecode.apps.journal.includeme')
 
         config.include('rhodecode.apps.repository.includeme')
         config.include('rhodecode.apps.repo_group.includeme')
         config.include('rhodecode.apps.user_group.includeme')
         config.include('rhodecode.apps.search.includeme')
         config.include('rhodecode.apps.user_profile.includeme')
         config.include('rhodecode.apps.user_group_profile.includeme')
         config.include('rhodecode.apps.my_account.includeme')
         config.include('rhodecode.apps.gist.includeme')
 
         config.include('rhodecode.apps.svn_support.includeme')
         config.include('rhodecode.apps.ssh_support.includeme')
         config.include('rhodecode.apps.debug_style')
 
     if load_all:
         config.include('rhodecode.integrations.includeme')
         config.include('rhodecode.integrations.routes.includeme')
 
     config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
     settings['default_locale_name'] = settings.get('lang', 'en')
     config.add_translation_dirs('rhodecode:i18n/')
 
     # Add subscribers.
     if load_all:
         log.debug('Adding subscribers....')
         config.add_subscriber(scan_repositories_if_enabled,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_metadata_if_needed,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_usage_data,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
-    # Set the authorization policy.
-    authz_policy = ACLAuthorizationPolicy()
-    config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     config.add_renderer(
         name='json_ext',
         factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
 
     config.add_renderer(
         name='string_html',
         factory='rhodecode.lib.string_renderer.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     log.debug('processing rhodecode.includes data...')
     for inc in includes:
         config.include(inc)
 
     # custom not found view, if our pyramid app doesn't know how to handle
     # the request pass it to potential VCS handling ap
     config.add_notfound_view(not_found_view)
     if not settings.get('debugtoolbar.enabled', False):
         # disabled debugtoolbar handle all exceptions via the error_handlers
         config.add_view(error_handler, context=Exception)
 
     # all errors including 403/404/50X
     config.add_view(error_handler, context=HTTPError)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
     """
     registry = config.registry
     settings = registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
         pyramid_app, settings)
     registry.ae_client = _ae_client
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):
         start = time.time()
         try:
             return pyramid_app(environ, start_response)
         finally:
             # Dispose current database session and rollback uncommitted
             # transactions.
             meta.Session.remove()
 
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
-            pool = meta.Base.metadata.bind.engine.pool
+            pool = meta.get_engine().pool
             log.debug('sa pool status: %s', pool.status())
             total = time.time() - start
             log.debug('Request processing finalized: %.4fs', total)
 
     return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     global_settings_maker = SettingsMaker(global_config)
     global_settings_maker.make_setting('debug', default=False, parser='bool')
     debug_enabled = asbool(global_config.get('debug'))
 
     settings_maker = SettingsMaker(settings)
 
     settings_maker.make_setting(
         'logging.autoconfigure',
         default=False,
         parser='bool')
 
     logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
     settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
 
     # Default includes, possible to change as a user
     pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     settings_maker.make_setting('rhodecode.edition', 'Community Edition')
     settings_maker.make_setting('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
     settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
     settings_maker.make_setting('is_test', False, parser='bool')
     settings_maker.make_setting('gzip_responses', False, parser='bool')
 
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
     settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
     settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
     settings_maker.make_setting('statsd.statsd_prefix', '')
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
-    settings_maker.make_setting('vcs.hooks.host', '127.0.0.1')
+    settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
     settings_maker.make_setting('vcs.server.protocol', 'http')
     settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
     settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
     settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
     settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
     settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
     settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
 
     settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
 
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
     scm_app_impl = settings['vcs.scm_app_implementation']
     if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
         settings['vcs.scm_app_implementation'] = 'http'
 
     settings_maker.make_setting('appenlight', False, parser='bool')
 
     temp_store = tempfile.gettempdir()
     tmp_cache_dir = os.path.join(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = settings_maker.make_setting(
         'cache_dir',
         default=tmp_cache_dir, default_when_empty=True,
         parser='dir:ensured')
 
     # exception store cache
     settings_maker.make_setting(
         'exception_tracker.store_path',
         default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
         parser='dir:ensured'
     )
 
     settings_maker.make_setting(
         'celerybeat-schedule.path',
         default=os.path.join(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
         parser='file:ensured'
     )
 
     settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
     settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
     # cache_general
     settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
     settings_maker.make_setting('rc_cache.cache_general.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_general.db'))
 
     # cache_perms
     settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
-    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms.db'))
+    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms_db'))
 
     # cache_repo
     settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
-    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo.db'))
+    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo_db'))
 
     # cache_license
     settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
-    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license.db'))
+    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license_db'))
 
     # cache_repo_longterm memory, 96H
     settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
 
     # sql_cache_short
     settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
     settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
 
+    # archive_cache
+    settings_maker.make_setting('archive_cache.store_dir', os.path.join(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
+    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
+
     settings_maker.env_expand()
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings

rhodecode/config/patches.py

@@ -1,99 +1,100 b''
 
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Compatibility patches.
 
 Please keep the following principles in mind:
 
 * Keep imports local, so that importing this module does not cause too many
   side effects by itself.
 
 * Try to make patches idempotent, calling them multiple times should not do
   harm. If that is not possible, ensure that the second call explodes.
 
 """
 
 
 def inspect_getargspec():
     """
     Pyramid rely on inspect.getargspec to lookup the signature of
     view functions. This is not compatible with cython, therefore we replace
     getargspec with a custom version.
     Code is inspired by the inspect module from Python-3.4
     """
     import inspect
 
     def _isCython(func):
         """
         Private helper that checks if a function is a cython function.
         """
         return func.__class__.__name__ == 'cython_function_or_method'
 
     def unwrap(func):
         """
         Get the object wrapped by *func*.
 
         Follows the chain of :attr:`__wrapped__` attributes returning the last
         object in the chain.
 
         *stop* is an optional callback accepting an object in the wrapper chain
         as its sole argument that allows the unwrapping to be terminated early
         if the callback returns a true value. If the callback never returns a
         true value, the last object in the chain is returned as usual. For
         example, :func:`signature` uses this to stop unwrapping if any object
         in the chain has a ``__signature__`` attribute defined.
 
         :exc:`ValueError` is raised if a cycle is encountered.
         """
         f = func  # remember the original func for error reporting
         memo = {id(f)}  # Memoise by id to tolerate non-hashable objects
         while hasattr(func, '__wrapped__'):
             func = func.__wrapped__
             id_func = id(func)
             if id_func in memo:
                 raise ValueError('wrapper loop when unwrapping {!r}'.format(f))
             memo.add(id_func)
         return func
 
     def custom_getargspec(func):
         """
         Get the names and default values of a function's arguments.
 
         A tuple of four things is returned: (args, varargs, varkw, defaults).
         'args' is a list of the argument names (it may contain nested lists).
         'varargs' and 'varkw' are the names of the * and ** arguments or None.
         'defaults' is an n-tuple of the default values of the last n arguments.
         """
 
         func = unwrap(func)
 
         if inspect.ismethod(func):
             func = func.im_func
         if not inspect.isfunction(func):
             if not _isCython(func):
                 raise TypeError('{!r} is not a Python or Cython function'
                                 .format(func))
         args, varargs, varkw = inspect.getargs(func.func_code)
         return inspect.ArgSpec(args, varargs, varkw, func.func_defaults)
 
-    inspect.getargspec = custom_getargspec
+    #TODO: fix it and test it on python3.11
+    inspect.getargspec = inspect.getfullargspec #custom_getargspec
 
     return inspect

rhodecode/config/rcextensions/helpers/extract_pre_files.py

@@ -1,147 +1,147 b''
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 us in hooks::
 
     from .helpers import extract_pre_files
     # returns list of dicts with key-val fetched from extra fields
     file_list = extract_pre_files.run(**kwargs)
 
 """
 import re
 import collections
 import json
 
 from rhodecode.lib import diffs
 from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff
 from rhodecode.lib.vcs.backends.git.diff import GitDiff
-from vcsserver.utils import safe_int
+from rhodecode.lib.str_utils import safe_int
 
 
 def get_svn_files(repo, vcs_repo, refs):
     txn_id = refs[0]
     files = []
 
     stdout, stderr = vcs_repo.run_svn_command(
         ['svnlook', 'changed', repo.repo_full_path, '--transaction', txn_id])
 
     svn_op_to_rc_op = {
         'A': 'A',
         'U': 'M',
         'D': 'D',
     }
 
     for entry in stdout.splitlines():
         parsed_entry = {
             'raw_diff': '',
             'filename': '',
             'chunks': [],
             'ops': {},
             'file_size': 0
         }
 
         op = entry[0]
         path = entry[1:].strip()
 
         rc_op = svn_op_to_rc_op.get(op) or '?'
         parsed_entry['filename'] = path
         parsed_entry['operation'] = rc_op
 
         if rc_op in ['A', 'M']:
 
             stdout, stderr = vcs_repo.run_svn_command(
                 ['svnlook', 'filesize', repo.repo_full_path, path, '--transaction', txn_id],
                 _safe=True
             )
 
             if "Path '{}' is not a file".format(path.rstrip('/')) in stderr:
                 # skip dirs
                 continue
 
             parsed_entry['file_size'] = safe_int(stdout.strip()) or 0
 
         files.append(parsed_entry)
 
     return files
 
 
 def get_hg_files(repo, vcs_repo, refs):
     files = []
     return files
 
 
 def get_git_files(repo, vcs_repo, refs):
     files = []
 
     for data in refs:
         # we should now extract commit data
         old_rev = data['old_rev']
         new_rev = data['new_rev']
 
         if '00000000' in old_rev:
             # new branch, we don't need to extract nothing
             return files
 
         git_env = dict(data['git_env'])
 
         cmd = [
             'diff', old_rev, new_rev
         ]
 
         stdout, stderr = vcs_repo.run_git_command(cmd, extra_env=git_env)
         vcs_diff = GitDiff(stdout)
 
-        diff_processor = diffs.DiffProcessor(vcs_diff, format='newdiff')
+        diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff')
         # this is list of dicts with diff information
         # _parsed[0].keys()
         # ['raw_diff', 'old_revision', 'stats', 'original_filename',
         #  'is_limited_diff', 'chunks', 'new_revision', 'operation',
         #  'exceeds_limit', 'filename']
         files = _parsed = diff_processor.prepare()
 
     return files
 
 
 def run(*args, **kwargs):
     from rhodecode.model.db import Repository
 
     vcs_type = kwargs['scm']
     # use temp name then the main one propagated
     repo_name = kwargs.pop('REPOSITORY', None) or kwargs['repository']
 
     repo = Repository.get_by_repo_name(repo_name)
     vcs_repo = repo.scm_instance(cache=False)
 
     files = []
 
     if vcs_type == 'git':
         for rev_data in kwargs['commit_ids']:
             new_environ = dict((k, v) for k, v in rev_data['git_env'])
         files = get_git_files(repo, vcs_repo, kwargs['commit_ids'])
 
     if vcs_type == 'hg':
         for rev_data in kwargs['commit_ids']:
             new_environ = dict((k, v) for k, v in rev_data['hg_env'])
         files = get_hg_files(repo, vcs_repo, kwargs['commit_ids'])
 
     if vcs_type == 'svn':
         files = get_svn_files(repo, vcs_repo, kwargs['commit_ids'])
 
     return files

rhodecode/config/rcextensions/utils.py

@@ -1,199 +1,200 b''
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import os
 import string
 import functools
 import collections
-import urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.parse
+import urllib.error
 
 log = logging.getLogger('rhodecode.' + __name__)
 
 
 class HookResponse(object):
     def __init__(self, status, output):
         self.status = status
         self.output = output
 
     def __add__(self, other):
         other_status = getattr(other, 'status', 0)
         new_status = max(self.status, other_status)
         other_output = getattr(other, 'output', '')
         new_output = self.output + other_output
 
         return HookResponse(new_status, new_output)
 
     def __bool__(self):
         return self.status == 0
 
 
 class DotDict(dict):
 
     def __contains__(self, k):
         try:
             return dict.__contains__(self, k) or hasattr(self, k)
-        except:
+        except Exception:
             return False
 
     # only called if k not found in normal places
     def __getattr__(self, k):
         try:
             return object.__getattribute__(self, k)
         except AttributeError:
             try:
                 return self[k]
             except KeyError:
                 raise AttributeError(k)
 
     def __setattr__(self, k, v):
         try:
             object.__getattribute__(self, k)
         except AttributeError:
             try:
                 self[k] = v
             except:
                 raise AttributeError(k)
         else:
             object.__setattr__(self, k, v)
 
     def __delattr__(self, k):
         try:
             object.__getattribute__(self, k)
         except AttributeError:
             try:
                 del self[k]
             except KeyError:
                 raise AttributeError(k)
         else:
             object.__delattr__(self, k)
 
     def toDict(self):
         return unserialize(self)
 
     def __repr__(self):
         keys = list(self.keys())
         keys.sort()
         args = ', '.join(['%s=%r' % (key, self[key]) for key in keys])
         return '%s(%s)' % (self.__class__.__name__, args)
 
     @staticmethod
     def fromDict(d):
         return serialize(d)
 
 
 def serialize(x):
     if isinstance(x, dict):
         return DotDict((k, serialize(v)) for k, v in x.items())
     elif isinstance(x, (list, tuple)):
         return type(x)(serialize(v) for v in x)
     else:
         return x
 
 
 def unserialize(x):
     if isinstance(x, dict):
         return dict((k, unserialize(v)) for k, v in x.items())
     elif isinstance(x, (list, tuple)):
         return type(x)(unserialize(v) for v in x)
     else:
         return x
 
 
 def _verify_kwargs(func_name, expected_parameters, kwargs):
     """
     Verify that exactly `expected_parameters` are passed in as `kwargs`.
     """
     expected_parameters = set(expected_parameters)
     kwargs_keys = set(kwargs.keys())
     if kwargs_keys != expected_parameters:
         missing_kwargs = expected_parameters - kwargs_keys
         unexpected_kwargs = kwargs_keys - expected_parameters
         raise AssertionError(
             "func:%s: missing parameters: %r, unexpected parameters: %s" %
             (func_name, missing_kwargs, unexpected_kwargs))
 
 
 def has_kwargs(required_args):
     """
     decorator to verify extension calls arguments.
 
     :param required_args:
     """
     def wrap(func):
         def wrapper(*args, **kwargs):
             _verify_kwargs(func.__name__, required_args.keys(), kwargs)
             # in case there's `calls` defined on module we store the data
             maybe_log_call(func.__name__, args, kwargs)
             log.debug('Calling rcextensions function %s', func.__name__)
             return func(*args, **kwargs)
         return wrapper
     return wrap
 
 
 def maybe_log_call(name, args, kwargs):
     from rhodecode.config import rcextensions
     if hasattr(rcextensions, 'calls'):
         calls = rcextensions.calls
         calls[name].append((args, kwargs))
 
 
-def str2bool(_str):
+def str2bool(_str) -> bool:
     """
     returns True/False value from given string, it tries to translate the
     string into boolean
 
     :param _str: string value to translate into boolean
-    :rtype: boolean
-    :returns: boolean from given string
+    :returns: bool from given string
     """
     if _str is None:
         return False
     if _str in (True, False):
         return _str
     _str = str(_str).strip().lower()
     return _str in ('t', 'true', 'y', 'yes', 'on', '1')
 
 
 def aslist(obj, sep=None, strip=True):
     """
     Returns given string separated by sep as list
 
     :param obj:
     :param sep:
     :param strip:
     """
     if isinstance(obj, (str,)):
         lst = obj.split(sep)
         if strip:
             lst = [v.strip() for v in lst]
         return lst
     elif isinstance(obj, (list, tuple)):
         return obj
     elif obj is None:
         return []
     else:
         return [obj]
 
 
 class UrlTemplate(string.Template):
 
     def safe_substitute(self, **kws):
         # url encode the kw for usage in url
         kws = {k: urllib.parse.quote(str(v)) for k, v in kws.items()}
         return super(UrlTemplate, self).safe_substitute(**kws)

rhodecode/config/settings_maker.py

@@ -1,204 +1,167 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import textwrap
 import string
 import functools
 import logging
 import tempfile
 import logging.config
+from rhodecode.lib.type_utils import str2bool, aslist
+
 log = logging.getLogger(__name__)
 
+
 # skip keys, that are set here, so we don't double process those
 set_keys = {
     '__file__': ''
 }
 
 
-def str2bool(_str):
-    """
-    returns True/False value from given string, it tries to translate the
-    string into boolean
-
-    :param _str: string value to translate into boolean
-    :rtype: boolean
-    :returns: boolean from given string
-    """
-    if _str is None:
-        return False
-    if _str in (True, False):
-        return _str
-    _str = str(_str).strip().lower()
-    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
-
-
-def aslist(obj, sep=None, strip=True):
-    """
-    Returns given string separated by sep as list
-
-    :param obj:
-    :param sep:
-    :param strip:
-    """
-    if isinstance(obj, str):
-        if obj in ['', ""]:
-            return []
-
-        lst = obj.split(sep)
-        if strip:
-            lst = [v.strip() for v in lst]
-        return lst
-    elif isinstance(obj, (list, tuple)):
-        return obj
-    elif obj is None:
-        return []
-    else:
-        return [obj]
-
-
 class SettingsMaker(object):
 
     def __init__(self, app_settings):
         self.settings = app_settings
 
     @classmethod
     def _bool_func(cls, input_val):
         return str2bool(input_val)
 
     @classmethod
     def _int_func(cls, input_val):
         return int(input_val)
 
     @classmethod
+    def _float_func(cls, input_val):
+        return float(input_val)
+
+    @classmethod
     def _list_func(cls, input_val, sep=','):
         return aslist(input_val, sep=sep)
 
     @classmethod
     def _string_func(cls, input_val, lower=True):
         if lower:
             input_val = input_val.lower()
         return input_val
 
     @classmethod
-    def _float_func(cls, input_val):
-        return float(input_val)
-
-    @classmethod
     def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
 
         # ensure we have our dir created
         if not os.path.isdir(input_val) and ensure_dir:
             os.makedirs(input_val, mode=mode)
 
         if not os.path.isdir(input_val):
             raise Exception('Dir at {} does not exist'.format(input_val))
         return input_val
 
     @classmethod
     def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
         dirname = os.path.dirname(input_val)
         cls._dir_func(dirname, ensure_dir=ensure_dir)
         return input_val
 
     @classmethod
     def _key_transformator(cls, key):
         return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
 
     def maybe_env_key(self, key):
         # now maybe we have this KEY in env, search and use the value with higher priority.
         transformed_key = self._key_transformator(key)
         envvar_value = os.environ.get(transformed_key)
         if envvar_value:
             log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
 
         return envvar_value
 
     def env_expand(self):
         replaced = {}
         for k, v in self.settings.items():
             if k not in set_keys:
                 envvar_value = self.maybe_env_key(k)
                 if envvar_value:
                     replaced[k] = envvar_value
                     set_keys[k] = envvar_value
 
         # replace ALL keys updated
         self.settings.update(replaced)
 
     def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
         """
         Helper to enable debug on running instance
         :return:
         """
 
         if not str2bool(self.settings.get('logging.autoconfigure')):
             log.info('logging configuration based on main .ini file')
             return
 
         if logging_conf is None:
             logging_conf = self.settings.get('logging.logging_conf_file') or ''
 
         if not os.path.isfile(logging_conf):
             log.error('Unable to setup logging based on %s, '
                       'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
             return
 
-        with open(logging_conf, 'rb') as f:
+        with open(logging_conf, 'rt') as f:
             ini_template = textwrap.dedent(f.read())
             ini_template = string.Template(ini_template).safe_substitute(
                 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
                 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
             )
 
         with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
             log.info('Saved Temporary LOGGING config at %s', f.name)
             f.write(ini_template)
 
         logging.config.fileConfig(f.name)
         os.remove(f.name)
 
     def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
         input_val = self.settings.get(key, default)
 
         if default_when_empty and not input_val:
             # use default value when value is set in the config but it is empty
             input_val = default
 
         parser_func = {
             'bool': self._bool_func,
             'int': self._int_func,
+            'float': self._float_func,
             'list': self._list_func,
             'list:newline': functools.partial(self._list_func, sep='/n'),
             'list:spacesep': functools.partial(self._list_func, sep=' '),
             'string': functools.partial(self._string_func, lower=lower),
             'dir': self._dir_func,
             'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
             'file': self._file_path_func,
             'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
             None: lambda i: i
         }[parser]
 
         envvar_value = self.maybe_env_key(key)
         if envvar_value:
             input_val = envvar_value
             set_keys[key] = input_val
 
         self.settings[key] = parser_func(input_val)
         return self.settings[key]

rhodecode/config/utils.py

@@ -1,103 +1,119 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import platform
 
 from rhodecode.model import init_model
 
 
 def configure_vcs(config):
     """
     Patch VCS config with some RhodeCode specific stuff
     """
     from rhodecode.lib.vcs import conf
     import rhodecode.lib.vcs.conf.settings
 
     conf.settings.BACKENDS = {
         'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
         'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
         'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
     }
 
     conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol']
     conf.settings.HOOKS_HOST = config['vcs.hooks.host']
     conf.settings.HOOKS_DIRECT_CALLS = config['vcs.hooks.direct_calls']
     conf.settings.DEFAULT_ENCODINGS = config['default_encoding']
     conf.settings.ALIASES[:] = config['vcs.backends']
     conf.settings.SVN_COMPATIBLE_VERSION = config['vcs.svn.compatible_version']
 
 
 def initialize_database(config):
     from rhodecode.lib.utils2 import engine_from_config, get_encryption_key
     engine = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(engine, encryption_key=get_encryption_key(config))
 
 
 def initialize_test_environment(settings, test_env=None):
     if test_env is None:
         test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
 
     from rhodecode.lib.utils import (
         create_test_directory, create_test_database, create_test_repositories,
         create_test_index)
     from rhodecode.tests import TESTS_TMP_PATH
     from rhodecode.lib.vcs.backends.hg import largefiles_store
     from rhodecode.lib.vcs.backends.git import lfs_store
 
     # test repos
     if test_env:
         create_test_directory(TESTS_TMP_PATH)
         # large object stores
         create_test_directory(largefiles_store(TESTS_TMP_PATH))
         create_test_directory(lfs_store(TESTS_TMP_PATH))
 
         create_test_database(TESTS_TMP_PATH, settings)
         create_test_repositories(TESTS_TMP_PATH, settings)
         create_test_index(TESTS_TMP_PATH, settings)
 
 
 def get_vcs_server_protocol(config):
     return config['vcs.server.protocol']
 
 
 def set_instance_id(config):
     """
     Sets a dynamic generated config['instance_id'] if missing or '*'
     E.g instance_id = *cluster-1 or instance_id = *
     """
 
     config['instance_id'] = config.get('instance_id') or ''
     instance_id = config['instance_id']
     if instance_id.startswith('*') or not instance_id:
         prefix = instance_id.lstrip('*')
         _platform_id = platform.uname()[1] or 'instance'
         config['instance_id'] = '{prefix}uname:{platform}-pid:{pid}'.format(
             prefix=prefix,
             platform=_platform_id,
             pid=os.getpid())
 
 
 def get_default_user_id():
-    from rhodecode.model.db import User, Session
-    user_id = Session()\
-        .query(User.user_id)\
-        .filter(User.username == User.DEFAULT_USER)\
-        .scalar()
+    DEFAULT_USER = 'default'
+    from sqlalchemy import text
+    from rhodecode.model import meta
+
+    engine = meta.get_engine()
+    with meta.SA_Session(engine) as session:
+        result = session.execute(text("SELECT user_id from users where username = :uname"), {'uname': DEFAULT_USER})
+        user_id = result.first()[0]
+
     return user_id
+
+
+def get_default_base_path():
+    from sqlalchemy import text
+    from rhodecode.model import meta
+
+    engine = meta.get_engine()
+    with meta.SA_Session(engine) as session:
+        result = session.execute(text("SELECT ui_value from rhodecode_ui where ui_key = '/'"))
+        base_path = result.first()[0]
+
+    return base_path